Thursday, December 13, 2012

blpapi Client -- Java Example

In a previous post, we have shown how to implement blpapi and create an application that listens on port 8195. We now try to show how to utilize that app.

The client-server communication is via TCP/IP and exchange information using XML.

The server expects request schema:

<anybeginningtag>
     <Instrument>
     </Instrument>
     <Fields>
     </Fields>
     <Overrides>
          <Override></Override>
     </Overrides>
</anybeginningtag>

In return, expect response schema:

<BBCommAdapterResponse>
     <FLDS1></FLDS1>
     <FLDS2></FLDS2>
</BBCommAdapterResponse>

FLDS1, FLDS2 are the Bloomberg FLDS field names you have requested for in <Fields></Fields>.

Java Example

Create a model for which you will be holding values.

package scratch;
public class SampleModel {
 private String accDays;
 private String aiCurrent;
 private String price;
 private String yield;

 public String getAccDays() {
  return accDays;
 }
 public void setAccDays(String accDays) {
  this.accDays = accDays;
 }
 public String getAiCurrent() {
  return aiCurrent;
 }
 public void setAiCurrent(String aiCurrent) {
  this.aiCurrent = aiCurrent;
 }
 public String getPrice() {
  return price;
 }
 public void setPrice(String price) {
  this.price = price;
 }
 public String getYield() {
  return yield;
 }
 public void setYield(String yield) {
  this.yield = yield;
 }

 public void printContents() {
  StringBuilder sb = new StringBuilder();
  sb.append("accDays: " + accDays + "\t ");
  sb.append("aiCurrent: " + aiCurrent + "\t ");
  sb.append("price: " + price + "\t ");
  sb.append("yield: " + yield);
  System.out.println(sb.toString());
 }
}


Next, create the client.

package scratch;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.net.InetAddress;
import java.net.Socket;
import java.net.UnknownHostException;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;


public class ClientSocketProcessor extends DefaultHandler {

 private final int hostPort = 8195;
 private Socket connection;

 private SampleModel model;
 private String tmpValue;

 @Override
 public void startElement(String uri, String localName, String qName,
   Attributes attributes) throws SAXException {

  // the response from BBCommAdapter is enclosed with Element tag
  // <BBCommAdapterResponse></BBCommAdapterResponse>
  if (qName.equalsIgnoreCase("BBCommAdapterResponse")) {
   model = new SampleModel();
  }

 }
 @Override
 public void endElement(String uri, String localName, String qName)
   throws SAXException {
  // depending on what your model is
  // the Element tags are Bloomberg FLDS field names

  if (qName.equalsIgnoreCase("DAYS_ACC")) {
   model.setAccDays(tmpValue);
  }

  if (qName.equalsIgnoreCase("INT_ACC")) {
   model.setAiCurrent(tmpValue);
  }

  if (qName.equalsIgnoreCase("PX_ASK")) {
   model.setPrice(tmpValue);
  }

  if (qName.equalsIgnoreCase("YLD_CNV_ASK")) {
   model.setYield(tmpValue);
  }
 }

 @Override
 public void characters(char ch[], int start, int length)
   throws SAXException {
  tmpValue = new String(ch, start, length);
 }

 private void parseXMLAndSetValues(String xmlString) {
  System.out.println("XML: " + xmlString);
  InputSource inputSource = new InputSource();
  inputSource.setCharacterStream(new StringReader(xmlString));

  SAXParserFactory factory = SAXParserFactory.newInstance();
  try {
   SAXParser parser = factory.newSAXParser();
   XMLReader reader = parser.getXMLReader();
   reader.setContentHandler(this);
   reader.parse(inputSource);
 
  } catch (ParserConfigurationException e) {
   System.out.println("ParserConfigurationException encountered in ClientSocketProcessor::parseXMLAndSetValues.");
  } catch (SAXException e) {
   System.out.println("SAXException encountered in ClientSocketProcessor::parseXMLAndSetValues.");
  } catch (IOException e) {
   System.out.println("IOException encountered in ClientSocketProcessor::parseXMLAndSetValues.");
  }
 }

 private void transformString(BufferedReader socketReader) {
  StringBuilder sb = new StringBuilder();
  String response = null;
  try {
   while((response = socketReader.readLine()) != null) {
    sb.append(response);
   }
   // remember to disconnect
   socketReader.close();
  } catch (IOException e) {
   System.out.println("IOException encountered in ClientSocketProcessor::transformString.");
  }

  if (sb.toString().length() == 0) {
   return;
  } else {
   parseXMLAndSetValues(sb.toString());
  }
 }

 private String createSimpleRequest(String instrument) {
  StringBuilder sb = new StringBuilder();

  sb.append("<SimpleRequestTag>");

  sb.append("<Instrument>");
  sb.append(instrument);
  sb.append("</Instrument>");

  sb.append("<Fields>");
  sb.append("DAYS_ACC,INT_ACC");
  sb.append("</Fields>");
  sb.append("</SimpleRequestTag>");

  return sb.toString();
 }

 private String createRequestWithOverride1(String instrument, String settlementDate) {
  StringBuilder sb = new StringBuilder();

  sb.append("<RequestWithOverrideTag>");

  sb.append("<Instrument>");
  sb.append(instrument);
  sb.append("</Instrument>");

  sb.append("<Fields>");
  sb.append("DAYS_ACC,INT_ACC");
  sb.append("</Fields>");

  sb.append("<Overrides>");
  sb.append("<Override>");
  sb.append("SETTLE_DT," + settlementDate);
  sb.append("</Override>");
  sb.append("</Overrides>");

  sb.append("</RequestWithOverrideTag>");

  return sb.toString();
 }

 private String createRequestWithOverride2(String instrument, String settlementDate, String price) {
  StringBuilder sb = new StringBuilder();

  sb.append("<RequestWithOverrideTag>");

  sb.append("<Instrument>");
  sb.append(instrument);
  sb.append("</Instrument>");

  sb.append("<Fields>");
  sb.append("YLD_CNV_ASK");
  sb.append("</Fields>");

  sb.append("<Overrides>");

  sb.append("<Override>");
  sb.append("PX_ASK," + price);
  sb.append("</Override>");

  sb.append("<Override>");
  sb.append("SETTLE_DT," + settlementDate);
  sb.append("</Override>");

  sb.append("</Overrides>");

  sb.append("</RequestWithOverrideTag>");

  return sb.toString();
 }

 private String createRequestWithOverride3(String instrument, String settlementDate, String yield) {
  StringBuilder sb = new StringBuilder();

  sb.append("<RequestWithOverrideTag>");

  sb.append("<Instrument>");
  sb.append(instrument);
  sb.append("</Instrument>");

  sb.append("<Fields>");
  sb.append("PX_ASK");
  sb.append("</Fields>");

  sb.append("<Overrides>");

  sb.append("<Override>");
  sb.append("YLD_CNV_ASK," + yield);
  sb.append("</Override>");

  sb.append("<Override>");
  sb.append("SETTLE_DT," + settlementDate);
  sb.append("</Override>");

  sb.append("</Overrides>");

  sb.append("</RequestWithOverrideTag>");

  return sb.toString();
 }

 private SampleModel doSimpleRequest(String instrument) {
  String requestString = createSimpleRequest(instrument);

  try {
   InetAddress host = InetAddress.getLocalHost();
   String hostIp = host.getHostAddress();
   connection = new Socket(hostIp, hostPort);
   connection.setTcpNoDelay(true);
 
   // write text to stream
   BufferedWriter socketWriter = new BufferedWriter(new OutputStreamWriter(connection.getOutputStream()));
   socketWriter.write(requestString);
   socketWriter.flush();
 
   // read text from stream
   BufferedReader socketReader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
   transformString(socketReader);
 
  } catch (UnknownHostException e) {
   System.out.println("UnknownHostException encountered in ClientSocketProcessor::doSimpleRequest.");
  } catch (IOException e) {
   System.out.println("IOException encountered in ClientSocketProcessor::doSimpleRequest.");
  }
  return model;
 }

 private SampleModel doRequestWithOverride1(String instrument, String settlementDate) {
  String requestString = createRequestWithOverride1(instrument, settlementDate);
  try {
   InetAddress host = InetAddress.getLocalHost();
   String hostIp = host.getHostAddress();
   connection = new Socket(hostIp, hostPort);
   connection.setTcpNoDelay(true);
 
   // write text to stream
   BufferedWriter socketWriter = new BufferedWriter(new OutputStreamWriter(connection.getOutputStream()));
   socketWriter.write(requestString);
   socketWriter.flush();
 
   // read text from stream
   BufferedReader socketReader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
   transformString(socketReader);
 
  } catch (UnknownHostException e) {
   System.out.println("UnknownHostException encountered in ClientSocketProcessor::doRequestWithOverride1.");
  } catch (IOException e) {
   System.out.println("IOException encountered in ClientSocketProcessor::doRequestWithOverride1.");
  }
  return model;
 }

 private SampleModel doRequestWithOverride2(String instrument, String settlementDate, String price) {
  String requestString = createRequestWithOverride2(instrument, settlementDate, price);
  try {
   InetAddress host = InetAddress.getLocalHost();
   String hostIp = host.getHostAddress();
   connection = new Socket(hostIp, hostPort);
   connection.setTcpNoDelay(true);
 
   // write text to stream
   BufferedWriter socketWriter = new BufferedWriter(new OutputStreamWriter(connection.getOutputStream()));
   socketWriter.write(requestString);
   socketWriter.flush();
 
   // read text from stream
   BufferedReader socketReader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
   transformString(socketReader);
 
  } catch (UnknownHostException e) {
   System.out.println("UnknownHostException encountered in ClientSocketProcessor::doRequestWithOverride2.");
  } catch (IOException e) {
   System.out.println("IOException encountered in ClientSocketProcessor::doRequestWithOverride2.");
  }
  return model;
 }

 private SampleModel doRequestWithOverride3(String instrument, String settlementDate, String yield) {
  String requestString = createRequestWithOverride3(instrument, settlementDate, yield);
  try {
   InetAddress host = InetAddress.getLocalHost();
   String hostIp = host.getHostAddress();
   connection = new Socket(hostIp, hostPort);
   connection.setTcpNoDelay(true);
 
   // write text to stream
   BufferedWriter socketWriter = new BufferedWriter(new OutputStreamWriter(connection.getOutputStream()));
   socketWriter.write(requestString);
   socketWriter.flush();
 
   // read text from stream
   BufferedReader socketReader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
   transformString(socketReader);
 
  } catch (UnknownHostException e) {
   System.out.println("UnknownHostException encountered in ClientSocketProcessor::doRequestWithOverride3.");
  } catch (IOException e) {
   System.out.println("IOException encountered in ClientSocketProcessor::doRequestWithOverride3.");
  }
  return model;
 }

 public static void main(String[] args) {
  ClientSocketProcessor processor = new ClientSocketProcessor();
  SampleModel model1 = processor.doSimpleRequest("IDG000010406 Corp");
  SampleModel model2 = processor.doRequestWithOverride1("JPM 3.25 09/23/22 Corp", "20121221");
  SampleModel model3 = processor.doRequestWithOverride2("EJ3726316 Corp", "20121228", "99.23");
  SampleModel model4 = processor.doRequestWithOverride3("RPGB 8 07/31 Corp", "20121228", "7.12345");
  model1.printContents();
  model2.printContents();
  model3.printContents();
  model4.printContents();
 }
}


When this code is run, it should give below results:

XML: <BBCommAdapterResponse><DAYS_ACC>33</DAYS_ACC><INT_ACC>0.6039</INT_ACC></BBCommAdapterResponse>
XML: <BBCommAdapterResponse><DAYS_ACC>87</DAYS_ACC><INT_ACC>0.78541667</INT_ACC></BBCommAdapterResponse>
XML: <BBCommAdapterResponse><YLD_CNV_ASK>3.3429092</YLD_CNV_ASK></BBCommAdapterResponse>
XML: <BBCommAdapterResponse><PX_ASK>108.940976195453</PX_ASK></BBCommAdapterResponse>
accDays: 33  aiCurrent: 0.6039  price: null  yield: null
accDays: 87  aiCurrent: 0.78541667  price: null  yield: null
accDays: null  aiCurrent: null  price: null  yield: 3.3429092
accDays: null  aiCurrent: null  price: 108.940976195453  yield: null

Writing a Server Socket for Bloomberg's blpapi

Problem:

You have several internal applications (Java, Python, Excel, etc.) that need prices from Bloomberg. What is an ideal way to deal with this?

Solution:

Use Bloomberg's blpapi and create an application that listens to a particular port.

Walkthrough of Implementation:

Bloomberg's blpapi now come in 4 flavors - Java, .NET, C/C++, Python. A prerequisite in being able to talk to Bloomberg servers is having a Bloomberg terminal installed in the local machine. These Bloomberg terminals are naturally Windows only applications and therefore our language of choice is .NET.

I started from scratch. I downloaded the latest blpapi from here. I also downloaded VS 2012 Express for Windows Desktop here.

Next, I created a windows application project (not a console application). Why? Here are a few reasons:
  • I want this application to have a single Form in order for me to diagnose.
  • I don't want my end-user to see a console lying around the desktop.
  • I also want this Form to live in the system tray once the application has started up. I only bring it up when I want to diagnose.
Using the PROJECT menu:
  • Add Bloomberglp.Blpapi.dll via Add References.
  • Install log4net package via Manage NuGet Packages. 
 Now we are ready to code.

Start by writing the class which will handle all the requests.

using Event = Bloomberglp.Blpapi.Event;
using Element = Bloomberglp.Blpapi.Element;
using InvalidRequestException = Bloomberglp.Blpapi.InvalidRequestException;
using Message = Bloomberglp.Blpapi.Message;
using Name = Bloomberglp.Blpapi.Name;
using Request = Bloomberglp.Blpapi.Request;
using Service = Bloomberglp.Blpapi.Service;
using Session = Bloomberglp.Blpapi.Session;
using SessionOptions = Bloomberglp.Blpapi.SessionOptions;
using Identity = Bloomberglp.Blpapi.Identity;
using CorrelationID = Bloomberglp.Blpapi.CorrelationID;
using TraceLevel = System.Diagnostics.TraceLevel;
using Datatype = Bloomberglp.Blpapi.Schema.Datatype;
using String = System.String;
using ArrayList = System.Collections.ArrayList;
using Math = System.Math;
using Exception = System.Exception;
using System.Collections.Generic;
using System.Globalization;
using System.Text;
using System.Xml;
using System.IO;
using log4net;

namespace your.choice.of.namespace
{
    /// <summary>
    /// Class that takes care of direct communication with Bloomberg's blpapi.
    /// </summary>
    [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1001:TypesThatOwnDisposableFieldsShouldBeDisposable")]
    class BloombergRequestHandler
    {

        private static readonly ILog log = log4net.LogManager.GetLogger(typeof(BloombergRequestHandler));
        protected Session session;
        protected int uuid { get; set; }
        protected String ipAddress { get; set; }

        private const String INPUT_INSTRUMENT = "instrument";
        private const String INPUT_FIELDS = "fields";
        private const String INPUT_OVERRIDES = "overrides";
        private const String INPUT_OVERRIDE = "override";

        private static readonly Name SECURITIES = new Name("securities");
        private static readonly Name FIELDS = new Name("fields");
        private static readonly Name SECURITY_DATA = new Name("securityData");
        private static readonly Name FIELD_DATA = new Name("fieldData");
        private static readonly Name SECURITY = new Name("security");
        private static readonly Name FIELD_ID = new Name("fieldId");
        private static readonly Name VALUE = new Name("value");
        private static readonly Name OVERRIDES = new Name("overrides");
        private static readonly Name EIDDATA = new Name("eidData");

        private const String SESSION_TERMINATED = "SessionTerminated";
        private const String SESSION_STARTUP_FAILURE = "SessionStartupFailure";

        protected const String REFDATASERVICE = "//blp/refdata";
        protected const String REFDATAREQUEST = "ReferenceDataRequest";

        protected const String HOST = "localhost";
        protected const int PORT = 8194;

        public void openServices()
        {
            try
            {
                log.Info("Opening //blp/refdata service...");
                session.OpenService(REFDATASERVICE);
                log.Info("//blp/refdata service opened.");
            }
            catch (Exception)
            {
                log.Fatal("Failed opening //blp/refdata service.");
                log.Fatal("Application was launched on a machine without Bloomberg Communication Process installed.");
                log.Fatal("Please make sure that either Desktop API or Server API is installed.");
            }

        }
        public virtual void startSession()
        {
            SessionOptions sessionOptions = new SessionOptions();
            sessionOptions.ServerHost = HOST;
            sessionOptions.ServerPort = PORT;
            session = new Session(sessionOptions);

            try
            {
                log.Info("Starting session...");
                session.Start();
                log.Info("Session started.");
            }
            catch (Exception)
            {
                log.Error("Failure in startSession");
            }
        }

        public void killSession()
        {
            if (session != null)
            {
                try
                {
                    session.Stop();
                }
                catch (Exception)
                {
                    session = null;
                    log.Error("Failure in killSession");
                }
            }
        }

        public BloombergRequestHandler()
        {
            startSession();
            openServices();
        }

        private String generateResponseXMLNode(String tag, String value)
        {
            StringBuilder sb = new StringBuilder();
            sb.Append("<");
            sb.Append(tag);
            sb.Append(">");
            sb.Append(value);
            sb.Append("</");
            sb.Append(tag);
            sb.Append(">");
            return sb.ToString();
        }

        private void processScalarField(StringBuilder sb, Element secField)
        {
            log.Info("Processing scalar RefData " + secField.Name + " = " + secField.GetValueAsString());
            sb.Append(generateResponseXMLNode(secField.Name.ToString(), secField.GetValueAsString()));
        }

        private void processVectorField(StringBuilder sb, Element secField)
        {
            log.Info("Processing vector RefData " + secField.Name);
            sb.Append("<");
            sb.Append(secField.Name.ToString());
            sb.Append(">");
            int numValues = secField.NumValues;
            for (int bvCtr = 0; bvCtr < numValues; bvCtr++)
            {
                Element bulkElement = secField.GetValueAsElement(bvCtr);
                int numBulkValues = bulkElement.NumElements;
                sb.Append("<");
                sb.Append(bulkElement.Name.ToString());
                sb.Append(">");
                for (int beCtr = 0; beCtr < numBulkValues; beCtr++)
                {
                    Element elem = bulkElement.GetElement(beCtr);
                    sb.Append(generateResponseXMLNode(elem.Name.ToString(), elem.GetValueAsString()));
                }
                sb.Append("</");
                sb.Append(bulkElement.Name.ToString());
                sb.Append(">");
            }
            sb.Append("</");
            sb.Append(secField.Name.ToString());
            sb.Append(">");
        }

        private void processResponseEvent(Event eventObj, StringBuilder sb)
        {
            foreach (Message msg in eventObj)
            {
                Element secDataArray = msg.GetElement(SECURITY_DATA);
                int numSecDataArray = secDataArray.NumValues;
                for (int i = 0; i < numSecDataArray; ++i)
                {
                    Element secData = secDataArray.GetValueAsElement(i);
                    String security = secData.GetElementAsString(SECURITY);
                    log.Info("Processing results for : " + security);
                    if (secData.HasElement(FIELD_DATA))
                    {
                        Element secFields = secData.GetElement(FIELD_DATA);
                        if (secFields.NumElements > 0)
                        {
                            int numFields = secFields.NumElements;
                            for (int j = 0; j < numFields; ++j)
                            {
                                Element secField = secFields.GetElement(j);
                                if (secField.Datatype.Equals(Datatype.SEQUENCE))
                                {
                                    processVectorField(sb, secField);
                                }
                                else
                                {
                                    processScalarField(sb, secField);
                                }
                            }
                        }
                    }
                }
            }
        }

        private void eventLoop(StringBuilder sb)
        {
            bool done = false;
            while (!done)
            {
                Event eventObj = session.NextEvent();
                if (eventObj.Type.Equals(Event.EventType.PARTIAL_RESPONSE))
                {
                    processResponseEvent(eventObj, sb);
                }
                else if (eventObj.Type.Equals(Event.EventType.RESPONSE))
                {
                    processResponseEvent(eventObj, sb);
                    done = true;
                }
                else
                {
                    foreach (Message msg in eventObj)
                    {
                        if (eventObj.Type.Equals(Event.EventType.SESSION_STATUS))
                        {
                            if (msg.MessageType.Equals(SESSION_TERMINATED) || msg.MessageType.Equals(SESSION_STARTUP_FAILURE))
                            {
                                done = true;
                            }
                        }
                    }
                }
            }
        }

        private String sendRefDataRequest(String instrument, ArrayList paramFields, Dictionary<String, object> paramOverrides)
        {
            StringBuilder sb = new StringBuilder();
            if (session != null)
            {

                try
                {
                    Service refDataService = session.GetService(REFDATASERVICE);
                    Request request = refDataService.CreateRequest(REFDATAREQUEST);

                    // add securities to request
                    Element securities = request.GetElement(SECURITIES);
                    securities.AppendValue(instrument);

                    // add fields to request
                    Element fields = request.GetElement(FIELDS);
                    foreach (String field in paramFields)
                    {
                        fields.AppendValue(field.ToString());
                    }

                    // add scalar overrides
                    Element overrides = request.GetElement(OVERRIDES);
                    foreach (KeyValuePair<String, object> pair in paramOverrides)
                    {
                        Element ovr = overrides.AppendElement();
                        ovr.SetElement(FIELD_ID, pair.Key);
                        ovr.SetElement(VALUE, pair.Value.ToString());
                    }

                    session.SendRequest(request, null);
                    eventLoop(sb);
                }
                catch (Exception)
                {
                    // nothing further
                }
            }

            return sb.ToString();
        }

        private Dictionary<String, object> generateInputs(String inputXML)
        {
            Dictionary<String, object> inputs = new Dictionary<String, object>();
            Dictionary<String, object> overrides = new Dictionary<String, object>();
            log.Info("XML: " + inputXML);

            XmlDocument doc = new XmlDocument();
            doc.LoadXml(inputXML);
            XmlElement root = doc.DocumentElement;

            foreach(XmlElement e in root){
                if (e.Name.ToLower().Equals(INPUT_INSTRUMENT)) {
                    inputs.Add(INPUT_INSTRUMENT, e.InnerText);
                }
            }

            foreach (XmlElement e in root)
            {
                if (e.Name.ToLower().Equals(INPUT_FIELDS))
                {
                    ArrayList inputFields = new ArrayList();
                    String[] fields = e.InnerText.Split(',');
                    foreach (String sField in fields)
                    {
                        inputFields.Add(sField);
                    }
                    inputs.Add(INPUT_FIELDS, inputFields);
                }
            }

            foreach (XmlElement e in root)
            {
                if (e.Name.ToLower().Equals(INPUT_OVERRIDES))
                {
                    foreach (XmlElement inner in e)
                    {
                        if (inner.Name.ToLower().Equals(INPUT_OVERRIDE))
                        {
                            String[] fields = inner.InnerText.Split(',');
                            overrides.Add(fields[0], fields[1]);
                        }
                    }
                }
            }

            inputs.Add(INPUT_OVERRIDES, overrides);
            return inputs;
        }

        /// <summary>
        /// Generates an XML output for apps that gather data with BBCommAdapter.
        /// </summary>
        /// <param name="inputXML">input XML that contains instrument, fields, and/or overrides</param>
        /// <returns>XML string whose tags are the Bloomberg FLDS fields name(s) used in the query</returns>
        public String generateBloombergResponse(String inputXML)
        {
            Dictionary<String, object> inputs = generateInputs(inputXML);

            String instrument = null;
            object oInstrument;
            if (inputs.TryGetValue(INPUT_INSTRUMENT, out oInstrument))
            {
                instrument = (String)oInstrument;
            }

            ArrayList fields = new ArrayList();
            object oFields;
            if (inputs.TryGetValue(INPUT_FIELDS, out oFields))
            {
                fields = (ArrayList)oFields;
            }

            Dictionary<String, object> overrides = new Dictionary<String, object>();
            object oOverrides;
            if (inputs.TryGetValue(INPUT_OVERRIDES, out oOverrides))
            {
                overrides = (Dictionary<String, object>)oOverrides;
            }

            String result = sendRefDataRequest(instrument, fields, overrides);
            killSession();

            // finalize the XML response
            StringBuilder sb = new StringBuilder();
            sb.Append("<BBCommAdapterResponse>");
            sb.Append(result);
            sb.Append("</BBCommAdapterResponse>");

            return sb.ToString();
        }
}

Next, write the server socket.

using System;
using System.Text;
using System.Threading;
using System.Net;
using System.Net.Sockets;
using System.IO;
using System.Xml;
using log4net;
using your.choice.of.namespace;

namespace your.choice.of.namespace
{
    /// <summary>
    /// Class that process inbound and outbound communication using port 8195.
    /// Although a new thread is spawned for each TCP/IP connection, the communication is synchronous per thread.
    /// </summary>
    class ServerSocketProcessor
    {
        private static readonly ILog log = log4net.LogManager.GetLogger(typeof(ServerSocketProcessor));

        private const int maxConnections = 10;
        private const int listenPort = 8195;
        private Thread listenerThread;
        private TcpListener serverSocket;
        private IPAddress serverIpAddress;
        private TcpClient client;
        private String clientIpAddress;

        public ServerSocketProcessor()
        {
            try
            {
                IPHostEntry ipHostInfo = Dns.GetHostEntry(Dns.GetHostName());
                serverIpAddress = ipHostInfo.AddressList[0];
                serverSocket = new TcpListener(serverIpAddress, listenPort);
                listenerThread = new Thread(new ThreadStart(ListenForClients));
                listenerThread.Start();
            }
            catch (Exception)
            {
                log.Error("Unable to set-up a server socket on port: " + listenPort);
            }
        }

        private void HandleClientComm(object client)
        {
            TcpClient tcpClient = (TcpClient)client;
            NetworkStream clientStream = tcpClient.GetStream();

            while (true)
            {

                StringBuilder sb = new StringBuilder();
                byte[] readBuffer = new byte[1024];
                int numBytesRead;

                try
                {
                    numBytesRead = 0;
                    do
                    {
                        numBytesRead = clientStream.Read(readBuffer, 0, readBuffer.Length);
                        sb.Append(Encoding.UTF8.GetString(readBuffer, 0, numBytesRead));
                    } while (clientStream.DataAvailable);
                }
                catch (Exception)
                {
                    log.Error("Client has disconnected.");
                    break;
                }

                if (numBytesRead == 0)
                {
                    log.Info("Client has disconnected.");
                    break;
                }

                String inputXML = sb.ToString();
                UTF8Encoding encoder = new UTF8Encoding();
                byte[] responseBuffer = new byte[4096];
                BloombergRequestHandler handler = new BloombergRequestHandler();
                String result;
                if (serverIpAddress.ToString().Equals(clientIpAddress))
                {
                    log.Info("Processing for allowed client.");
                    result = handler.generateBloombergResponse(inputXML);
                    responseBuffer = encoder.GetBytes(result);
                }
                else
                {
                    log.Info("No processing: Exception caught.");
                    result = "Operation not allowed when client's IP is different from this IP ";
                    responseBuffer = encoder.GetBytes(result);
                }

                // prepare the response
                log.Info("Preparing response...");
                log.Info("Writing response to client stream...");
                clientStream.Write(responseBuffer, 0, responseBuffer.Length);
                log.Info("Flushing stream...");
                clientStream.Flush();
                log.Info("Closing stream...");
                clientStream.Close();
            }
            tcpClient.Close();
        }

        private void ListenForClients()
        {
            this.serverSocket.Start();
            while (true)
            {
                // blocks until a client has connected to server
                client = this.serverSocket.AcceptTcpClient();
                clientIpAddress = client.Client.RemoteEndPoint.ToString().Split(':')[0];
                log.Info("Client has connected...");
                log.Info("Local (Server) IP: " + serverIpAddress + "; Remote (Client) IP: " + clientIpAddress);

                // create a thread to handle communication with connected client
                Thread clientThread = new Thread(new ParameterizedThreadStart(HandleClientComm));
                clientThread.Start(client);
            }
        }
    }
}


Finally, write the entry point of the application.

using System;
using System.Windows.Forms;
using System.Diagnostics;
using log4net;
using log4net.Config;
using your.choice.of.namespace;

namespace your.choice.of.namespace
{
    static class BBCommAdapter
    {
        private static readonly ILog log = log4net.LogManager.GetLogger(typeof(BBCommAdapter));

        public static Process ExistingProcess()
        {
            Process curr = Process.GetCurrentProcess();
            Process[] procs = Process.GetProcessesByName(curr.ProcessName);
            foreach(Process p in procs) {
                if (p.Id != curr.Id && (p.MainModule.FileName == curr.MainModule.FileName)) {
                    return p;
                }
            }
            return null;
        }

        /// <summary>
        /// The main entry point for the application.
        /// </summary>
        [STAThread]
        static void Main()
        {
            log4net.Config.BasicConfigurator.Configure();
            if (ExistingProcess() != null) {
                log.Error("Another instance of BBCommAdapter is already running with pid: " + ExistingProcess().Id);
                return;
            }

            log.Info("Initializing server...");
            ServerSocketProcessor processor = new ServerSocketProcessor();

            Application.EnableVisualStyles();
            Application.SetCompatibleTextRenderingDefault(false);
            BBCommAdapterForm appForm = new BBCommAdapterForm();
           
            log.Info("Starting a fresh instance of BBCommAdapter with pid:" + Process.GetCurrentProcess().Id);
            Application.Run(appForm);

        }
    }
}


As for the Windows Form, add 3 fields, a button, a notifyIcon, and contextMenuStrip. Set the notifyIcon's context menu to contextMenuStrip:

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;
using ArrayList = System.Collections.ArrayList;
using System.Diagnostics;
using System.Runtime.InteropServices;
using your.choice.of.namespace;

namespace your.choice.of.namespace
{
    public partial class BBCommAdapterForm : Form
    {

        // used to disable the (X) button on form
        const int MF_BYPOSITION = 0x400;
        [DllImport("User32")]
        private static extern int RemoveMenu(IntPtr hMenu, int nPosition, int wFlags);
        [DllImport("User32")]
        private static extern IntPtr GetSystemMenu(IntPtr hWnd, bool bRevert);
        [DllImport("User32")]
        private static extern int GetMenuItemCount(IntPtr hWnd);


        public BBCommAdapterForm()
        {
            InitializeComponent();
        }

        private String generateInputXML()
        {
            StringBuilder sb = new StringBuilder();
            sb.Append("<TestConnection>");
            sb.Append("<INSTRUMENT>");
            sb.Append(fldInstrument.Text);
            sb.Append("</INSTRUMENT>");
            sb.Append("<FIELDS>");
            sb.Append(fldField.Text);
            sb.Append("</FIELDS>");
            // only here to complete the actual request XML schema
            //sb.Append("<OVERRIDES>");
            //sb.Append("<OVERRIDE>");
            //sb.Append("OVRFLDSKey1, OVRValue1");
            //sb.Append("</OVERRIDE>");
            //sb.Append("<OVERRIDE>");
            //sb.Append("OVRFLDSKey2, OVRValue2");
            //sb.Append("</OVERRIDE>");
            //sb.Append("</OVERRIDES>");
            //sb.Append("</TestConnection>");
            return sb.ToString();
        }

        private void btnTestConnection_Click(object sender, EventArgs e)
        {
            BloombergRequestHandler handler = new BloombergUnrestrictedRequestHandler();
            String inputXML = generateInputXML();
            String results = handler.generateBloombergResponse(inputXML);
            fldResult.Text = results;

        }
        private void BBCommAdapterForm_Resize(object sender, EventArgs e)
        {
            if (this.WindowState == FormWindowState.Minimized)
            {
                notifyIcon.Visible = true;
                notifyIcon.BalloonTipText = "BBCommAdapter is running in background";
                notifyIcon.BalloonTipTitle = "BBCommAdapter";
                notifyIcon.BalloonTipIcon = ToolTipIcon.Info;
                notifyIcon.ShowBalloonTip(500);
                Hide();
            }
            else if (this.WindowState == FormWindowState.Normal)
            {
                Show();
                notifyIcon.Visible = false;
            }
        }

        private void notifyIcon_MouseDoubleClick(object sender, MouseEventArgs e)
        {
            Show();
            this.WindowState = FormWindowState.Normal;
            notifyIcon.Visible = false;
        }

        private void menuRestore_Click(object sender, EventArgs e)
        {
            Show();
            this.WindowState = FormWindowState.Normal;
            notifyIcon.Visible = false;
        }

        private void menuExit_Click(object sender, EventArgs e)
        {
            if (MessageBox.Show("Close BBCommAdapter application?", "Exit BBCommAdapter", MessageBoxButtons.YesNo) != DialogResult.Yes)
            {
                // do nothing
            }
            else
            {
                // kill itself
                Process[] localByName = Process.GetProcessesByName("BBCommAdapter");
                foreach (Process p in localByName)
                {
                    p.Kill();
                }
            }
        }

        private void BBCommAdapterForm_FormClosing(object sender, FormClosingEventArgs e)
        {
            if (MessageBox.Show("Close BBCommAdapter application?", "Exit BBCommAdapter", MessageBoxButtons.YesNo) != DialogResult.Yes)
            {
                e.Cancel = true;
                notifyIcon.Visible = true;
                notifyIcon.BalloonTipText = "BBCommAdapter is running in background";
                notifyIcon.BalloonTipTitle = "BBCommAdapter";
                notifyIcon.BalloonTipIcon = ToolTipIcon.Info;
                notifyIcon.ShowBalloonTip(500);
                Hide();
            }
            else
            {
                // kill itself
                Process[] localByName = Process.GetProcessesByName("BBCommAdapter");
                foreach (Process p in localByName)
                {
                    p.Kill();
                }
            }
        }

        private void BBCommAdapterForm_Shown(object sender, EventArgs e)
        {
            this.Visible = false;
            this.WindowState = FormWindowState.Minimized;
            notifyIcon.Visible = true;
            notifyIcon.BalloonTipText = "BBCommAdapter is running in background";
            notifyIcon.BalloonTipTitle = "BBCommAdapter";
            notifyIcon.BalloonTipIcon = ToolTipIcon.Info;
        }

        private void BBCommAdapterForm_Load(object sender, EventArgs e)
        {
            IntPtr hMenu = GetSystemMenu(this.Handle, false);
            int menuItemCount = GetMenuItemCount(hMenu);
            RemoveMenu(hMenu, menuItemCount - 1, MF_BYPOSITION);
        }
    }
}

Thursday, December 6, 2012

Building log4cxx with VS 2012 on Windows 7

The ideas in this post come from here. The original post was for VS 2010. I'm posting for VS 2012 on Windows 7. The only important note is that _MSC_VER of VS 2012 is 1700.

Steps:
  1. Download log4cxx ZIP package from http://logging.apache.org/log4cxx/download.html, and extract its contents.
  2. Download latest available apr and apr-util ZIP package from http://apr.apache.org/download.cgi.
  3. unzip apr-1.x.y-win32-src.zip (manually extract the contents)
  4. rename apr-1.x.y apr (rename the folder)
  5. unzip apr-util-1.x.y-win32-src.zip (manually extract the contents)
  6. rename apr-util-1.x.y apr-util (rename the folder)
  7. cd apache-log4cxx-0.10.0
  8. configure
  9. Open apr-util\include\apu.hw.
  10. Find the line starting with "#define APU_HAVE_APR_ICONV".
  11. Change the constant to 0 and save the file.
  12. Open apr-util\include\apr_ldap.hw.
  13. Find the line starting with "#define APR_HAS_LDAP".
  14. Change the constant to 0 and save the file.
  15. Open VS 2012 and open project log4cxx.dsw.
  16. Click yes to whatever VS prompts you with (it has something to do with conversion).
  17. Once the projects have been converted, go to Header Files folder of log4cxx.
  18. Open log4cxx.h within VS.
  19. Change line containing "#if defined(_MSC_VER) && !defined(LOG4CXX_STATIC) && defined(LOG4CXX)" to "#if defined(_MSC_VER) && _MSC_VER < 1700 && !defined(LOG4CXX_STATIC) && defined(LOG4CXX)".
  20. Change line containing "#elif defined(_MSC_VER) && !defined(LOG4CXX_STATIC)" to "#elif defined(_MSC_VER) && _MSC_VER < 1700 && !defined(LOG4CXX_STATIC)".
  21. Build log4cxx.

Tuesday, September 18, 2012

Python Bloomberg API

Abstract

I know Bloomberg has made it's API open to the public. Unfortunately, I use Python and there begins my dilemma. It is not a Bloomberg supported programming language. Speaking of which, after any convesation I have with someone from their Helpdesk, there's nothing much I can do but bang my head on top of my desk.

I have written this document to provide ample information on how to interface with Bloomberg using Python. I have endured a great amount of pain in order to make sure that information here will work for all Bloomberg installations.

Bloomberg.Data.1

This is the legacy way of doing things. It DOESN'T work on all Bloomberg installations. But for completeness sake, the code is as simple as:

import win32com.client
BLP = win32com.client.Dispatch('Bloomberg.Data.1')

V3 API

comtypes, win32com, and ctypes are 3 python modules that I have experimented with. I will only provide information for comtypes and win32com since my tango with ctypes has gone into an infinite loop. According to Bloomberg, the library for V3 API can be accessed via registered or unregistered mode. The modules below will determine which one you have. I have broken down the modules for easy maintenance.

bbg_helper.py


import logger
import win32com.client, win32com.client.gencache
import comtypes.client as COM
import blpapicom2_unregistered as unreg_mode

try:
    logger.info('Trying blpapicom2.Session.1 registered mode (comtypes.client)')
    session = COM.CreateObject('blpapicom2.Session.1')
    logger.info('comtypes.client blpapicom2.Session.1 registered mode is OK')
except Exception as e:
    logger.warn('FAILED IN STARTING blpapicom2.Session.1 WITH MESSAGE: %s', str(e))
    try:
        logger.info('Trying blpapicom.Session registered mode (comtypes.client)')
        session = COM.CreateObject('blpapicom.Session')
        logger.info('comtypes.client blpapicom.Session registered mode is OK')
    except Exception as e:
        logger.warn('FAILED IN STARTING blpapicom.Session WITH MESSAGE: %s', str(e))
        try:
            logger.info('Trying blpapicom2.Session.1 registered mode (win32com.client)')
            session = win32com.client.CastTo(win32com.client.gencache.EnsureDispatch('blpapicom2.Session.1'),'ISession')
            logger.info(win32com.client blpapicom2.Session.1 registered mode is OK)
        except Exception as e:
            logger.warn('FAILED IN STARTING blpapicom2.Session.1 WITH MESSAGE: %s', str(e))
            try:
                logger.info('Trying blpapicom.Session registered mode (win32com.client)')
                session = win32com.client.CastTo(win32com.client.gencache.EnsureDispatch('blpapicom.Session'),'ISession')
                logger.info('win32com.client blpapicom.Session registered mode is OK')
            except Exception as e:
)               logger.warn('FAILED IN STARTING blpapicom.Session WITH MESSAGE: %s', str(e))
                try:
                    session = unreg_mode.BlpCoCreateInstance()
                except:
                    pass

blpapicom2_unregistered.py

import sys
import win32com.client
import comtypes.client as COM
import logger
import blpapicom_unregistered as unreg_mode

# config for Windows XP users
API_XP        = unreg_mode.API2_XP

# config for Windows 7 users
API_WIN7      = unreg_mode.API2_WIN7

def BlpCoCreateInstance():
    '''
        Return a Session object
    '''

    Session = None
    Module = None
    try:
        logger.info('Trying blpapicom2.dll un-registered mode (comtypes.client)')
        if sys.getwindowsversion()[0] < 6: 
            logger.info('Trying for Windows XP config')
            Module = COM.GetModule(API_XP)
        else: 
            logger.info('Trying for Windows 7 config')
            Module = COM.GetModule(API_WIN7)

        if Module is not None:
            Session = COM.CreateObject(Module.Session)
        logger.info('comtypes.client blpapicom2.dll un-registered mode is OK')
    except Exception as e:
        logger.warn('FAILED IN STARTING blpapicom2.dll WITH MESSAGE: %s', str(e))
        try:
            logger.info('Trying blpapicom2.dll un-registered mode (win32com.client)')
            Session = win32com.client.CastTo(win32com.client.Dispatch('blpapicom2.Session'),'ISession')
            logger.info('win32com.client blpapicom2.dll un-registered mode is OK')
        except Exception as e:
            logger.warn('FAILED IN STARTING blpapicom2.dll WITH MESSAGE: %s', str(e))
            try:
                Session = unreg_mode.BlpCoCreateInstance()
            except:
                pass

    return Session

blpapicom_unregistered.py

import sys
import win32com.client
import comtypes.client as COM
import logger
import ctypes_unregistered as unreg_mode

# config for Windows XP users
API2_XP         = unreg_mode.API2_XP
API_XP          = unreg_mode.API_XP

# config for Windows 7 users
API2_WIN7     = unreg_mode.API2_WIN7
API_WIN7      = unreg_mode.API_WIN7

def BlpCoCreateInstance():
    '''
        Return a Session object
    '''
    Session = None
    Module = None
    try:
        logger.info('Trying blpapicom.dll un-registered mode (comtypes.client)')
        if sys.getwindowsversion()[0] < 6: 
            logger.info('Trying for Windows XP config')
            Module = COM.GetModule(API_XP)
        else: 
            logger.info('Trying for Windows 7 config')
            Module = COM.GetModule(API_WIN7)

        if Module is not None:
            Session = COM.CreateObject(Module.Session)
        logger.info('comtypes.client blpapicom.dll un-registered mode is OK')
    except Exception as e:
        logger.warn('FAILED IN STARTING blpapicom.dll WITH MESSAGE: %s', str(e))
        try:
            logger.info('Trying blpapicom.dll un-registered mode (win32com.client)')
            Session = win32com.client.CastTo(win32com.client.Dispatch('blpapicom.Session'),'ISession')
            logger.info('win32com.client blpapicom.dll un-registered mode is OK')
        except Exception as e:
            try:
                Session = unreg_mode.BlpCoCreateInstance()
            except:
                pass

    return Session

ctypes_unregistered.py

import sys
import ctypes
import logger

OLEAUT32 = ctypes.oledll.oleaut32
OLE32    = ctypes.oledll.ole32
OLE32.CoInitialize(0)

# IIDs
RCLSID  = '{9FDBE237-38A5-4d8c-BB9C-2EB55FB1EABE}'
RIID    = '{4AC751C2-BB10-4702-BB05-791D93BB461C}'

BYTE    = ctypes.c_byte
WORD    = ctypes.c_int
DWORD   = ctypes.c_long

# config for Windows XP users
ROOT_XP         = r'C:\Program Files\BLP\API'
LOADER_XP       = ROOT_XP + '\\' + 'bbloaderv3.dll'
API2_XP         = ROOT_XP + '\\' + 'blpapicom2.dll'
API_XP          = ROOT_XP + '\\' + 'blpapicom.dll'

# config for Windows 7 users
ROOT_WIN7     = r'C:\Program Files (x86)\BLP\API'
LOADER_WIN7   = ROOT_WIN7 + '\\' + 'bbloaderv3.dll'
API2_WIN7     = ROOT_WIN7 + '\\' + 'blpapicom2.dll'
API_WIN7      = ROOT_WIN7 + '\\' + 'blpapicom.dll'

def PrintGUIDValue(guid):
    '''
        Return GUID _fields_ values
    '''
    return "Data1: %s; Data2: %s; Data3: %s; Data4: %s" %(str(guid.Data1), str(guid.Data2), str(guid.Data3),"".join(str(guid.Data4)))

class GUID(ctypes.Structure):
    _fields_ = [
        ("Data1", ctypes.c_long),
        ("Data2", ctypes.c_int),
        ("Data3", ctypes.c_int),
        ("Data4", ctypes.c_byte * 8),
        ]

    def __init__(self, name=None):
        if name is not None:
            self._as_parameter = OLE32.CLSIDFromString(unicode(name), ctypes.byref(self))

    def __repr__(self):
        s = (ctypes.c_wchar * 39)()
        OLE32.StringFromGUID2(ctypes.byref(self), s, 39)
        return "guid: %s" % s.value

    def __str__(self):
        s = (ctypes.c_wchar * 39)()
        OLE32.StringFromGUID2(ctypes.byref(self), s, 39)
        return s.value

    def __cmp__(self, other):
        if isinstance(other, GUID):
            return not OLE32.IsEqualGUID(ctypes.byref(self), ctypes.byref(other))
        return -1

    def __nonzero__(self):
        result = str(buffer(self)) != "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
        return result

    def __eq__(self, other, IsEqualGUID=OLE32.IsEqualGUID):
        return isinstance(other, GUID) and IsEqualGUID(ctypes.byref(self), ctypes.byref(other))

    def PrintGUIDValue(self):
        ''' Return GUID _fields_ values '''
        return PrintGUIDValue(self)

def BlpCoCreateInstance():
    '''
        Implementation is incomplete. DO NOT USE!
    '''
    Session = None
    bbloaderv3 = None
    blpapicom2 = None
    if sys.getwindowsversion()[0] < 6:
        bbloaderv3 = ctypes.windll.LoadLibrary(LOADER_XP)
        blpapicom2 = API_XP
    else:
        bbloaderv3 = ctypes.windll.LoadLibrary(LOADER_ISPACE)
        blpapicom2 = API2_WIN7

    rclsidguid = GUID(RCLSID)
    riidguid = GUID(RIID)
    rclsid = ctypes.byref(rclsidguid)
    riid   = ctypes.byref(riidguid)

    logger.info(repr(rclsidguid) + " := " + rclsidguid.PrintGUIDValue())
    logger.info(repr(riidguid) + " := " + riidguid.PrintGUIDValue())
    BlpCoCreateInstance = getattr(bbloaderv3, 'BlpCoCreateInstance')
    BlpCoCreateInstance.restype = ctypes.c_char_p

    bbloaderv3.BlpCoCreateInstance(blpapicom2, rclsid, riid)

    return Session

"""
We are trying to mimic how Bloomberg uses the data control in unregistered mode (VBA).
Below is taken from their WAPI documentation Ref# COMv3RegUnreg

' Please note that the full path must be included to the location of the
' bbloaderv3.dll file. If this path is not correct, please, replace it
' with the correct path

Private Declare Function BlpCoCreateIntance Lib "bbloaderv3.dll" _
    (ByVal lpszFilePathName As String, ByRef rclsid As GUID, riid As GUID) As Object

Private Declare Function CLSIDFromString Lib "ole32.dll" (pstCLS As Long, _
    clsid As GUID) As Long

Private Declare Function GetActiveObject Lib "oleaut32.dll" (lpRclsid As Long, _
    pvReserved As Long, lpUnk As Long) As Long

Private Type GUID
    Data1 As Long
    Data2 As Integer
    Data3 As Integer
    Data4 (7) As Byte
End Type

Dim rclsid As GUID
Dim riid as GUID
Dim FileName As String
Private WithEvents session As Object

Private Function GuidFromStGuid(ByVal stGuid As String) As GUID

    Dim rc As Long

    If Left$(stGuid, 7) = "{guid {" Then
        If Right$(stGuid, 2) = "}}" Then
            stGuid = Mid$(stGuid, 7, 38)
        End If
    End If

    rc = CLSIDFromString(ByVal StrPtr(stGuid), GuidFromStGuid)

End Function

' Instantiate the Bloomberg COM Data Control
Private Sub Class_Initialize()

    FileName = "blpapicom2.dll"
    Set session = BlpCoCreateInstance(FileName, _
        GuidFromStGuid("{guid {9FDBE237-38A5-4d8c-BB9C-2EB55FB1EABE}}"), _
        GuidFromStGuid("{guid {4AC751C2-BB10-4702-BB05-791D93BB461C}}"))

    session.Start

End Sub
"""

Session

The ultimate goal is to get a Session object. Once that is done, sending request should be a breeze.

import datetime

TODAY = datetime.datetime.today()
APIREFDATA_SVC    = '//blp/refdata'
RESPONSE = 5
PARTIAL_RESPONSE = 6

def MakeRequest(security, fldList, overrideFields=None, overrideValues=None):
    '''
        Request test harness
    '''
    result = []
    session.QueueEvents = True
    session.Start()
    session.OpenService(APIREFDATA_SVC)
    service = session.GetService(APIREFDATA_SVC)

    req = service.CreateRequest("ReferenceDataRequest")

    #not sure if this will work
    sec = req.GetElement('securities')
    sec.AppendValue(security)

    flds = req.GetElement('fields')
    for fld in fldList:
        flds.AppendValue(fld)

    if overrideFields:
        overridables = zip(overrideFields, overrideValues)
        overrides = req.GetElement("overrides")
        for overrideField, overrideValue in overridables:
            override = overrides.AppendElment()
            override.SetElement('fieldId', overrideField)
            override.SetElement('value', overrideValue)

    session.SendRequest(req)

    while True:
        eventObj = session.NextEvent()
        if eventObj.EventType == PARTIAL_RESPONSE or eventObj.EventType == RESPONSE:
            it = eventObj.CreateMessageIterator()

            while it.Next():
                msg = it.Message
                numSecurities = msg.GetElement("securityData").NumValues
                for secIndex in xrange(0, numSecurities):
                    security = msg.GetElement("securityData").GetValue(secIndex)

                    fields = security.GetElement("fieldData")
                    numFields = fields.NumElements
                    for fldIndex in xrange(0, numFields):
                        field = fields.GetElement(fldIndex).Value
                        result.append(field)
            if eventObj.EventType == RESPONSE:
                break
    return result

def MakeHistoricalDataRequest(security, fldList, overrideFields=None, overrideValues=None, dateFrom=None, dateTo=None):
    '''
        Historical request test harness
    '''
    result = []
    session.QueueEvents = True
    session.Start()
    session.OpenService(APIREFDATA_SVC)
    service = session.GetService(APIREFDATA_SVC)

    req = service.CreateRequest("HistoricalDataRequest")

    sec = req.GetElement('securities')
    sec.AppendValue(security)

    flds = req.GetElement('fields')
    for fld in fldList:
        flds.AppendValue(fld)

    if dateFrom:
        req.Set('startDate', dateFrom)
    else:
        req.Set('startDate', TODAY.strftime('%Y%m%d'))

    if dateTo:
        req.Set('endDate', dateTo)
    else:
        req.Set('endDate', TODAY.strftime('%Y%m%d'))
    req.Set('periodicitySelection', 'DAILY')

    if overrideFields:
        overridables = zip(overrideFields, overrideValues)
        overrides = req.GetElement("overrides")
        for overrideField, overrideValue in overridables:
            override = overrides.AppendElment()
            override.SetElement('fieldId', overrideField)
            override.SetElement('value', overrideValue)

    session.SendRequest(req)

    while True:
        eventObj = session.NextEvent()
        if eventObj.EventType == PARTIAL_RESPONSE or eventObj.EventType == RESPONSE:
            it = eventObj.CreateMessageIterator()

            while it.Next():
                msg      = it.Message
                security = msg.GetElement("securityData")
                #secVal   = security.GetElement("security")
                field_exceptions = security.GetElement("fieldExceptions")

                # process exceptions
                if field_exceptions.NumValues > 0:
                    element = field_exceptions.GetValuesAsElement(0)
                    field_id = element.GetElement("fieldId")
                    error_info = element.GetElement("errorInfo")
                    error_message = error_info.GetElement("message")
                    logger.warn('Exception with %s with message: %s' %(field_id, error_message))

                # process field data
                field_data = security.GetElement("fieldData")
                numFields = field_data.NumValues
                for fldIndex in xrange(0, numFields):
                    element = field_data.GetValueAsElement(fldIndex)

                    # repack everything by date
                    subResult = []
                    for field in fldList:
                        if element.HasElement(field):
                            subResult.append(element.GetElement(field).Value)
                    result.append(subResult)

            if eventObj.EventType == RESPONSE:
                break

    return result