ADD: Add SymbolMap helpers to C++

Author: threecgreen

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## 0.15.0 - TBD
 
+### Enhancements
+- Added `PitSymbolMap` helper for keeping track of symbology mappings in Live
+
 ### Bug fixes
 - Fixed misaligned read undefined behavior when decoding records
 

README.md

@@ -78,28 +78,26 @@ Here is a simple program that fetches 10 seconds of trades for all ES mini futur
 ```cpp
 #include <chrono>
 #include <databento/live.hpp>
+#include <databento/symbol_map.hpp>
 #include <iostream>
 #include <string>
 #include <thread>
-#include <unordered_map>
 
 using namespace databento;
 
 int main() {
-  std::unordered_map<std::uint32_t, std::string> symbol_mappings;
+  PitSymbolMap symbol_mappings;
 
   auto client =
       LiveBuilder{}.SetKeyFromEnv().SetDataset("GLBX.MDP3").BuildThreaded();
 
   auto handler = [&symbol_mappings](const Record& rec) {
+    symbol_mappings.OnRecord(rec);
     if (rec.Holds<TradeMsg>()) {
       auto trade = rec.Get<TradeMsg>();
       std::cout << "Received trade for "
                 << symbol_mappings[trade.hd.instrument_id] << ':' << trade
                 << '\n';
-    } else if (rec.Holds<SymbolMappingMsg>()) {
-      auto mapping = rec.Get<SymbolMappingMsg>();
-      symbol_mappings[mapping.hd.instrument_id] = mapping.STypeOutSymbol();
     }
     return KeepGoing::Continue;
   };

cmake/SourcesAndHeaders.cmake

@@ -19,6 +19,7 @@ set(headers
   include/databento/metadata.hpp
   include/databento/publishers.hpp
   include/databento/record.hpp
+  include/databento/symbol_map.hpp
   include/databento/symbology.hpp
   include/databento/timeseries.hpp
   include/databento/with_ts_out.hpp
@@ -51,6 +52,7 @@ set(sources
   src/metadata.cpp
   src/publishers.cpp
   src/record.cpp
+  src/symbol_map.cpp
   src/symbology.cpp
   src/detail/file_stream.cpp
   src/detail/http_client.cpp

example/live/readme.cpp

@@ -3,28 +3,26 @@
 // NOLINTBEGIN(google-build-using-namespace)
 #include <chrono>
 #include <databento/live.hpp>
+#include <databento/symbol_map.hpp>
 #include <iostream>
 #include <string>
 #include <thread>
-#include <unordered_map>
 
 using namespace databento;
 
 int main() {
-  std::unordered_map<std::uint32_t, std::string> symbol_mappings;
+  PitSymbolMap symbol_mappings;
 
   auto client =
       LiveBuilder{}.SetKeyFromEnv().SetDataset("GLBX.MDP3").BuildThreaded();
 
   auto handler = [&symbol_mappings](const Record& rec) {
+    symbol_mappings.OnRecord(rec);
     if (rec.Holds<TradeMsg>()) {
       auto trade = rec.Get<TradeMsg>();
       std::cout << "Received trade for "
                 << symbol_mappings[trade.hd.instrument_id] << ':' << trade
                 << '\n';
-    } else if (rec.Holds<SymbolMappingMsg>()) {
-      auto mapping = rec.Get<SymbolMappingMsg>();
-      symbol_mappings[mapping.hd.instrument_id] = mapping.STypeOutSymbol();
     }
     return KeepGoing::Continue;
   };

example/live/simple.cpp

@@ -1,29 +1,29 @@
 #include <csignal>  // sig_atomic_t
 #include <cstdint>
+#include <databento/constants.hpp>
+#include <databento/dbn.hpp>
+#include <databento/enums.hpp>
+#include <databento/live.hpp>
+#include <databento/live_threaded.hpp>
+#include <databento/log.hpp>
+#include <databento/record.hpp>
+#include <databento/symbol_map.hpp>
+#include <databento/with_ts_out.hpp>
 #include <iostream>
 #include <memory>
-#include <unordered_map>
-
-#include "databento/constants.hpp"
-#include "databento/dbn.hpp"
-#include "databento/enums.hpp"
-#include "databento/live.hpp"
-#include "databento/live_threaded.hpp"
-#include "databento/log.hpp"
-#include "databento/record.hpp"
-#include "databento/with_ts_out.hpp"
 
 static std::sig_atomic_t volatile gSignal;
 
 int main() {
-  std::unordered_map<std::uint32_t, std::string> symbol_mappings;
+  databento::PitSymbolMap symbol_mappings;
   std::unique_ptr<databento::ILogReceiver> log_receiver{
       new databento::ConsoleLogReceiver{databento::LogLevel::Debug}};
 
   auto client = databento::LiveBuilder{}
                     .SetLogReceiver(log_receiver.get())
                     .SetKeyFromEnv()
                     .SetDataset(databento::dataset::kGlbxMdp3)
+                    .SetUpgradePolicy(databento::VersionUpgradePolicy::Upgrade)
                     .BuildThreaded();
 
   // Set up signal handler for Ctrl+C
@@ -44,7 +44,7 @@ int main() {
       case RType::Mbo: {
         auto ohlcv = rec.Get<databento::WithTsOut<databento::MboMsg>>();
         std::cout << "Received tick for "
-                  << symbol_mappings.at(ohlcv.rec.hd.instrument_id)
+                  << symbol_mappings[ohlcv.rec.hd.instrument_id]
                   << " with ts_out " << ohlcv.ts_out.time_since_epoch().count()
                   << ": " << ohlcv.rec << '\n';
         break;
@@ -56,9 +56,7 @@ int main() {
       }
       case RType::SymbolMapping: {
         auto mapping = rec.Get<databento::SymbolMappingMsg>();
-        std::cout << "Received symbol mapping: " << mapping << '\n';
-        symbol_mappings.emplace(mapping.hd.instrument_id,
-                                mapping.STypeInSymbol());
+        symbol_mappings.OnSymbolMapping(mapping);
         break;
       }
       case RType::System: {

include/databento/symbol_map.hpp

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <unordered_map>
+
+#include "databento/compat.hpp"
+#include "databento/record.hpp"
+
+namespace databento {
+// A point-in-time symbol map. Useful for working with live symbology or
+// a historical request over a single day or other situations where the
+// symbol mappings are known not to change.
+class PitSymbolMap {
+ public:
+  using Store = std::unordered_map<std::uint32_t, std::string>;
+
+  PitSymbolMap() = default;
+
+  bool IsEmpty() const { return map_.empty(); }
+  std::size_t Size() const { return map_.size(); }
+  const Store& Map() const { return map_; }
+  Store& Map() { return map_; }
+  Store::const_iterator Find(std::uint32_t instrument_id) const {
+    return map_.find(instrument_id);
+  }
+  std::string& operator[](std::uint32_t instrument_id) {
+    return map_[instrument_id];
+  }
+  void OnRecord(const Record& rec);
+  template <typename SymbolMappingRec>
+  void OnSymbolMapping(const SymbolMappingRec& symbol_mapping);
+
+ private:
+  Store map_;
+};
+
+// Forward declare explicit instantiation
+extern template void PitSymbolMap::OnSymbolMapping(
+    const SymbolMappingMsgV1& symbol_mapping);
+extern template void PitSymbolMap::OnSymbolMapping(
+    const SymbolMappingMsgV2& symbol_mapping);
+}  // namespace databento

src/symbol_map.cpp

@@ -0,0 +1,33 @@
+#include "databento/symbol_map.hpp"
+
+#include "databento/compat.hpp"
+
+using databento::PitSymbolMap;
+
+template <typename SymbolMappingRec>
+void PitSymbolMap::OnSymbolMapping(const SymbolMappingRec& symbol_mapping) {
+  const auto it = map_.find(symbol_mapping.hd.instrument_id);
+  if (it == map_.end()) {
+    map_.emplace(symbol_mapping.hd.instrument_id,
+                 symbol_mapping.STypeOutSymbol());
+  } else {
+    it->second = symbol_mapping.STypeOutSymbol();
+  }
+}
+
+void PitSymbolMap::OnRecord(const Record& record) {
+  if (record.RType() == RType::SymbolMapping) {
+    // Version compat
+    if (record.Header().Size() >= sizeof(SymbolMappingMsgV2)) {
+      OnSymbolMapping(record.Get<SymbolMappingMsgV2>());
+    } else {
+      OnSymbolMapping(record.Get<SymbolMappingMsgV1>());
+    }
+  }
+}
+
+// Explicit instantiation
+template void PitSymbolMap::OnSymbolMapping(
+    const SymbolMappingMsgV1& symbol_mapping);
+template void PitSymbolMap::OnSymbolMapping(
+    const SymbolMappingMsgV2& symbol_mapping);

test/CMakeLists.txt

@@ -27,24 +27,25 @@ set(
   test_sources
   src/batch_tests.cpp
   src/datetime_tests.cpp
-  src/dbn_tests.cpp
   src/dbn_decoder_tests.cpp
+  src/dbn_tests.cpp
   src/file_stream_tests.cpp
   src/flag_set_tests.cpp
   src/historical_tests.cpp
   src/http_client_tests.cpp
-  src/live_tests.cpp
   src/live_blocking_tests.cpp
+  src/live_tests.cpp
   src/live_threaded_tests.cpp
   src/log_tests.cpp
   src/metadata_tests.cpp
-  src/mock_lsg_server.cpp
   src/mock_http_server.cpp
+  src/mock_lsg_server.cpp
   src/mock_tcp_server.cpp
   src/record_tests.cpp
   src/scoped_thread_tests.cpp
   src/shared_channel_tests.cpp
   src/stream_op_helper_tests.cpp
+  src/symbol_map_tests.cpp
   src/symbology_tests.cpp
   src/tcp_client_tests.cpp
   src/zstd_stream_tests.cpp

test/src/symbol_map_tests.cpp

@@ -0,0 +1,58 @@
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <unordered_map>
+
+#include "databento/compat.hpp"
+#include "databento/record.hpp"
+#include "databento/symbol_map.hpp"
+
+namespace databento {
+namespace test {
+template <typename SM>
+SM GenMapping(std::uint32_t instrument_id, const char* stype_out_symbol) {
+  SM res = {};
+  res.hd = RecordHeader{sizeof(SM) / RecordHeader::kLengthMultiplier,
+                        RType::SymbolMapping,
+                        1,
+                        instrument_id,
+                        {}};
+  std::strncpy(res.stype_out_symbol.data(), stype_out_symbol,
+               res.stype_out_symbol.size());
+  return res;
+}
+
+TEST(PitSymbolMapTests, TestOnSymbolMapping) {
+  PitSymbolMap target;
+  target.OnSymbolMapping(GenMapping<SymbolMappingMsgV1>(1, "AAPL"));
+  target.OnSymbolMapping(GenMapping<SymbolMappingMsgV2>(2, "TSLA"));
+  target.OnSymbolMapping(GenMapping<SymbolMappingMsgV1>(3, "MSFT"));
+  std::unordered_map<std::uint32_t, std::string> exp{
+      {1, "AAPL"}, {2, "TSLA"}, {3, "MSFT"}};
+  ASSERT_EQ(target.Map(), exp);
+  target.OnSymbolMapping(GenMapping<SymbolMappingMsgV1>(10, "AAPL"));
+  target.OnSymbolMapping(GenMapping<SymbolMappingMsgV2>(1, "MSFT"));
+  ASSERT_EQ(target[1], "MSFT");
+}
+
+TEST(PitSymbolMapTests, TestOnRecord) {
+  PitSymbolMap target;
+  auto sm1 = GenMapping<SymbolMappingMsgV1>(1, "AAPL");
+  target.OnRecord(Record{&sm1.hd});
+  auto sm2 = GenMapping<SymbolMappingMsgV2>(2, "TSLA");
+  target.OnRecord(Record{&sm2.hd});
+  sm1 = GenMapping<SymbolMappingMsgV1>(3, "MSFT");
+  target.OnRecord(Record{&sm1.hd});
+  std::unordered_map<std::uint32_t, std::string> exp{
+      {1, "AAPL"}, {2, "TSLA"}, {3, "MSFT"}};
+  ASSERT_EQ(target.Map(), exp);
+  sm1 = GenMapping<SymbolMappingMsgV1>(10, "AAPL");
+  target.OnRecord(Record{&sm1.hd});
+  sm2 = GenMapping<SymbolMappingMsgV2>(1, "MSFT");
+  target.OnRecord(Record{&sm2.hd});
+  ASSERT_EQ(target[1], "MSFT");
+}
+}  // namespace test
+}  // namespace databento