ADD: Add C++ symbol map

Author: threecgreen

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## 0.20.0 - TBD
+
+This release improves historical symbology support with the new `TsSymbolMap` class that
+handles mapping historical records to a text symbol. To support this class, several types
+for date fields were changed from strings or ints to `date::year_month_day`.
+
+### Enhancements
+- Added `TsSymbolMap` to support historical symbology where mappings change between days
+- Added `PitSymbol` map constructor from `Metadata` and a `date::year_month_day`
+- Added `Metadata::CreateSymbolMap` and `Metadata::CreateSymbolMapForDate` methods for
+  creating symbology maps from historical metadata
+- Added `SymbologyResolution::CreateSymbolMap` method for creating a symbology map from
+  a symbology resolution response
+
+### Breaking changes
+- Added new dependency on [Howard Hinnant's date library](https://howardhinnant.github.io/date/date.html)
+- Removed type `StrMappingInterval`. `MappingInterval` is now also used in `SymbologyResolution`.
+- Changed type of `start_date` and `end_date` in `MappingInterval` to `date::year_month_day`
+- Added `stype_in` and `stype_out` fields to `SymbologyResolution` to support creating
+  a `TsSymbolMap`
+
 ## 0.19.1 - 2024-06-25
 
 ### Enhancements

CMakeLists.txt

@@ -181,6 +181,38 @@ else()
   # Ignore compiler warnings in headers
   add_system_include_property(httplib)
 endif()
+# date
+if(${PROJECT_NAME_UPPERCASE}_USE_EXTERNAL_DATE)
+  # Check if date target already exists
+  if(TARGET date::date)
+    get_target_property(DATE_SOURCE_DIR date::date SOURCE_DIR)
+    message(STATUS "date::date already available as a target at ${DATE_SOURCE_DIR}")
+    get_target_property(DATE_INCLUDE_DIRS date::date INTERFACE_INCLUDE_DIRECTORIES)
+    if(DATE_INCLUDE_DIRS)
+      message(STATUS "date::date include directories: ${DATE_INCLUDE_DIRS}")
+    endif()
+  else()
+    find_package(date REQUIRED)
+  endif()
+else()
+  set(date_version 3.0.1)
+  if(CMAKE_VERSION VERSION_LESS 3.24)
+    FetchContent_Declare(
+      date_src
+      URL https://github.com/HowardHinnant/date/archive/refs/tags/v${date_version}.tar.gz
+    )
+  else()
+    # DOWNLOAD_EXTRACT_TIMESTAMP added in 3.24
+    FetchContent_Declare(
+      date_src
+      URL https://github.com/HowardHinnant/date/archive/refs/tags/v${date_version}.tar.gz
+      DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    )
+  endif()
+  FetchContent_MakeAvailable(date_src)
+  # Ignore compiler warnings in headers
+  add_system_include_property(date)
+endif()
 # openSSL
 find_package(OpenSSL REQUIRED)
 if(OPENSSL_FOUND)
@@ -204,6 +236,7 @@ endif()
 target_link_libraries(
   ${PROJECT_NAME}
   PUBLIC
+    date::date
     httplib::httplib
     nlohmann_json::nlohmann_json
     OpenSSL::Crypto

README.md

@@ -64,11 +64,12 @@ You'll need to ensure the following dependencies are installed:
 - [Zstandard (zstd)](https://github.com/facebook/zstd)
 - [nlohmann\_json (header-only)](https://github.com/nlohmann/json)
 - [cpp-httplib (header-only)](https://github.com/yhirose/cpp-httplib)
+- [date (header-only)](https://github.com/HowardHinnant/date)
 - [dirent (Windows-only, header-only)](https://github.com/tronkko/dirent)
 
-By default, cpp-httplib and nlohmann\_json are downloaded by CMake as part of the build process.
+By default, date, cpp-httplib and nlohmann\_json are downloaded by CMake as part of the build process.
 If you would like to use a local version of these libraries, enable the CMake flag
-`DATABENTO_ENABLE_EXTERNAL_HTTPLIB` or `DATABENTO_ENABLE_EXTERNAL_JSON`.
+`DATABENTO_ENABLE_EXTERNAL_DATE`, `DATABENTO_ENABLE_EXTERNAL_HTTPLIB`, or `DATABENTO_ENABLE_EXTERNAL_JSON` respectively.
 
 #### Ubuntu
 
@@ -126,26 +127,32 @@ To run this program, set the `DATABENTO_API_KEY` environment variable with an ac
 
 ### Historical
 
-Here is a simple program that fetches 10 minutes worth of historical trades for the entire CME Globex market:
+Here is a simple program that fetches 10 minutes worth of historical trades for two CME futures:
 
 ```cpp
-#include <databento/constants.hpp>
+#include <databento/dbn.hpp>
 #include <databento/historical.hpp>
+#include <databento/symbol_map.hpp>
 #include <iostream>
 
 using namespace databento;
 
 int main() {
-  auto client = HistoricalBuilder{}.SetKeyFromEnv().Build();
-  auto print_trades = [](const Record& record) {
+  auto client = HistoricalBuilder{}.SetKey("$YOUR_API_KEY").Build();
+  TsSymbolMap symbol_map;
+  auto decode_symbols = [&symbol_map](const Metadata& metadata) {
+    symbol_map = metadata.CreateSymbolMap();
+  };
+  auto print_trades = [&symbol_map](const Record& record) {
     const auto& trade_msg = record.Get<TradeMsg>();
-    std::cout << trade_msg << '\n';
+    std::cout << "Received trade for " << symbol_map.At(trade_msg) << ": "
+              << trade_msg << '\n';
     return KeepGoing::Continue;
   };
-  client.TimeseriesGetRange("GLBX.MDP3",
-                            {"2022-06-10T14:30", "2022-06-10T14:40"},
-                            kAllSymbols, Schema::Trades, SType::RawSymbol,
-                            SType::InstrumentId, {}, {}, print_trades);
+  client.TimeseriesGetRange(
+      "GLBX.MDP3", {"2022-06-10T14:30", "2022-06-10T14:40"}, kAllSymbols,
+      Schema::Trades, SType::RawSymbol, SType::InstrumentId, {}, decode_symbols,
+      print_trades);
 }
 ```
 

example/historical/readme.cpp

@@ -1,22 +1,28 @@
 // Duplicate of the example usage code from the README.md to ensure
 // it compiles and to be able to clang-format it.
 // NOLINTBEGIN(google-build-using-namespace)
-#include <databento/constants.hpp>
+#include <databento/dbn.hpp>
 #include <databento/historical.hpp>
+#include <databento/symbol_map.hpp>
 #include <iostream>
 
 using namespace databento;
 
 int main() {
   auto client = HistoricalBuilder{}.SetKey("$YOUR_API_KEY").Build();
-  auto print_trades = [](const Record& record) {
+  TsSymbolMap symbol_map;
+  auto decode_symbols = [&symbol_map](const Metadata& metadata) {
+    symbol_map = metadata.CreateSymbolMap();
+  };
+  auto print_trades = [&symbol_map](const Record& record) {
     const auto& trade_msg = record.Get<TradeMsg>();
-    std::cout << trade_msg << '\n';
+    std::cout << "Received trade for " << symbol_map.At(trade_msg) << ": "
+              << trade_msg << '\n';
     return KeepGoing::Continue;
   };
-  client.TimeseriesGetRange("GLBX.MDP3",
-                            {"2022-06-10T14:30", "2022-06-10T14:40"},
-                            kAllSymbols, Schema::Trades, SType::RawSymbol,
-                            SType::InstrumentId, {}, {}, print_trades);
+  client.TimeseriesGetRange(
+      "GLBX.MDP3", {"2022-06-10T14:30", "2022-06-10T14:40"}, {"ESM2", "NQZ2"},
+      Schema::Trades, SType::RawSymbol, SType::InstrumentId, {}, decode_symbols,
+      print_trades);
 }
 // NOLINTEND(google-build-using-namespace)

example/historical/symbology_resolve.cpp

@@ -16,7 +16,7 @@ int main(int argc, char* argv[]) {
   const auto stype_out = databento::FromString<databento::SType>(argv[3]);
 
   std::vector<std::string> symbols;
-  for (int i = 6; i < argc; ++i) {
+  for (int i = 5; i < argc; ++i) {
     symbols.emplace_back(argv[i]);
   }
 

include/databento/dbn.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <date/date.h>
+
 #include <cstdint>
 #include <ostream>
 #include <string>
@@ -9,13 +11,15 @@
 #include "databento/enums.hpp"
 
 namespace databento {
+// Forward declare
+class PitSymbolMap;
+class TsSymbolMap;
+
 struct MappingInterval {
-  // The start date of the interval (inclusive) as
-  // YYYYMMDD e.g. 2022-10-08 is represented as 20221008
-  std::uint32_t start_date;
-  // The end date of the interval (exclusive) as
-  // YYYYMMDD e.g. 2022-10-08 is represented as 20221008
-  std::uint32_t end_date;
+  // The start date of the interval (inclusive).
+  date::year_month_day start_date;
+  // The end date of the interval (exclusive).
+  date::year_month_day end_date;
   std::string symbol;
 };
 
@@ -69,6 +73,16 @@ struct Metadata {
   // Symbol mappings containing a native symbol and its mapping intervals.
   std::vector<SymbolMapping> mappings;
 
+  // Creates a symbology mapping from instrument ID to text symbol for the given
+  // date.
+  //
+  // This method is useful when working with a historical request over a single
+  // day or in other situations where you're sure the mappings don't change
+  // during the time range of the request. Otherwise, `SymbolMap()` is
+  // recommmended.
+  PitSymbolMap CreateSymbolMapForDate(date::year_month_day date) const;
+  // Creates a symbology mapping from instrument ID and date to text symbol.
+  TsSymbolMap CreateSymbolMap() const;
   // Upgrades the metadata according to `upgrade_policy` if necessary.
   void Upgrade(VersionUpgradePolicy upgrade_policy);
 };

include/databento/detail/json_helpers.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <date/date.h>
 #include <nlohmann/json.hpp>
 
 #include <map>  // multimap
@@ -89,6 +90,10 @@ template <>
 std::vector<std::string> ParseAt(const std::string& endpoint,
                                  const nlohmann::json& json,
                                  const std::string& key);
+template <>
+date::year_month_day ParseAt(const std::string& endpoint,
+                             const nlohmann::json& json,
+                             const std::string& key);
 
 }  // namespace detail
 }  // namespace databento

include/databento/record.hpp

@@ -6,6 +6,7 @@
 #include <cstring>  // strncmp
 #include <string>
 #include <tuple>  // tie
+#include <type_traits>
 
 #include "databento/constants.hpp"  // kSymbolCstrLen
 #include "databento/datetime.hpp"   // UnixNanos
@@ -40,6 +41,15 @@ struct RecordHeader {
   }
 };
 
+// Type trait helper for templated functions accepting DBN records.
+template <typename, typename = std::void_t<>>
+struct has_header : std::false_type {};
+template <typename T>
+struct has_header<T, std::void_t<decltype(std::declval<T>().hd)>>
+    : std::is_same<decltype(std::declval<T>().hd), RecordHeader> {};
+template <typename T>
+constexpr bool has_header_v = has_header<T>::value;
+
 class Record {
  public:
   explicit Record(RecordHeader* record) : record_{record} {}

include/databento/symbol_map.hpp

@@ -1,30 +1,101 @@
 #pragma once
+#include <date/date.h>
 
 #include <cstdint>
+#include <map>
+#include <memory>
 #include <string>
 #include <unordered_map>
+#include <utility>
 
 #include "databento/compat.hpp"
 #include "databento/record.hpp"
 
 namespace databento {
-// A point-in-time symbol map. Useful for working with live symbology or
-// a historical request over a single day or other situations where the
-// symbol mappings are known not to change.
+// Forward declare
+struct Metadata;
+
+/// A timeseries symbol map. Useful for working with historical
+/// data.
+class TsSymbolMap {
+ public:
+  // UTC date and instrument ID to text symbol.
+  using Store = std::map<std::pair<date::year_month_day, std::uint32_t>,
+                         std::shared_ptr<const std::string>>;
+
+  TsSymbolMap() = default;
+  explicit TsSymbolMap(const Metadata& metadata);
+
+  bool IsEmpty() const { return map_.empty(); }
+  std::size_t Size() const { return map_.size(); }
+  const Store& Map() const { return map_; }
+  Store& Map() { return map_; }
+  Store::const_iterator Find(date::year_month_day date,
+                             std::uint32_t instrument_id) const {
+    return map_.find(std::make_pair(date, instrument_id));
+  }
+  template <typename R>
+  Store::const_iterator Find(const R& rec) const {
+    static_assert(
+        has_header_v<R>,
+        "must be a DBN record struct with an `hd` RecordHeader field");
+    date::year_month_day index_date{
+        date::sys_days{date::floor<date::days>(rec.IndexTs())}};
+    return map_.find(std::make_pair(index_date, rec.hd.instrument_id));
+  }
+  const std::string& At(date::year_month_day date,
+                        std::uint32_t instrument_id) const {
+    return *map_.at(std::make_pair(date, instrument_id));
+  }
+  template <typename R>
+  const std::string& At(const R& rec) const {
+    static_assert(
+        has_header_v<R>,
+        "must be a DBN record struct with an `hd` RecordHeader field");
+    date::year_month_day index_date{
+        date::sys_days{date::floor<date::days>(rec.IndexTs())}};
+    return *map_.at(std::make_pair(index_date, rec.hd.instrument_id));
+  }
+  void Insert(std::uint32_t instrument_id, date::year_month_day start_date,
+              date::year_month_day end_date,
+              const std::shared_ptr<const std::string>& symbol);
+
+ private:
+  Store map_;
+};
+
+// A point-in-time symbol map. Useful for working with live
+// symbology or a historical request over a single day or other
+// situations where the symbol mappings are known not to change.
 class PitSymbolMap {
  public:
+  // Instrument ID to text symbol
   using Store = std::unordered_map<std::uint32_t, std::string>;
 
   PitSymbolMap() = default;
+  PitSymbolMap(const Metadata& metadata, date::year_month_day date);
 
   bool IsEmpty() const { return map_.empty(); }
   std::size_t Size() const { return map_.size(); }
   const Store& Map() const { return map_; }
   Store& Map() { return map_; }
+  Store::const_iterator Find(const Record& rec) const {
+    return map_.find(rec.Header().instrument_id);
+  }
   Store::const_iterator Find(std::uint32_t instrument_id) const {
     return map_.find(instrument_id);
   }
-  std::string& operator[](std::uint32_t instrument_id) {
+  template <typename R>
+  const std::string& At(const R& rec) const {
+    static_assert(
+        has_header_v<R>,
+        "must be a DBN record struct with an `hd` RecordHeader field");
+    return map_.at(rec.hd.instrument_id);
+  }
+  const std::string& At(const Record& rec) const {
+    return map_.at(rec.Header().instrument_id);
+  }
+  const std::string& operator[](std::uint32_t instrument_id) {
     return map_[instrument_id];
   }
   void OnRecord(const Record& rec);