REF: Use optional in DBN metadata

Author: threecgreen

CHANGELOG.md

@@ -9,6 +9,9 @@
   `HistoricalClient::TimeseriesGetRangeToFile()` to a `std::filesystem::path`
 - Changed the type of the `file_path` parameter in the `DbnFileStore`, `InFileStream`,
   and `OutFileStream` constructors to a `std::filesystem::path`
+- Made `stype_in` and `schema` optionals in `Metadata` to align with how
+  `Metadata` is represented in other languages and removed the boolean
+  `has_mixed_stype_in` and `has_mixed_schema` fields
 
 ## 0.36.0 - 2025-05-27
 

include/databento/dbn.hpp

@@ -3,6 +3,7 @@
 #include <date/date.h>
 
 #include <cstdint>
+#include <optional>
 #include <ostream>
 #include <string>
 #include <vector>
@@ -37,12 +38,10 @@ struct Metadata {
   std::uint8_t version;
   // The dataset code.
   std::string dataset;
-  // Indicates the DBN stream may contain multiple record types and data
-  // schemas. If true, the schema field should be ignored.
-  bool has_mixed_schema;
-  // The data record schema which affects the type record of present. If
-  // has_mixed_schema is true, this field should be ignored.
-  Schema schema;
+  // The data record schema which affects the type record of present. Will be
+  // nullopt for live where there can be a mix of `schema`s across
+  // subscriptions.
+  std::optional<Schema> schema;
   // The UNIX timestamp of the query start, or the first record if the file was
   // split.
   UnixNanos start;
@@ -51,12 +50,9 @@ struct Metadata {
   UnixNanos end;
   // The maximum number of records for the query.
   std::uint64_t limit;
-  // Indicates the DBN stream may been create with a mix of stype_in. This
-  // will always be the case for live data where stype_in is specified per
-  // subscription.
-  bool has_mixed_stype_in;
-  // The input symbology type. Should be ignored if has_mixed_schemas is true.
-  SType stype_in;
+  // The input symbology type. Will be nullopt for live data where there can be
+  // a mix of `stype_in`s across subscriptions.
+  std::optional<SType> stype_in;
   // The output symbology type.
   SType stype_out;
   // Whether the records contain an appended send timestamp.
@@ -105,13 +101,10 @@ inline bool operator!=(const SymbolMapping& lhs, const SymbolMapping& rhs) {
 
 inline bool operator==(const Metadata& lhs, const Metadata& rhs) {
   return lhs.version == rhs.version && lhs.dataset == rhs.dataset &&
-         (lhs.has_mixed_schema ? rhs.has_mixed_schema
-                               : lhs.schema == rhs.schema) &&
-         lhs.start == rhs.start && lhs.end == rhs.end &&
-         lhs.limit == rhs.limit &&
-         (lhs.has_mixed_stype_in ? rhs.has_mixed_stype_in
-                                 : lhs.stype_in == rhs.stype_in) &&
-         lhs.stype_out == rhs.stype_out && lhs.ts_out == rhs.ts_out &&
+         lhs.schema == rhs.schema && lhs.start == rhs.start &&
+         lhs.end == rhs.end && lhs.limit == rhs.limit &&
+         lhs.stype_in == rhs.stype_in && lhs.stype_out == rhs.stype_out &&
+         lhs.ts_out == rhs.ts_out &&
          lhs.symbol_cstr_len == rhs.symbol_cstr_len &&
          lhs.symbols == rhs.symbols && lhs.partial == rhs.partial &&
          lhs.not_found == rhs.not_found && lhs.mappings == rhs.mappings;

src/dbn.cpp

@@ -34,12 +34,10 @@ std::ostream& operator<<(std::ostream& stream, const Metadata& metadata) {
                     .Build()
                     .AddField("version", metadata.version)
                     .AddField("dataset", metadata.dataset)
-                    .AddField("has_mixed_schema", metadata.has_mixed_schema)
                     .AddField("schema", metadata.schema)
                     .AddField("start", metadata.start)
                     .AddField("end", metadata.end)
                     .AddField("limit", metadata.limit)
-                    .AddField("has_mixed_stype_in", metadata.has_mixed_stype_in)
                     .AddField("stype_in", metadata.stype_in)
                     .AddField("stype_out", metadata.stype_out)
                     .AddField("ts_out", metadata.ts_out)

src/dbn_decoder.cpp

@@ -4,6 +4,7 @@
 
 #include <algorithm>  // copy
 #include <cstring>    // strncmp
+#include <optional>
 #include <vector>
 
 #include "databento/compat.hpp"
@@ -121,12 +122,9 @@ databento::Metadata DbnDecoder::DecodeMetadataFields(
   res.dataset = Consume(buffer, kDatasetCstrLen, "dataset");
   const auto raw_schema = Consume<std::uint16_t>(buffer);
   if (raw_schema == kNullSchema) {
-    res.has_mixed_schema = true;
-    // must initialize
-    res.schema = Schema::Mbo;
+    res.schema = std::nullopt;
   } else {
-    res.has_mixed_schema = false;
-    res.schema = static_cast<Schema>(raw_schema);
+    res.schema = {static_cast<Schema>(raw_schema)};
   }
   res.start =
       UnixNanos{std::chrono::nanoseconds{Consume<std::uint64_t>(buffer)}};
@@ -138,12 +136,9 @@ databento::Metadata DbnDecoder::DecodeMetadataFields(
   }
   const auto raw_stype_in = Consume<std::uint8_t>(buffer);
   if (raw_stype_in == kNullSType) {
-    res.has_mixed_stype_in = true;
-    // must initialize
-    res.stype_in = SType::InstrumentId;
+    res.stype_in = std::nullopt;
   } else {
-    res.has_mixed_stype_in = false;
-    res.stype_in = static_cast<SType>(raw_stype_in);
+    res.stype_in = {static_cast<SType>(raw_stype_in)};
   }
   res.stype_out = static_cast<SType>(Consume<std::uint8_t>(buffer));
   res.ts_out = static_cast<bool>(Consume<std::uint8_t>(buffer));

src/dbn_encoder.cpp

@@ -94,10 +94,10 @@ void DbnEncoder::EncodeMetadata(const Metadata& metadata, IWritable* output) {
   const auto [length, end_padding] = CalcLength(metadata);
   EncodeAsBytes(length, output);
   EncodeFixedLenCStr(kDatasetCstrLen, metadata.dataset, output);
-  if (metadata.has_mixed_schema) {
-    EncodeAsBytes(kNullSchema, output);
+  if (metadata.schema.has_value()) {
+    EncodeAsBytes(*metadata.schema, output);
   } else {
-    EncodeAsBytes(metadata.schema, output);
+    EncodeAsBytes(kNullSchema, output);
   }
   EncodeAsBytes(metadata.start, output);
   EncodeAsBytes(metadata.end, output);
@@ -106,10 +106,10 @@ void DbnEncoder::EncodeMetadata(const Metadata& metadata, IWritable* output) {
     // backwards compatibility for record_count
     EncodeAsBytes(kNullRecordCount, output);
   }
-  if (metadata.has_mixed_stype_in) {
-    EncodeAsBytes(kNullSType, output);
+  if (metadata.stype_in.has_value()) {
+    EncodeAsBytes(*metadata.stype_in, output);
   } else {
-    EncodeAsBytes(metadata.stype_in, output);
+    EncodeAsBytes(kNullSType, output);
   }
   EncodeAsBytes(metadata.stype_out, output);
   EncodeAsBytes(static_cast<std::uint8_t>(metadata.ts_out), output);

src/stream_op_helper.hpp

@@ -5,6 +5,7 @@
 #include <cstring>  // strlen
 #include <iomanip>  // quoted
 #include <ios>      // boolalpha
+#include <optional>
 #include <ostream>  // ostream
 #include <sstream>  // stringstream
 #include <string>
@@ -56,6 +57,15 @@ class StreamOpHelper {
 
   void FmtToStream(const std::ostringstream& val) { stream_ << val.str(); }
 
+  template <typename T>
+  void FmtToStream(const std::optional<T>& val) {
+    if (val.has_value()) {
+      stream_ << *val;
+    } else {
+      stream_ << "nullopt";
+    }
+  }
+
   template <std::size_t N>
   void FmtToStream(const std::array<char, N>& val) {
     stream_ << '"';

src/symbol_map.cpp

@@ -14,8 +14,8 @@ using databento::TsSymbolMap;
 
 namespace {
 bool IsInverse(const databento::Metadata& metadata) {
-  if (!metadata.has_mixed_stype_in) {
-    if (metadata.stype_in == databento::SType::InstrumentId) {
+  if (metadata.stype_in.has_value()) {
+    if (*metadata.stype_in == databento::SType::InstrumentId) {
       return true;
     }
     if (metadata.stype_out == databento::SType::InstrumentId) {

tests/src/dbn_encoder_tests.cpp

@@ -18,12 +18,10 @@ TEST(DbnEncoderTests, TestEncodeDecodeMetadataIdentity) {
   const Metadata metadata{
       kDbnVersion,
       dataset::kGlbxMdp3,
-      false,
       Schema::Mbp10,
       UnixNanos{std::chrono::nanoseconds{1657230820000000000}},
       UnixNanos{std::chrono::nanoseconds{1658960170000000000}},
       0,
-      false,
       SType::RawSymbol,
       SType::InstrumentId,
       true,

tests/src/dbn_tests.cpp

@@ -11,12 +11,10 @@ TEST(DbnTests, TestMetadataToString) {
   const Metadata target{
       kDbnVersion,
       dataset::kGlbxMdp3,
-      false,
       Schema::Ohlcv1D,
       UnixNanos{std::chrono::seconds{1696959347}},
       UnixNanos{std::chrono::seconds{1696950000}},
       {},
-      false,
       SType::RawSymbol,
       SType::InstrumentId,
       false,
@@ -30,12 +28,10 @@ TEST(DbnTests, TestMetadataToString) {
   ASSERT_EQ(res, R"(Metadata {
     version = 3,
     dataset = "GLBX.MDP3",
-    has_mixed_schema = false,
     schema = ohlcv-1d,
     start = 2023-10-10T17:35:47.000000000Z,
     end = 2023-10-10T15:00:00.000000000Z,
     limit = 0,
-    has_mixed_stype_in = false,
     stype_in = raw_symbol,
     stype_out = instrument_id,
     ts_out = false,

tests/src/live_blocking_tests.cpp

@@ -77,7 +77,7 @@ TEST_F(LiveBlockingTests, TestStartAndUpgrade) {
                               .BuildBlocking();
     const auto metadata = target.Start();
     EXPECT_EQ(metadata.version, exp_version);
-    EXPECT_TRUE(metadata.has_mixed_schema);
+    EXPECT_FALSE(metadata.schema.has_value());
     EXPECT_EQ(metadata.dataset, dataset::kGlbxMdp3);
   }
 }
@@ -557,7 +557,7 @@ TEST_F(LiveBlockingTests, TestReconnectAndResubscribe) {
   ASSERT_TRUE(std::holds_alternative<LiveSubscription::NoStart>(
       target.Subscriptions()[0].start));
   const auto metadata = target.Start();
-  EXPECT_TRUE(metadata.has_mixed_schema);
+  EXPECT_FALSE(metadata.schema.has_value());
   const auto rec2 = target.NextRecord();
   ASSERT_TRUE(rec2.Holds<TradeMsg>());
   ASSERT_EQ(rec2.Get<TradeMsg>(), kRec);