VER: Release 0.36.0

Author: threecgreen

.gitignore

@@ -2,4 +2,5 @@ build
 CMakeSettings.json
 vcpkg_installed
 .vs
+.cache
 !*.zst

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## 0.36.0 - 2025-05-27
+
+This version marks the release of DBN version 3 (DBNv3), which is the new default.
+API methods and `DbnDecoder` support decoding all versions of DBN, but now default to
+upgrading data to version 3.
+
+### Enhancements
+- Added `UpgradeToV3` variant to `VersionUpgradePolicy` enum
+- Version 1 and 2 structs can be converted to version 3 structs with templated `Upgrade`
+  method
+- Metadata will now always be encoded with a length divisible by 8 bytes for better
+  alignment
+- Added `is_last` field to live subscription requests which will be used to improve the
+  handling of split subscription requests
+
+### Breaking changes
+- Release of DBN version 3:
+  - Definition schema:
+    - Updated `InstrumentDefMsg` with new `leg_` fields to support multi-leg strategy
+      definitions.
+    - Expanded `raw_instrument_id` to 64 bits to support more venues. Like other 64-bit
+      integer fields, its value will now be quoted in JSON
+    - Removed `trading_reference_date`, `trading_reference_price`, and
+      `settl_price_type` fields which will be normalized in the statistics schema
+    - Removed `md_security_trading_status` better served by the status schema
+    - Updated `asset` to 11 bytes and `kAssetCstrLen` to match
+  - Statistics schema:
+    - Updated `StatMsg` has an expanded 64-bit `quantity` field. `kUndefStatQuantity`
+      has been updated to match
+    - The previous `StatMsg` has been moved to `v2::StatMsg` or `StatMsgV2`
+
+### Bug fixes
+- Fixed "Zstd error decompressing: Operation made no progress over multiple calls, due
+  to output buffer being full" error with `TimeseriesGetRange`
+- Fixed missing implementation of `HistoricalBuilder::SetLogReceiver`
+
 ## 0.35.1 - 2025-05-20
 
 ### Bug fixes

CMakeLists.txt

@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.24..4.0)
 
 project(
   databento
-  VERSION 0.35.1
+  VERSION 0.36.0
   LANGUAGES CXX
   DESCRIPTION "Official Databento client library"
 )

cmake/SourcesAndHeaders.cmake

@@ -71,5 +71,5 @@ set(sources
   src/symbol_map.cpp
   src/symbology.cpp
   src/v1.cpp
-  src/v3.cpp
+  src/v2.cpp
 )

examples/historical/timeseries_get_range.cpp

@@ -1,34 +1,15 @@
-#include <chrono>
-#include <ctime>
-#include <iomanip>
 #include <iostream>  // setw
-#ifdef _WIN32
-// _mkgmtime is equivalent to timegm
-#define timegm _mkgmtime
-#endif
 
 #include "databento/constants.hpp"
-#include "databento/datetime.hpp"
 #include "databento/enums.hpp"
 #include "databento/historical.hpp"
 
-// Converts a date to Unix Epoch nanoseconds
-databento::UnixNanos DateToUnixNanos(int year, int month, int day) {
-  std::tm time{};
-  time.tm_year = year - 1900;
-  // January is 0
-  time.tm_mon = month - 1;
-  time.tm_mday = day;
-  return databento::UnixNanos{std::chrono::seconds{::timegm(&time)}};
-}
-
 int main() {
   auto client = databento::HistoricalBuilder{}.SetKeyFromEnv().Build();
-  const databento::UnixNanos start = DateToUnixNanos(2022, 10, 3);
-  const databento::UnixNanos end = DateToUnixNanos(2022, 10, 4);
   const auto limit = 1000;
   client.TimeseriesGetRange(
-      databento::dataset::kGlbxMdp3, {start, end}, {"ESZ2"},
+      databento::dataset::kGlbxMdp3,
+      databento::DateTimeRange<std::string>{"2022-10-03"}, {"ESZ2"},
       databento::Schema::Trades, databento::SType::RawSymbol,
       databento::SType::InstrumentId, limit,
       [](databento::Metadata&& metadata) { std::cout << metadata << '\n'; },

include/databento/constants.hpp

@@ -16,16 +16,16 @@ static constexpr auto kUndefOrderSize =
     std::numeric_limits<std::uint32_t>::max();
 // The sentinel value for an unset statistic quantity.
 static constexpr auto kUndefStatQuantity =
-    std::numeric_limits<std::int32_t>::max();
+    std::numeric_limits<std::int64_t>::max();
 // The sentinel value for an unset or null timestamp.
 static constexpr auto kUndefTimestamp =
     std::numeric_limits<std::uint64_t>::max();
 // The current version of the DBN encoding.
-static constexpr auto kDbnVersion = 2;
+static constexpr auto kDbnVersion = 3;
 // The length of fixed-length symbol strings.
 static constexpr auto kSymbolCstrLen = 71;
 // The length of fixed-length asset string.
-static constexpr auto kAssetCstrLen = 7;
+static constexpr auto kAssetCstrLen = 11;
 // The multiplier for converting the `length` field in `RecordHeader` to bytes.
 static constexpr std::size_t kRecordHeaderLengthMultiplier = 4;
 

include/databento/dbn_decoder.hpp

@@ -15,7 +15,8 @@
 
 namespace databento {
 // DBN decoder. Set upgrade_policy to control how DBN version 1 data should be
-// handled. Defaults to upgrading DBNv1 data to version 2 (the current version).
+// handled. Defaults to upgrading DBN versions 1 and 2 to version 3 (the current
+// version).
 class DbnDecoder {
  public:
   DbnDecoder(ILogReceiver* log_receiver, InFileStream file_stream);

include/databento/dbn_encoder.hpp

@@ -32,7 +32,8 @@ class DbnEncoder {
   void EncodeRecord(const Record& record);
 
  private:
-  static std::uint32_t CalcLength(const Metadata& metadata);
+  static std::pair<std::uint32_t, std::uint32_t> CalcLength(
+      const Metadata& metadata);
 
   IWritable* output_;
 };

include/databento/detail/buffer.hpp

@@ -3,6 +3,7 @@
 #include <cstddef>
 #include <memory>
 #include <new>
+#include <ostream>
 
 #include "databento/ireadable.hpp"
 #include "databento/iwritable.hpp"
@@ -22,22 +23,32 @@ class Buffer : public IReadable, public IWritable {
   void WriteAll(const char* data, std::size_t length);
   void WriteAll(const std::byte* data, std::size_t length) override;
 
-  std::byte*& WriteBegin() { return write_pos_; }
+  std::byte* WriteBegin() { return write_pos_; }
   std::byte* WriteEnd() { return end_; }
   const std::byte* WriteBegin() const { return write_pos_; }
   const std::byte* WriteEnd() const { return end_; }
+  // Indicate how many bytes were written
+  void Fill(std::size_t length) { write_pos_ += length; }
   std::size_t WriteCapacity() const {
     return static_cast<std::size_t>(end_ - write_pos_);
   }
 
-  /// Will throw if `length > ReadCapacity()`.
+  // Will throw if `length > ReadCapacity()`.
   void ReadExact(std::byte* buffer, std::size_t length) override;
   std::size_t ReadSome(std::byte* buffer, std::size_t max_length) override;
 
-  std::byte*& ReadBegin() { return read_pos_; }
+  std::byte* ReadBegin() { return read_pos_; }
   std::byte* ReadEnd() { return write_pos_; }
   const std::byte* ReadBegin() const { return read_pos_; }
   const std::byte* ReadEnd() const { return write_pos_; }
+  // Indicate how mnay bytes were read
+  void Consume(std::size_t length) {
+    read_pos_ += length;
+    if (static_cast<std::size_t>(read_pos_ - buf_.get()) > (Capacity() / 2)) {
+      Shift();
+    }
+  }
+  void ConsumeNoShift(std::size_t length) { read_pos_ += length; }
   std::size_t ReadCapacity() const {
     return static_cast<std::size_t>(write_pos_ - read_pos_);
   }
@@ -52,6 +63,8 @@ class Buffer : public IReadable, public IWritable {
   void Reserve(std::size_t capacity);
   void Shift();
 
+  friend std::ostream& operator<<(std::ostream& stream, const Buffer& buffer);
+
  private:
   static constexpr std::align_val_t kAlignment{8};
 

include/databento/enums.hpp

@@ -303,6 +303,7 @@ enum class StatUpdateAction : std::uint8_t {
 enum class VersionUpgradePolicy : std::uint8_t {
   AsIs = 0,
   UpgradeToV2 = 1,
+  UpgradeToV3 = 2,
 };
 
 namespace status_action {