VER: Release 0.9.1

Author: threecgreen

.github/workflows/build.yaml

@@ -4,7 +4,6 @@ name: build
 
 on:
   push:
-    branches: [main, dev]
 
 jobs:
   ubuntu:
@@ -20,9 +19,20 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: Install dependencies
+        # pcre is a dependency of cppcheck
         run: |
           sudo apt-get update
-          sudo apt-get install libzstd-dev ninja-build
+          sudo apt-get install libpcre3 libpcre3-dev libzstd-dev ninja-build
+      - name: Install cppcheck
+        run: |
+          git clone https://github.com/danmar/cppcheck.git
+          cd cppcheck
+          cmake -S. -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DUSE_MATCHCOMPILER=1 \
+            -DHAVE_RULES=1
+          cmake --build build --config Release
+          sudo cmake --install build --prefix /usr
       - name: Install gtest
         uses: MarkusJx/googletest-installer@v1.1
       - name: CMake configure
@@ -33,6 +43,7 @@ jobs:
             -DDATABENTO_ENABLE_UNIT_TESTING=1 \
             -DDATABENTO_ENABLE_EXAMPLES=1 \
             -DDATABENTO_ENABLE_CLANG_TIDY=1 \
+            -DDATABENTO_ENABLE_CPPCHECK=1 \
             -DDATABENTO_ENABLE_ASAN=1 \
             -DDATABENTO_ENABLE_UBSAN=1
       - name: CMake build
@@ -48,16 +59,16 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: Install dependencies
-        run: brew install cmake googletest openssl@3 ninja zstd
+        run: brew install cmake cppcheck googletest openssl@3 ninja zstd
       # Don't enable clang-tidy on macOS because it's incredibly slow
       - name: CMake configure
         run: |
           cmake -S . -B build \
             -GNinja \
             -DDATABENTO_ENABLE_UNIT_TESTING=1 \
             -DDATABENTO_ENABLE_EXAMPLES=1 \
-            -DDATABENTO_ENABLE_ASAN=1 \
-            -DDATABENTO_ENABLE_UBSAN=1 \
+            -DDATABENTO_ENABLE_CPPCHECK=1 \
+            -DDATABENTO_ENABLE_TSAN=1 \
             -DOPENSSL_ROOT_DIR=$(brew --prefix openssl@3)
       - name: CMake build
         run: cmake --build build

.github/workflows/release.yaml

@@ -32,19 +32,17 @@ jobs:
       - name: Set output
         id: vars
         run: |
-          echo "name=tag_name::v$(scripts/get_version.sh)" >> $GITHUB_OUTPUT
-          echo "name=release_name::$(scripts/get_version.sh)" >> $GITHUB_OUTPUT
+          echo "TAG_NAME=v$(scripts/get_version.sh)" >> $GITHUB_ENV
+          echo "RELEASE_NAME=$(scripts/get_version.sh)" >> $GITHUB_ENV
+          echo "## Release notes" > NOTES.md
+          sed -n '/^## /{n; :a; /^## /q; p; n; ba}' CHANGELOG.md >> NOTES.md
 
       # Create GitHub release
       - name: Create release
         id: create-release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          TAG_NAME: ${{ steps.vars.outputs.tag_name }}
-          RELEASE_NAME: ${{ steps.vars.outputs.release_name }}
+        uses: softprops/action-gh-release@v1
         with:
+          name: ${{ env.RELEASE_NAME }}
           tag_name: ${{ env.TAG_NAME }}
-          release_name: ${{ env.RELEASE_NAME }}
-          draft: false
-          prerelease: false
+          append_body: true
+          body_path: ./NOTES.md

CHANGELOG.md

@@ -1,99 +1,158 @@
 # Changelog
 
+## 0.9.1 - 2023-07-11
+
+#### Enhancements
+- Added constants for dataset codes for Databento Equity Basic and OPRA Pillar
+- Added `const char*` getters to records for fixed-length `char` arrays
+- Added `RType` getter to `Record`
+
+#### Bug fixes
+- Batch live subscriptions to avoid hitting max message length
+- Fix bug in Zstd decompression
+- Fix `Historical::BatchDownload` truncating file before writing each chunk
+
 ## 0.9.0 - 2023-06-13
+
+#### Enhancements
 - Added `Reconnect` methods to `LiveBlocking` and `LiveThreaded`
 - Added optional `exception_callback` argument to `LiveThreaded::Start` to improve
   error handling options
 - Added batch download support data files (`condition.json` and `symbology.json`)
 - Added support for logging warnings from Historical API
+- Relaxed 10 minute minimum request time range restriction
+
+#### Breaking changes
 - Changed `use_ts_out` default to `false`
+
+#### Bug fixes
 - Fixed missing definition for `operator==` for `ImbalanceMsg`
-- Removed 10 minute minimum request time range restriction
 
 ## 0.8.0 - 2023-05-16
+
+#### Enhancements
 - Changed `end` and `end_date` to optional to support new forward-fill behaviour
+
+#### Breaking changes
 - Renamed `booklevel` MBP field to `levels` for brevity and consistent naming
 - Removed `open_interest_qty` and `cleared_volume` fields from definitions schema
   that were always unset
 
 ## 0.7.0 - 2023-04-28
+
+#### Enhancements
 - Added initial support for live data with `LiveBlocking` and `LiveThreaded` clients
 - Added support for statistics schema
 - Added `SystemMsg` and `ErrorMsg` records for use in live data
 - Added `strike_price`, `strike_price_currency`, and `instrument_class` to `InstrumentDefMsg`
-- Renamed `BatchJob.cost` to `cost_usd` and value now expressed as US dollars
 - Added `FixedPx` helper class for formatting fixed prices
 - Added configurable log receiver `ILogReceiver`
 - Added `instrument_class`, `strike_price`, and `strike_price_currency` to definition schema
 - Added additional `condition` variants for `DatasetConditionDetail` (degraded, pending, missing)
 - Added additional member `last_modified_date` to `DatasetConditionDetail` Added `has_mixed_schema`, `has_mixed_stype_in`, and `ts_out` to `Metadata` to support live data
+- Added optional `compression` parameter to `BatchSubmitJob`
+
+#### Breaking changes
 - Removed `related` and `related_security_id` from `InstrumentDefMsg`
+- Renamed `BatchJob.cost` to `cost_usd` and value now expressed as US dollars
 - Renamed `SType::ProductId` to `SType::InstrumentId` and `SType::Native` to `SType::RawSymbol`
 - Renamed `RecordHeader::product_id` to `instrument_id`
 - Renamed `InstrumentDefMsg::symbol` to `raw_symbol`
 - Renamed `SymbolMapping::native_symbol` to `raw_symbol`
-- Deprecated `SType::Smart` to split into `SType::Parent` and `SType::Continuous`
 - Changed `expiration` and `action` type to `UnixNanos`
 - Changed some fields to enums in `InstrumentDefMsg`
-- Added optional `compression` parameter to `BatchSubmitJob`
+
+#### Deprecations
+- Deprecated `SType::Smart` to split into `SType::Parent` and `SType::Continuous`
+
+#### Bug fixes
 - Fixed parsing of `BatchSubmitJob` response
 - Fixed invalid read in `DbnDecoder`
 - Fixed memory leak in `TryCreateDir`
 
 ## 0.6.1 - 2023-03-28
-- Fixed Zstd decoding of files with multiple frames
+
+#### Breaking changes
 - Removed usage of unreliable `std::ifstream::readsome`
 
+#### Bug fixes
+- Fixed Zstd decoding of files with multiple frames
+
 ## 0.6.0 - 2023-03-24
+
+#### Enhancements
 - Added support for imbalance schema
 - Added support for decoding `ts_out` field
+- Added flags `kSnapshot` and `kMaybeBadBook`
+
+#### Breaking changes
 - Removed `record_count` from `Metadata`
 - Changed `Historical::BatchDownload` to return the paths of the downloaded files
-- Added flags `kSnapshot` and `kMaybeBadBook`
 
 ## 0.5.0 - 2023-03-13
+
+#### Enhancements
 - Added `Historical::MetadataGetDatasetRange`
+
+#### Breaking changes
 - Changed `MetadataGetDatasetCondition` to return `vector<DatasetConditionDetail>`
 - Removed `MetadataListCompressions` (redundant with docs)
 - Removed `MetadataListEncodings` (redundant with docs)
 - Removed optional `start` and `end` params from `MetadataListSchemas` (redundant)
 - Renamed `FileBento` to `DbnFileStore`
 
 ## 0.4.0 - 2023-03-02
-- Renamed DBZ to DBN
-  - Renamed `DbzParser` to `DbnDecoder`
-- Renamed `TimeseriesStream` to `TimeseriesGetRange`
-- Refactored rtypes
-  - Introduced separate rtypes for each OHLCV schema
+
+#### Enhancements
+
 - Added live gateway resolution
 - Added `SymbolMappingMsg` and `ErrorMsg` records
-- Improved API for `flags` record fields
 - Added `Action` and `Side` enums
-- Changed `kAllSymbols` representation
-- Fixed usage of as a system library
-- Removed `is_full_universe` and `is_example` fields from `BatchJob`
-- Disabled unit testing by default
-- Added `PKGBUILD` to demonstrate installation
-- Made `start_date` and `end_date` optional for
-  `Historical::MetadataGetDatasetCondition`
 - Added `available_start_date` and `available_end_date` to
   `DatasetConditionInfo`
+- Made `start_date` and `end_date` optional for
+  `Historical::MetadataGetDatasetCondition`
+- Improved API for `flags` record fields
+- Added `PKGBUILD` to demonstrate installation
+- Disabled unit testing by default
+
+#### Breaking changes
+- Removed `is_full_universe` and `is_example` fields from `BatchJob`
+- Refactored rtypes
+  - Introduced separate rtypes for each OHLCV schema
+- Renamed DBZ to DBN
+  - Renamed `DbzParser` to `DbnDecoder`
+- Renamed `TimeseriesStream` to `TimeseriesGetRange`
+- Changed `kAllSymbols` representation
+
+#### Bug fixes
+- Fixed usage of as a system library
 
 ## 0.3.0 - 2023-01-06
+
+#### Enhancements
 - Added support for definition schema
-- Fixed cancellation in `Historical::TimeseriesStream`
-- Fixed race condition in `Historical::TimeseriesStream` exception handling
-- Fixed gtest linker error on macOS
 - Added option for CMake to download gtest
-- Renamed `TickMsg` to `MboMsg`
-- Changed `flags` fields to unsigned
 - Updated `Flag` enum
+
+#### Breaking changes
 - Standardized getter method names to pascal case
 - Renamed `is_full_book` to `is_full_universe`
+- Renamed `TickMsg` to `MboMsg`
+- Changed `flags` fields to unsigned
+
+#### Bug fixes
+- Fixed cancellation in `Historical::TimeseriesStream`
+- Fixed race condition in `Historical::TimeseriesStream` exception handling
+- Fixed gtest linker error on macOS
 
 ## 0.2.0 - 2022-12-01
+
+#### Enhancements
 - Added `Historical::MetadataGetDatasetCondition`
 - Improved Zstd CMake integration
+
+#### Bug fixes
 - Fixed requesting all symbols for a dataset
 
 ## 0.1.0 - 2022-11-07

CMakeLists.txt

@@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.14)
 # Project details
 #
 
-project("databento" VERSION 0.9.0 LANGUAGES CXX)
+project("databento" VERSION 0.9.1 LANGUAGES CXX)
 string(TOUPPER ${PROJECT_NAME} PROJECT_NAME_UPPERCASE)
 
 #
@@ -135,6 +135,8 @@ else()
     )
   endif()
   FetchContent_MakeAvailable(json)
+  # Ignore compiler warnings in headers
+  add_system_include_property(nlohmann_json)
 endif()
 # cpp-httplib
 set(httplib_version 0.11.4)

README.md

@@ -2,6 +2,7 @@
 
 [![test](https://github.com/databento/databento-cpp/actions/workflows/build.yaml/badge.svg?branch=main)](https://github.com/databento/databento-cpp/actions/workflows/build.yaml)
 [![license](https://img.shields.io/github/license/databento/databento-cpp?color=blue)](./LICENSE)
+[![Slack](https://img.shields.io/badge/join_Slack-community-darkblue.svg?logo=slack)](https://join.slack.com/t/databento-hq/shared_invite/zt-1xk498wxs-9fUs_xhz5ypaGD~mhI_hVQ)
 
 The official C++ client library for [Databento](https://databento.com).
 The client supports both streaming real-time and historical market data through similar interfaces.

cmake/StaticAnalyzers.cmake

@@ -17,19 +17,13 @@ if(${PROJECT_NAME_UPPERCASE}_ENABLE_CPPCHECK)
     set(
       CMAKE_CXX_CPPCHECK ${CPPCHECK}
       --enable=all
-      --suppress=missingIncludeSystem
+      --suppress=missingIncludeSystem  # False positives
       --suppress=preprocessorErrorDirective
-      --suppress=unusedFunction
-      --suppress=unmatchedSuppression
+      --suppress=unusedFunction        # False positives
+      --suppress=unmatchedSuppression  # Support different cppcheck versions
       --inline-suppr
       --relative-paths=${CMAKE_SOURCE_DIR}
       --quiet
-      -I${CMAKE_SOURCE_DIR}/include
-      -I${CMAKE_SOURCE_DIR}/src
-      # Ignore tests
-      -i${CMAKE_SOURCE_DIR}/test
-      # Ignore third-party dependencies
-      -i${CMAKE_BINARY_DIR}
     )
     message(STATUS "Cppcheck finished setting up.")
   else()

include/databento/constants.hpp

@@ -18,7 +18,13 @@ static constexpr auto kUndefOrderSize =
 // This is not necessarily a comprehensive list of available datasets. Please
 // use `Historical.MetadataListDatasets` to retrieve an up-to-date list.
 namespace dataset {
+// The dataset code for Databento Equity Basic.
+static constexpr auto kDbeqBasic = "DBEQ.BASIC";
+// The dataset code for CME Globex MDP 3.0.
 static constexpr auto kGlbxMdp3 = "GLBX.MDP3";
+// The dataset code for OPRA.PILLAR.
+static constexpr auto kOpraPillar = "OPRA.PILLAR";
+// The dataset code for Nasdaq TotalView ITCH.
 static constexpr auto kXnasItch = "XNAS.ITCH";
 }  // namespace dataset
 }  // namespace databento

include/databento/historical.hpp

@@ -237,6 +237,8 @@ class Historical {
   using HttplibParams = std::multimap<std::string, std::string>;
 
   BatchJob BatchSubmitJob(const HttplibParams& params);
+  void StreamToFile(const std::string& url_path, const HttplibParams& params,
+                    const std::string& file_path);
   void DownloadFile(const std::string& url, const std::string& output_path);
   std::vector<BatchJob> BatchListJobs(const HttplibParams& params);
   std::vector<DatasetConditionDetail> MetadataGetDatasetCondition(

include/databento/record.hpp

@@ -36,6 +36,7 @@ class Record {
   explicit Record(RecordHeader* record) : record_{record} {}
 
   const RecordHeader& Header() const { return *record_; }
+  ::databento::RType RType() const { return record_->rtype; }
 
   template <typename T>
   bool Holds() const {
@@ -53,7 +54,7 @@ class Record {
 
   std::size_t Size() const;
   static std::size_t SizeOfSchema(Schema schema);
-  static RType RTypeFromSchema(Schema schema);
+  static ::databento::RType RTypeFromSchema(Schema schema);
 
  private:
   RecordHeader* record_;
@@ -168,6 +169,21 @@ static_assert(sizeof(OhlcvMsg) == 56, "OhlcvMsg size must match C");
 struct InstrumentDefMsg {
   static bool HasRType(RType rtype) { return rtype == RType::InstrumentDef; }
 
+  const char* Currency() const { return currency.data(); }
+  const char* SettlCurrency() const { return settl_currency.data(); }
+  const char* SecSubType() const { return secsubtype.data(); }
+  const char* RawSymbol() const { return raw_symbol.data(); }
+  const char* Group() const { return group.data(); }
+  const char* Exchange() const { return exchange.data(); }
+  const char* Asset() const { return asset.data(); }
+  const char* Cfi() const { return cfi.data(); }
+  const char* SecurityType() const { return security_type.data(); }
+  const char* UnitOfMeasure() const { return unit_of_measure.data(); }
+  const char* Underlying() const { return underlying.data(); }
+  const char* StrikePriceCurrency() const {
+    return strike_price_currency.data();
+  }
+
   RecordHeader hd;
   UnixNanos ts_recv;
   std::int64_t min_price_increment;
@@ -308,6 +324,9 @@ struct ErrorMsg {
 struct SymbolMappingMsg {
   static bool HasRType(RType rtype) { return rtype == RType::SymbolMapping; }
 
+  const char* STypeInSymbol() const { return stype_in_symbol.data(); }
+  const char* STypeOutSymbol() const { return stype_out_symbol.data(); }
+
   RecordHeader hd;
   std::array<char, 22> stype_in_symbol;
   std::array<char, 22> stype_out_symbol;