Skip to content

Commit e2ce872

Browse files
authored
VER: Release 0.14.0
See release notes.
2 parents b8553e3 + 8ee923c commit e2ce872

File tree

85 files changed

+1166
-298
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

85 files changed

+1166
-298
lines changed

CHANGELOG.md

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,47 @@
11
# Changelog
22

3+
## 0.14.0 - 2023-11-23
4+
5+
This release adds support for DBN v2.
6+
7+
DBN v2 delivers improvements to the `Metadata` header symbology, new `stype_in` and `stype_out`
8+
fields for `SymbolMappingMsg`, and extends the symbol field length for `SymbolMappingMsg` and
9+
`InstrumentDefMsg`. The entire change notes are available [here](https://github.com/databento/dbn/releases/tag/v0.14.0).
10+
Users who wish to convert DBN v1 files to v2 can use the `dbn-cli` tool available in the [databento-dbn](https://github.com/databento/dbn/) crate.
11+
On a future date, the Databento live and historical APIs will stop serving DBN v1.
12+
13+
This release is fully compatible with both DBN v1 and v2, and so should be seamless for most users.
14+
15+
### Enhancements
16+
- Added support for DBN encoding version 2 (DBNv2), affecting `SymbolMappingMsg`,
17+
`InstrumentDefMsg`, and `Metadata`
18+
- Version 1 structs can be converted to version 2 structs with the `ToV2()` method
19+
- Added `symbol_cstr_len` field to `Metadata` to indicate the length of fixed symbol
20+
strings
21+
- Added `stype_in` and `stype_out` fields to `SymbolMappingMsg` to provide more context
22+
with live symbology updates
23+
- Added `IndexTs` methods to every record type which returns the primary timestamp
24+
- Added `VersionUpgradePolicy` enum to allow specifying how to handle decoding records
25+
from prior DBN versions
26+
- Added `InstrumentDefMsgV2` and `SymbolMappingMsgV2` type aliases
27+
- Added `kDbnVersion` constant for current DBN version
28+
- Added `kSymbolCstrLen`, `kSymbolCstrLenV1`, and `kSymbolCstrLenV2` constants for the
29+
length of fixed-length symbol strings in different DBN versions
30+
- Added new publisher values in preparation for IFEU.IMPACT and NDEX.IMPACT datasets
31+
- Added new publisher values for consolidated DBEQ.BASIC and DBEQ.PLUS
32+
- Added `kMaxRecordLen` constant for the the length of the largest record type
33+
- Added ability to convert `FlagSet` to underlying representation
34+
35+
### Breaking changes
36+
- The old `InstrumentDefMsg` is now `InstrumentDefMsgV1` in `compat.hpp`
37+
- The old `SymbolMappingMsg` is now `SymbolMappingMsgV1` in `compat.hpp`
38+
- Converted the following enums to enum classes to allow safely adding new variants:
39+
`SecurityUpdateAction` and `SType`
40+
- Renamed `dummy` to `reserved` in `InstrumentDefMsg`
41+
- Removed `reserved2`, `reserved3`, `reserved4`, and `reserved5` from `InstrumentDefMsg`
42+
- Moved position of `strike_price` within `InstrumentDefMsg`
43+
- Removed deprecated `SecurityUpdateAction::Invalid` variant
44+
345
## 0.13.1 - 2023-10-23
446
### Enhancements
547
- Added new publisher values in preparation for DBEQ.PLUS

CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.14)
44
# Project details
55
#
66

7-
project("databento" VERSION 0.13.1 LANGUAGES CXX)
7+
project("databento" VERSION 0.14.0 LANGUAGES CXX)
88
string(TOUPPER ${PROJECT_NAME} PROJECT_NAME_UPPERCASE)
99

1010
#

cmake/SourcesAndHeaders.cmake

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
set(headers
22
include/databento/batch.hpp
3+
include/databento/compat.hpp
34
include/databento/constants.hpp
45
include/databento/datetime.hpp
56
include/databento/dbn.hpp
@@ -34,6 +35,7 @@ set(headers
3435

3536
set(sources
3637
src/batch.cpp
38+
src/compat.cpp
3739
src/datetime.cpp
3840
src/dbn.cpp
3941
src/dbn_decoder.cpp

include/databento/compat.hpp

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
// Record definitions from previous DBN versions and helper functions.
2+
#pragma once
3+
4+
#include <cstddef> // size_t
5+
#include <cstdint>
6+
7+
#include "databento/constants.hpp" // kSymbolCstrLen
8+
#include "databento/datetime.hpp" // UnixNanos
9+
#include "databento/enums.hpp"
10+
#include "databento/record.hpp"
11+
12+
namespace databento {
13+
static constexpr std::size_t kSymbolCstrLenV1 = 22;
14+
static constexpr std::size_t kSymbolCstrLenV2 = kSymbolCstrLen;
15+
16+
constexpr std::size_t VersionSymbolCstrLen(std::uint8_t version) {
17+
return version < 2 ? kSymbolCstrLenV1 : kSymbolCstrLenV2;
18+
}
19+
20+
using InstrumentDefMsgV2 = InstrumentDefMsg;
21+
using SymbolMappingMsgV2 = SymbolMappingMsg;
22+
23+
// DBN version 1 instrument definition.
24+
struct InstrumentDefMsgV1 {
25+
static bool HasRType(RType rtype) { return rtype == RType::InstrumentDef; }
26+
27+
InstrumentDefMsgV2 ToV2() const;
28+
const char* Currency() const { return currency.data(); }
29+
const char* SettlCurrency() const { return settl_currency.data(); }
30+
const char* SecSubType() const { return secsubtype.data(); }
31+
const char* RawSymbol() const { return raw_symbol.data(); }
32+
const char* Group() const { return group.data(); }
33+
const char* Exchange() const { return exchange.data(); }
34+
const char* Asset() const { return asset.data(); }
35+
const char* Cfi() const { return cfi.data(); }
36+
const char* SecurityType() const { return security_type.data(); }
37+
const char* UnitOfMeasure() const { return unit_of_measure.data(); }
38+
const char* Underlying() const { return underlying.data(); }
39+
const char* StrikePriceCurrency() const {
40+
return strike_price_currency.data();
41+
}
42+
43+
RecordHeader hd;
44+
UnixNanos ts_recv;
45+
std::int64_t min_price_increment;
46+
std::int64_t display_factor;
47+
UnixNanos expiration;
48+
UnixNanos activation;
49+
std::int64_t high_limit_price;
50+
std::int64_t low_limit_price;
51+
std::int64_t max_price_variation;
52+
std::int64_t trading_reference_price;
53+
std::int64_t unit_of_measure_qty;
54+
std::int64_t min_price_increment_amount;
55+
std::int64_t price_ratio;
56+
std::int32_t inst_attrib_value;
57+
std::uint32_t underlying_id;
58+
std::uint32_t raw_instrument_id;
59+
std::int32_t market_depth_implied;
60+
std::int32_t market_depth;
61+
std::uint32_t market_segment_id;
62+
std::uint32_t max_trade_vol;
63+
std::int32_t min_lot_size;
64+
std::int32_t min_lot_size_block;
65+
std::int32_t min_lot_size_round_lot;
66+
std::uint32_t min_trade_vol;
67+
std::array<char, 4> _reserved2;
68+
std::int32_t contract_multiplier;
69+
std::int32_t decay_quantity;
70+
std::int32_t original_contract_size;
71+
std::array<char, 4> _reserved3;
72+
std::uint16_t trading_reference_date;
73+
std::int16_t appl_id;
74+
std::uint16_t maturity_year;
75+
std::uint16_t decay_start_date;
76+
std::uint16_t channel_id;
77+
std::array<char, 4> currency;
78+
std::array<char, 4> settl_currency;
79+
std::array<char, 6> secsubtype;
80+
std::array<char, kSymbolCstrLenV1> raw_symbol;
81+
std::array<char, 21> group;
82+
std::array<char, 5> exchange;
83+
std::array<char, 7> asset;
84+
std::array<char, 7> cfi;
85+
std::array<char, 7> security_type;
86+
std::array<char, 31> unit_of_measure;
87+
std::array<char, 21> underlying;
88+
std::array<char, 4> strike_price_currency;
89+
InstrumentClass instrument_class;
90+
std::array<char, 2> _reserved4;
91+
std::int64_t strike_price;
92+
std::array<char, 6> _reserved5;
93+
MatchAlgorithm match_algorithm;
94+
std::uint8_t md_security_trading_status;
95+
std::uint8_t main_fraction;
96+
std::uint8_t price_display_format;
97+
std::uint8_t settl_price_type;
98+
std::uint8_t sub_fraction;
99+
std::uint8_t underlying_product;
100+
SecurityUpdateAction security_update_action;
101+
std::uint8_t maturity_month;
102+
std::uint8_t maturity_day;
103+
std::uint8_t maturity_week;
104+
UserDefinedInstrument user_defined_instrument;
105+
std::int8_t contract_multiplier_unit;
106+
std::int8_t flow_schedule_type;
107+
std::uint8_t tick_rule;
108+
// padding for alignment
109+
std::array<char, 3> dummy;
110+
};
111+
static_assert(sizeof(InstrumentDefMsgV1) == 360, "Size must match Rust");
112+
static_assert(alignof(InstrumentDefMsgV1) == 8, "Must have 8-byte alignment");
113+
114+
/// A symbol mapping message.
115+
struct SymbolMappingMsgV1 {
116+
static bool HasRType(RType rtype) { return rtype == RType::SymbolMapping; }
117+
118+
SymbolMappingMsgV2 ToV2() const;
119+
const char* STypeInSymbol() const { return stype_in_symbol.data(); }
120+
const char* STypeOutSymbol() const { return stype_out_symbol.data(); }
121+
122+
RecordHeader hd;
123+
std::array<char, kSymbolCstrLenV1> stype_in_symbol;
124+
std::array<char, kSymbolCstrLenV1> stype_out_symbol;
125+
// padding for alignment
126+
std::array<char, 4> dummy;
127+
UnixNanos start_ts;
128+
UnixNanos end_ts;
129+
};
130+
static_assert(sizeof(SymbolMappingMsgV1) == 80, "Size must match Rust");
131+
static_assert(alignof(SymbolMappingMsgV1) == 8, "Must have 8-byte alignment");
132+
133+
bool operator==(const InstrumentDefMsgV1& lhs, const InstrumentDefMsgV1& rhs);
134+
inline bool operator!=(const InstrumentDefMsgV1& lhs,
135+
const InstrumentDefMsgV1& rhs) {
136+
return !(lhs == rhs);
137+
}
138+
inline bool operator==(const SymbolMappingMsgV1& lhs,
139+
const SymbolMappingMsgV1& rhs) {
140+
return std::tie(lhs.hd, lhs.stype_in_symbol, lhs.stype_out_symbol,
141+
lhs.start_ts, lhs.end_ts) ==
142+
std::tie(rhs.hd, rhs.stype_in_symbol, rhs.stype_out_symbol,
143+
rhs.start_ts, rhs.end_ts);
144+
}
145+
inline bool operator!=(const SymbolMappingMsgV1& lhs,
146+
const SymbolMappingMsgV1& rhs) {
147+
return !(lhs == rhs);
148+
}
149+
std::string ToString(const InstrumentDefMsgV1& instr_def_msg);
150+
std::ostream& operator<<(std::ostream& stream,
151+
const InstrumentDefMsgV1& instr_def_msg);
152+
std::string ToString(const SymbolMappingMsgV1& symbol_mapping_msg);
153+
std::ostream& operator<<(std::ostream& stream,
154+
const SymbolMappingMsgV1& symbol_mapping_msg);
155+
} // namespace databento

include/databento/constants.hpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ static constexpr auto kUndefStatQuantity =
2020
// The sentinel value for an unset or null timestamp.
2121
static constexpr auto kUndefTimestamp =
2222
std::numeric_limits<std::uint64_t>::max();
23+
// The current version of the DBN encoding.
24+
static constexpr auto kDbnVersion = 2;
25+
// The length of fixed-length symbol strings.
26+
static constexpr auto kSymbolCstrLen = 71;
2327

2428
// This is not necessarily a comprehensive list of available datasets. Please
2529
// use `Historical.MetadataListDatasets` to retrieve an up-to-date list.
@@ -30,7 +34,7 @@ static constexpr auto kDbeqBasic = "DBEQ.BASIC";
3034
static constexpr auto kGlbxMdp3 = "GLBX.MDP3";
3135
// The dataset code for OPRA.PILLAR.
3236
static constexpr auto kOpraPillar = "OPRA.PILLAR";
33-
// The dataset code for Nasdaq TotalView ITCH.
37+
// The dataset code for Nasdaq TotalView-ITCH.
3438
static constexpr auto kXnasItch = "XNAS.ITCH";
3539
} // namespace dataset
3640
} // namespace databento

include/databento/dbn.hpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ struct Metadata {
5656
SType stype_out;
5757
// Whether the records contain an appended send timestamp.
5858
bool ts_out;
59+
// The length in bytes of fixed-length symbol strings, including a null
60+
// terminator byte.
61+
std::size_t symbol_cstr_len;
5962
// The original query input symbols from the request.
6063
std::vector<std::string> symbols;
6164
// Symbols that did not resolve for _at least one day_ in the query time
@@ -91,6 +94,7 @@ inline bool operator==(const Metadata& lhs, const Metadata& rhs) {
9194
(lhs.has_mixed_stype_in ? rhs.has_mixed_stype_in
9295
: lhs.stype_in == rhs.stype_in) &&
9396
lhs.stype_out == rhs.stype_out && lhs.ts_out == rhs.ts_out &&
97+
lhs.symbol_cstr_len == rhs.symbol_cstr_len &&
9498
lhs.symbols == rhs.symbols && lhs.partial == rhs.partial &&
9599
lhs.not_found == rhs.not_found && lhs.mappings == rhs.mappings;
96100
}

include/databento/dbn_decoder.hpp

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,50 +8,67 @@
88
#include "databento/dbn.hpp"
99
#include "databento/detail/file_stream.hpp"
1010
#include "databento/detail/shared_channel.hpp"
11+
#include "databento/enums.hpp" // Upgrade Policy
1112
#include "databento/ireadable.hpp"
1213
#include "databento/record.hpp"
1314

1415
namespace databento {
15-
// DBN decoder. Use either the DbnChannelDecoder or DbnFileDecoder
16-
// specialization.
16+
// DBN decoder. Set upgrade_policy to control how DBN version 1 data should be
17+
// handled. Currently it defaults to returning this data as-is, but this default
18+
// will change in a future version.
1719
class DbnDecoder {
1820
public:
1921
explicit DbnDecoder(detail::SharedChannel channel);
2022
explicit DbnDecoder(detail::FileStream file_stream);
2123
explicit DbnDecoder(std::unique_ptr<IReadable> input);
24+
DbnDecoder(std::unique_ptr<IReadable> input,
25+
VersionUpgradePolicy upgrade_policy);
2226

2327
// Decode metadata from the given buffer.
2428
static Metadata DecodeMetadata(const std::vector<std::uint8_t>& buffer);
2529
static std::pair<std::uint8_t, std::size_t> DecodeMetadataVersionAndSize(
2630
const std::uint8_t* buffer, std::size_t size);
2731
static Metadata DecodeMetadataFields(std::uint8_t version,
2832
const std::vector<std::uint8_t>& buffer);
33+
// Decodes a record possibly applying upgrading the data according to the
34+
// given version and upgrade policy. If an upgrade is applied,
35+
// compat_buffer is modified.
36+
static Record DecodeRecordCompat(
37+
std::uint8_t version, VersionUpgradePolicy upgrade_policy,
38+
std::array<std::uint8_t, kMaxRecordLen>* compat_buffer, Record rec);
2939

30-
// Should only be called once
40+
// Should be called exactly once.
3141
Metadata DecodeMetadata();
3242
// Lifetime of returned Record is until next call to DecodeRecord. Returns
3343
// nullptr once the end of the input has been reached.
3444
const Record* DecodeRecord();
3545

3646
private:
3747
static std::string DecodeSymbol(
48+
std::size_t symbol_cstr_len,
3849
std::vector<std::uint8_t>::const_iterator& buffer_it);
3950
static std::vector<std::string> DecodeRepeatedSymbol(
51+
std::size_t symbol_cstr_len,
4052
std::vector<std::uint8_t>::const_iterator& buffer_it,
4153
std::vector<std::uint8_t>::const_iterator buffer_end_it);
4254
static std::vector<SymbolMapping> DecodeSymbolMappings(
55+
std::size_t symbol_cstr_len,
4356
std::vector<std::uint8_t>::const_iterator& buffer_it,
4457
std::vector<std::uint8_t>::const_iterator buffer_end_it);
4558
static SymbolMapping DecodeSymbolMapping(
59+
std::size_t symbol_cstr_len,
4660
std::vector<std::uint8_t>::const_iterator& buffer_it,
4761
std::vector<std::uint8_t>::const_iterator buffer_end_it);
4862
bool DetectCompression();
4963
std::size_t FillBuffer();
5064
RecordHeader* BufferRecordHeader();
5165

66+
std::uint8_t version_{};
67+
VersionUpgradePolicy upgrade_policy_;
5268
std::unique_ptr<IReadable> input_;
53-
std::vector<std::uint8_t> buffer_;
69+
std::vector<std::uint8_t> read_buffer_;
5470
std::size_t buffer_idx_{};
71+
std::array<std::uint8_t, kMaxRecordLen> compat_buffer_{};
5572
Record current_record_{nullptr};
5673
};
5774
} // namespace databento

include/databento/dbn_file_store.hpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
11
#pragma once
22

3-
#include <memory> // unique_ptr
43
#include <string>
54

65
#include "databento/dbn.hpp" // Metadata
76
#include "databento/dbn_decoder.hpp" // DbnDecoder
7+
#include "databento/enums.hpp" // VersionUpgradePolicy
88
#include "databento/timeseries.hpp" // MetadataCallback, RecordCallback
99

1010
namespace databento {
1111
// A reader for DBN files.
1212
class DbnFileStore {
1313
public:
1414
explicit DbnFileStore(const std::string& file_path);
15+
DbnFileStore(const std::string& file_path,
16+
VersionUpgradePolicy upgrade_policy);
1517

1618
void Replay(const MetadataCallback& metadata_callback,
1719
const RecordCallback& record_callback);

0 commit comments

Comments
 (0)