Skip to content

Commit d71c6ae

Browse files
threecgreenthreecgreen
committed
ADD: Add DBN decoding test
1 parent ffd4543 commit d71c6ae

File tree

5 files changed

+141
-35
lines changed

5 files changed

+141
-35
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
### Bug fixes
1616
- Added missing `IndexTs()` method to `v1::InstrumentDefMsg`, `v2::InstrumentDefMsg`,
1717
and `v1::SymbolMappingMsg`
18+
- Fixed error in templated overload of `DbnDecoder::EncodeRecord()`
1819

1920
## 0.39.1 - 2025-07-22
2021

‎include/databento/dbn_encoder.hpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,17 @@ class DbnEncoder {
1919
void EncodeRecord(const R& record) {
2020
static_assert(has_header<R>::value,
2121
"must be a DBN record struct with an `hd` RecordHeader field");
22-
EncodeRecord(Record{&record.hd});
22+
// Safe to cast away const as EncodeRecord will not modify data
23+
const Record rec{const_cast<RecordHeader*>(&record.hd)};
24+
EncodeRecord(rec);
2325
}
2426
template <typename R>
2527
void EncodeRecord(const WithTsOut<R> record) {
2628
static_assert(has_header<R>::value,
2729
"must be a DBN record struct with an `hd` RecordHeader field");
28-
EncodeRecord(Record{&record.rec.hd});
30+
// Safe to cast away const as EncodeRecord will not modify data
31+
const Record rec{const_cast<RecordHeader*>(&record.hd)};
32+
EncodeRecord(rec);
2933
}
3034
void EncodeRecord(const Record& record);
3135

‎tests/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ set(
3131
src/datetime_tests.cpp
3232
src/dbn_decoder_tests.cpp
3333
src/dbn_encoder_tests.cpp
34+
src/dbn_file_store_tests.cpp
3435
src/dbn_tests.cpp
3536
src/file_stream_tests.cpp
3637
src/flag_set_tests.cpp

‎tests/src/dbn_decoder_tests.cpp‎

Lines changed: 15 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -235,8 +235,7 @@ INSTANTIATE_TEST_SUITE_P(
235235
// Expected data for these tests obtained using the `dbn` CLI tool
236236

237237
TEST_P(DbnDecoderSchemaTests, TestDecodeMbo) {
238-
const auto extension = GetParam().first;
239-
const auto version = GetParam().second;
238+
const auto [extension, version] = GetParam();
240239
ReadFromFile("mbo", extension, version);
241240

242241
const Metadata metadata = target_->DecodeMetadata();
@@ -293,8 +292,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeMbo) {
293292
}
294293

295294
TEST_P(DbnDecoderSchemaTests, TestDecodeMbp1) {
296-
const auto extension = GetParam().first;
297-
const auto version = GetParam().second;
295+
const auto [extension, version] = GetParam();
298296
ReadFromFile("mbp-1", extension, version);
299297

300298
const Metadata metadata = target_->DecodeMetadata();
@@ -361,8 +359,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeMbp1) {
361359
}
362360

363361
TEST_P(DbnDecoderSchemaTests, TestDecodeMbp10) {
364-
const auto extension = GetParam().first;
365-
const auto version = GetParam().second;
362+
const auto [extension, version] = GetParam();
366363
ReadFromFile("mbp-10", extension, version);
367364

368365
const Metadata metadata = target_->DecodeMetadata();
@@ -452,8 +449,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeMbp10) {
452449
}
453450

454451
TEST_P(DbnDecoderSchemaTests, TestDecodeCmbp1) {
455-
const auto extension = GetParam().first;
456-
const auto version = GetParam().second;
452+
const auto [extension, version] = GetParam();
457453
ReadFromFile("cmbp-1", extension, version);
458454

459455
const Metadata metadata = target_->DecodeMetadata();
@@ -514,8 +510,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeCmbp1) {
514510
}
515511

516512
TEST_P(DbnDecoderSchemaTests, TestDecodeCbbo) {
517-
const auto extension = GetParam().first;
518-
const auto version = GetParam().second;
513+
const auto [extension, version] = GetParam();
519514
ReadFromFile("cbbo-1s", extension, version);
520515

521516
const Metadata metadata = target_->DecodeMetadata();
@@ -572,8 +567,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeCbbo) {
572567
}
573568

574569
TEST_P(DbnDecoderSchemaTests, TestDecodeTbbo) {
575-
const auto extension = GetParam().first;
576-
const auto version = GetParam().second;
570+
const auto [extension, version] = GetParam();
577571
ReadFromFile("tbbo", extension, version);
578572

579573
const Metadata metadata = target_->DecodeMetadata();
@@ -638,8 +632,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeTbbo) {
638632
}
639633

640634
TEST_P(DbnDecoderSchemaTests, TestDecodeTrades) {
641-
const auto extension = GetParam().first;
642-
const auto version = GetParam().second;
635+
const auto [extension, version] = GetParam();
643636
ReadFromFile("trades", extension, version);
644637

645638
const Metadata metadata = target_->DecodeMetadata();
@@ -692,8 +685,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeTrades) {
692685
}
693686

694687
TEST_P(DbnDecoderSchemaTests, TestDecodeOhlcv1D) {
695-
const auto extension = GetParam().first;
696-
const auto version = GetParam().second;
688+
const auto [extension, version] = GetParam();
697689
ReadFromFile("ohlcv-1d", extension, version);
698690

699691
const Metadata metadata = target_->DecodeMetadata();
@@ -712,8 +704,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeOhlcv1D) {
712704
}
713705

714706
TEST_P(DbnDecoderSchemaTests, TestDecodeOhlcv1H) {
715-
const auto extension = GetParam().first;
716-
const auto version = GetParam().second;
707+
const auto [extension, version] = GetParam();
717708
ReadFromFile("ohlcv-1h", extension, version);
718709

719710
const Metadata metadata = target_->DecodeMetadata();
@@ -758,8 +749,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeOhlcv1H) {
758749
}
759750

760751
TEST_P(DbnDecoderSchemaTests, TestDecodeOhlcv1M) {
761-
const auto extension = GetParam().first;
762-
const auto version = GetParam().second;
752+
const auto [extension, version] = GetParam();
763753
ReadFromFile("ohlcv-1m", extension, version);
764754

765755
const Metadata metadata = target_->DecodeMetadata();
@@ -804,8 +794,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeOhlcv1M) {
804794
}
805795

806796
TEST_P(DbnDecoderSchemaTests, TestDecodeOhlcv1S) {
807-
const auto extension = GetParam().first;
808-
const auto version = GetParam().second;
797+
const auto [extension, version] = GetParam();
809798
ReadFromFile("ohlcv-1s", extension, version);
810799

811800
const Metadata metadata = target_->DecodeMetadata();
@@ -850,8 +839,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeOhlcv1S) {
850839
}
851840

852841
TEST_P(DbnDecoderSchemaTests, TestDecodeDefinition) {
853-
const auto extension = GetParam().first;
854-
const auto version = GetParam().second;
842+
const auto [extension, version] = GetParam();
855843
ReadFromFile("definition", extension, version);
856844

857845
const Metadata metadata = target_->DecodeMetadata();
@@ -895,8 +883,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeDefinition) {
895883
}
896884

897885
TEST_P(DbnDecoderSchemaTests, TestDecodeImbalance) {
898-
const auto extension = GetParam().first;
899-
const auto version = GetParam().second;
886+
const auto [extension, version] = GetParam();
900887
ReadFromFile("imbalance", extension, version);
901888

902889
const Metadata metadata = target_->DecodeMetadata();
@@ -928,8 +915,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeImbalance) {
928915
}
929916

930917
TEST_P(DbnDecoderSchemaTests, TestDecodeStatistics) {
931-
const auto extension = GetParam().first;
932-
const auto version = GetParam().second;
918+
const auto [extension, version] = GetParam();
933919
ReadFromFile("statistics", extension, version);
934920

935921
const Metadata metadata = target_->DecodeMetadata();
@@ -1039,9 +1025,7 @@ INSTANTIATE_TEST_SUITE_P(
10391025
});
10401026

10411027
TEST_P(DbnIdentityTests, TestIdentity) {
1042-
const auto version = std::get<0>(GetParam());
1043-
const auto schema = std::get<1>(GetParam());
1044-
const auto compression = std::get<2>(GetParam());
1028+
const auto [version, schema, compression] = GetParam();
10451029
const auto file_name = std::string{TEST_DATA_DIR "/test_data."} + ToString(schema) +
10461030
".v" + std::to_string(+version) +
10471031
(compression == Compression::Zstd ? ".dbn.zst" : ".dbn");
@@ -1124,6 +1108,4 @@ TEST_P(DbnIdentityTests, TestIdentity) {
11241108
}
11251109
ASSERT_EQ(file_decoder.DecodeRecord(), nullptr);
11261110
}
1127-
1128-
TEST_F(DbnDecoderTests, TestDbnIdentityWithTsOut) {}
11291111
} // namespace databento::tests

‎tests/src/dbn_file_store_tests.cpp‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#include <chrono>
2+
#include <cstddef>
3+
#include <cstdint>
4+
5+
#include "databento/datetime.hpp"
6+
#include "databento/dbn.hpp"
7+
#include "databento/dbn_encoder.hpp"
8+
#include "databento/dbn_file_store.hpp"
9+
#include "databento/detail/zstd_stream.hpp"
10+
#include "databento/enums.hpp"
11+
#include "databento/file_stream.hpp"
12+
#include "databento/flag_set.hpp"
13+
#include "databento/record.hpp"
14+
#include "databento/v1.hpp"
15+
#include "temp_file.hpp"
16+
17+
namespace databento::tests {
18+
TEST(DbnFileStoreTests, TestDecodeExtended) {
19+
const auto record_gen = [](const std::size_t i) {
20+
auto flags = FlagSet{};
21+
if (i % 2 == 0) {
22+
flags.SetBadTsRecv();
23+
}
24+
Action action;
25+
switch (i % 5) {
26+
case 0: {
27+
action = Action::Add;
28+
break;
29+
}
30+
case 1: {
31+
action = Action::Modify;
32+
break;
33+
}
34+
case 2: {
35+
action = Action::Cancel;
36+
break;
37+
}
38+
case 3: {
39+
action = Action::Trade;
40+
break;
41+
}
42+
case 4:
43+
default: {
44+
action = Action::Fill;
45+
}
46+
}
47+
Side side;
48+
switch (i % 7) {
49+
case 0:
50+
case 1:
51+
case 2: {
52+
side = Side::Ask;
53+
}
54+
case 3:
55+
case 4:
56+
case 5: {
57+
side = Side::Bid;
58+
}
59+
case 6:
60+
default: {
61+
side = Side::None;
62+
}
63+
}
64+
return MboMsg{
65+
RecordHeader{sizeof(MboMsg) / RecordHeader::kLengthMultiplier, RType::Mbo,
66+
static_cast<std::uint16_t>(Publisher::GlbxMdp3Glbx),
67+
static_cast<std::uint32_t>(i),
68+
UnixNanos{std::chrono::milliseconds{i}}},
69+
i % 5,
70+
25'000'000 + static_cast<std::int64_t>(i),
71+
static_cast<std::uint32_t>(i) % 1'000,
72+
flags,
73+
static_cast<std::uint8_t>(i % 16),
74+
action,
75+
side,
76+
UnixNanos{std::chrono::milliseconds{i} +
77+
std::chrono::nanoseconds{10 + i % 100}},
78+
std::chrono::milliseconds{2 + i % 34},
79+
static_cast<std::uint32_t>(i) / 2
80+
81+
};
82+
};
83+
TempFile temp_file{std::filesystem::temp_directory_path() /
84+
"test_decode_extended.dbn.zst"};
85+
86+
constexpr auto kExpSize = 100'000;
87+
{
88+
OutFileStream out_file{temp_file.Path()};
89+
detail::ZstdCompressStream stream{&out_file};
90+
DbnEncoder encoder{Metadata{1,
91+
ToString(Dataset::GlbxMdp3),
92+
Schema::Mbo,
93+
{},
94+
{},
95+
{},
96+
{},
97+
{},
98+
false,
99+
databento::v1::kSymbolCstrLen,
100+
std::vector<std::string>(4571)},
101+
&stream};
102+
for (std::size_t i = 0; i < kExpSize; ++i) {
103+
auto flags = FlagSet{};
104+
encoder.EncodeRecord(record_gen(i));
105+
}
106+
}
107+
std::size_t count = 0;
108+
DbnFileStore target{temp_file.Path()};
109+
while (const auto* rec = target.NextRecord()) {
110+
const auto* mbo = rec->GetIf<MboMsg>();
111+
ASSERT_NE(mbo, nullptr) << "Found non-MBO record with hd = " << rec->Header()
112+
<< " at count = " << count;
113+
ASSERT_EQ(*mbo, record_gen(count)) << "MboMsg mismatch at count = " << count;
114+
++count;
115+
}
116+
ASSERT_EQ(count, kExpSize);
117+
}
118+
} // namespace databento::tests

0 commit comments

Comments
 (0)