MOD: Use POST for TimeseriesGetRange methods

threecgreen · threecgreen · commit 6cd9ad920366 · 2025-06-25T14:28:02.000Z
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,8 @@
 
 ### Enhancements
 - Added `range_by_schema` field to `DatasetRange` struct
+- Changed historical `TimeseriesGetRange` and `TimeseriesGetRangeToFile` methods to use
+  a `POST` request to allow for requesting supported maximum of 2000 symbols
 - Added logging around `Historical::BatchDownload`
 - Changed the following Venue, Publisher, and Dataset descriptions:
   - "ICE Futures Europe (Financials)" renamed to "ICE Europe Financials"
diff --git a/include/databento/detail/http_client.hpp b/include/databento/detail/http_client.hpp
@@ -24,15 +24,23 @@ class HttpClient {
   nlohmann::json GetJson(const std::string& path,
                          const httplib::Params& params);
   nlohmann::json PostJson(const std::string& path,
-                          const httplib::Params& params);
+                          const httplib::Params& form_params);
   void GetRawStream(const std::string& path, const httplib::Params& params,
                     const httplib::ContentReceiver& callback);
+  void PostRawStream(const std::string& path,
+                     const httplib::Params& form_params,
+                     const httplib::ContentReceiver& callback);
 
  private:
+  static bool IsErrorStatus(int status_code);
+  static httplib::ResponseHandler MakeStreamResponseHandler(int& out_status);
+  static void CheckStatusAndStreamRes(const std::string& path, int status_code,
+                                      std::string&& err_body,
+                                      const httplib::Result& res);
+
   nlohmann::json CheckAndParseResponse(const std::string& path,
                                        httplib::Result&& res) const;
   void CheckWarnings(const httplib::Response& response) const;
-  static bool IsErrorStatus(int status_code);
 
   static const httplib::Headers kHeaders;
 
diff --git a/include/databento/historical.hpp b/include/databento/historical.hpp
@@ -230,8 +230,6 @@ class Historical {
   using HttplibParams = std::multimap<std::string, std::string>;
 
   BatchJob BatchSubmitJob(const HttplibParams& params);
-  void StreamToFile(const std::string& url_path, const HttplibParams& params,
-                    const std::filesystem::path& file_path);
   void DownloadFile(const std::string& url,
                     const std::filesystem::path& output_path);
   std::vector<BatchJob> BatchListJobs(const HttplibParams& params);
diff --git a/src/detail/http_client.cpp b/src/detail/http_client.cpp
@@ -53,25 +53,64 @@ void HttpClient::GetRawStream(const std::string& path,
   const std::string full_path = httplib::append_query_params(path, params);
   std::string err_body{};
   int err_status{};
-  const httplib::Result res = client_.Get(
-      full_path,
-      [&err_status](const httplib::Response& resp) {
-        if (HttpClient::IsErrorStatus(resp.status)) {
-          err_status = resp.status;
-        }
-        return true;
-      },
-      [&callback, &err_body, &err_status](const char* data,
-                                          std::size_t length) {
-        // if an error response was received, read all content into err_status
-        if (err_status > 0) {
-          err_body.append(data, length);
-          return true;
-        }
-        return callback(data, length);
-      });
-  if (err_status > 0) {
-    throw HttpResponseError{path, err_status, std::move(err_body)};
+  const httplib::Result res =
+      client_.Get(full_path, MakeStreamResponseHandler(err_status),
+                  [&callback, &err_body, &err_status](const char* data,
+                                                      std::size_t length) {
+                    // if an error response was received, read all content into
+                    // err_body
+                    if (err_status > 0) {
+                      err_body.append(data, length);
+                      return true;
+                    }
+                    return callback(data, length);
+                  });
+  CheckStatusAndStreamRes(path, err_status, std::move(err_body), res);
+}
+
+void HttpClient::PostRawStream(const std::string& path,
+                               const httplib::Params& form_params,
+                               const httplib::ContentReceiver& callback) {
+  std::string err_body{};
+  int err_status{};
+  httplib::Request req;
+  req.method = "POST";
+  req.set_header("Content-Type", "application/x-www-form-urlencoded");
+  req.path = path;
+  req.body = httplib::detail::params_to_query_str(form_params);
+  req.response_handler = MakeStreamResponseHandler(err_status);
+  req.content_receiver = [&callback, &err_body, &err_status](
+                             const char* data, std::size_t length,
+                             std::uint64_t, std::uint64_t) {
+    // if an error response was received, read all content into
+    // err_body
+    if (err_status > 0) {
+      err_body.append(data, length);
+      return true;
+    }
+    return callback(data, length);
+  };
+  // NOLINTNEXTLINE(clang-analyzer-unix.BlockInCriticalSection): dependency code
+  const httplib::Result res = client_.send(req);
+  CheckStatusAndStreamRes(path, err_status, std::move(err_body), res);
+}
+
+httplib::ResponseHandler HttpClient::MakeStreamResponseHandler(
+    int& out_status) {
+  return [&out_status](const httplib::Response& resp) {
+    if (HttpClient::IsErrorStatus(resp.status)) {
+      out_status = resp.status;
+    }
+    return true;
+  };
+}
+
+void HttpClient::CheckStatusAndStreamRes(const std::string& path,
+                                         int status_code,
+                                         std::string&& err_body,
+                                         const httplib::Result& res) {
+  if (status_code > 0) {
+    throw HttpResponseError{path, status_code, std::move(err_body)};
   }
   if (res.error() != httplib::Error::Success &&
       // canceled happens if `callback` returns false, which is based on the
diff --git a/src/historical.cpp b/src/historical.cpp
@@ -316,17 +316,6 @@ std::filesystem::path Historical::BatchDownload(
   return output_path;
 }
 
-void Historical::StreamToFile(const std::string& url_path,
-                              const HttplibParams& params,
-                              const std::filesystem::path& file_path) {
-  OutFileStream out_file{file_path};
-  this->client_.GetRawStream(
-      url_path, params, [&out_file](const char* data, std::size_t length) {
-        out_file.WriteAll(reinterpret_cast<const std::byte*>(data), length);
-        return true;
-      });
-}
-
 void Historical::DownloadFile(const std::string& url,
                               const std::filesystem::path& output_path) {
   static const std::string kMethod = "Historical::DownloadFile";
@@ -352,7 +341,12 @@ void Historical::DownloadFile(const std::string& url,
      << output_path;
   log_receiver_->Receive(LogLevel::Info, ss.str());
 
-  StreamToFile(path, {}, output_path);
+  OutFileStream out_file{output_path};
+  this->client_.GetRawStream(
+      path, {}, [&out_file](const char* data, std::size_t length) {
+        out_file.WriteAll(reinterpret_cast<const std::byte*>(data), length);
+        return true;
+      });
 
   if (log_receiver_->ShouldLog(LogLevel::Debug)) {
     ss.str("");
@@ -880,7 +874,7 @@ void Historical::TimeseriesGetRange(const HttplibParams& params,
   detail::DbnBufferDecoder decoder{metadata_callback, record_callback};
 
   bool early_exit = false;
-  this->client_.GetRawStream(
+  this->client_.PostRawStream(
       kTimeseriesGetRangePath, params,
       [&decoder, &early_exit](const char* data, std::size_t length) mutable {
         if (decoder.Process(data, length) == KeepGoing::Continue) {
@@ -959,7 +953,15 @@ databento::DbnFileStore Historical::TimeseriesGetRangeToFile(
 }
 databento::DbnFileStore Historical::TimeseriesGetRangeToFile(
     const HttplibParams& params, const std::filesystem::path& file_path) {
-  StreamToFile(kTimeseriesGetRangePath, params, file_path);
+  {
+    OutFileStream out_file{file_path};
+    this->client_.PostRawStream(
+        kTimeseriesGetRangePath, params,
+        [&out_file](const char* data, std::size_t length) {
+          out_file.WriteAll(reinterpret_cast<const std::byte*>(data), length);
+          return true;
+        });
+  }  // Flush out_file
   return DbnFileStore{log_receiver_, file_path,
                       VersionUpgradePolicy::UpgradeToV3};
 }
diff --git a/tests/include/mock/mock_http_server.hpp b/tests/include/mock/mock_http_server.hpp
@@ -5,8 +5,10 @@
 
 #include <cstddef>
 #include <map>
+#include <memory>
 #include <string>
 
+#include "databento/detail/buffer.hpp"
 #include "databento/detail/scoped_thread.hpp"
 #include "databento/record.hpp"
 
@@ -23,7 +25,7 @@ class MockHttpServer {
   ~MockHttpServer() { server_.stop(); }
 
   int ListenOnThread();
-  void MockBadRequest(const std::string& path, const nlohmann::json& json);
+  void MockBadPostRequest(const std::string& path, const nlohmann::json& json);
   void MockGetJson(const std::string& path, const nlohmann::json& json);
   void MockGetJson(const std::string& path,
                    const std::map<std::string, std::string>& params,
@@ -34,22 +36,31 @@ class MockHttpServer {
   void MockPostJson(const std::string& path,
                     const std::map<std::string, std::string>& params,
                     const nlohmann::json& json);
-  void MockStreamDbn(const std::string& path,
-                     const std::map<std::string, std::string>& params,
-                     const std::string& dbn_path);
-  void MockStreamDbn(const std::string& path,
-                     const std::map<std::string, std::string>& params,
-                     Record record, std::size_t count, std::size_t chunk_size);
-  void MockStreamDbn(const std::string& path,
-                     const std::map<std::string, std::string>& params,
-                     Record record, std::size_t count, std::size_t extra_bytes,
-                     std::size_t chunk_size);
+  void MockGetDbn(const std::string& path,
+                  const std::map<std::string, std::string>& params,
+                  const std::string& dbn_path);
+  void MockPostDbn(const std::string& path,
+                   const std::map<std::string, std::string>& params,
+                   const std::string& dbn_path);
+  void MockPostDbn(const std::string& path,
+                   const std::map<std::string, std::string>& params,
+                   Record record, std::size_t count, std::size_t chunk_size);
+  void MockPostDbn(const std::string& path,
+                   const std::map<std::string, std::string>& params,
+                   Record record, std::size_t count, std::size_t extra_bytes,
+                   std::size_t chunk_size);
 
  private:
+  using SharedConstBuffer = std::shared_ptr<const detail::Buffer>;
+
   static void CheckParams(const std::map<std::string, std::string>& params,
                           const httplib::Request& req);
   static void CheckFormParams(const std::map<std::string, std::string>& params,
                               const httplib::Request& req);
+  static SharedConstBuffer EncodeToBuffer(const std::string& dbn_path);
+  static httplib::Server::Handler MakeDbnStreamHandler(
+      const std::map<std::string, std::string>& params,
+      SharedConstBuffer&& buffer, std::size_t chunk_size);
 
   httplib::Server server_{};
   const int port_{};
diff --git a/tests/src/historical_tests.cpp b/tests/src/historical_tests.cpp
@@ -241,8 +241,8 @@ TEST_F(HistoricalTests, TestBatchDownloadAll) {
   const TempFile temp_dbn_file{tmp_path_ / "job123/test.dbn"};
   mock_server_.MockGetJson("/v0/batch.list_files", {{"job_id", kJobId}},
                            kListFilesResp);
-  mock_server_.MockStreamDbn("/v0/job_id/test.dbn", {},
-                             TEST_DATA_DIR "/test_data.mbo.v3.dbn");
+  mock_server_.MockGetDbn("/v0/job_id/test.dbn", {},
+                          TEST_DATA_DIR "/test_data.mbo.v3.dbn");
   mock_server_.MockGetJson("/v0/job_id/test_metadata.json", {{"key", "value"}});
   const auto port = mock_server_.ListenOnThread();
 
@@ -640,17 +640,17 @@ TEST_F(HistoricalTests, TestSymbologyResolve) {
 }
 
 TEST_F(HistoricalTests, TestTimeseriesGetRange_Basic) {
-  mock_server_.MockStreamDbn("/v0/timeseries.get_range",
-                             {{"dataset", dataset::kGlbxMdp3},
-                              {"symbols", "ESH1"},
-                              {"schema", "mbo"},
-                              {"start", "1609160400000711344"},
-                              {"end", "1609160800000711344"},
-                              {"encoding", "dbn"},
-                              {"stype_in", "raw_symbol"},
-                              {"stype_out", "instrument_id"},
-                              {"limit", "2"}},
-                             TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst");
+  mock_server_.MockPostDbn("/v0/timeseries.get_range",
+                           {{"dataset", dataset::kGlbxMdp3},
+                            {"symbols", "ESH1"},
+                            {"schema", "mbo"},
+                            {"start", "1609160400000711344"},
+                            {"end", "1609160800000711344"},
+                            {"encoding", "dbn"},
+                            {"stype_in", "raw_symbol"},
+                            {"stype_out", "instrument_id"},
+                            {"limit", "2"}},
+                           TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst");
   const auto port = mock_server_.ListenOnThread();
 
   databento::Historical target{&logger_, kApiKey, "localhost",
@@ -676,16 +676,16 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_Basic) {
 }
 
 TEST_F(HistoricalTests, TestTimeseriesGetRange_NoMetadataCallback) {
-  mock_server_.MockStreamDbn("/v0/timeseries.get_range",
-                             {{"dataset", dataset::kGlbxMdp3},
-                              {"start", "2022-10-21T13:30"},
-                              {"end", "2022-10-21T20:00"},
-                              {"symbols", "CYZ2"},
-                              {"schema", "tbbo"},
-                              {"encoding", "dbn"},
-                              {"stype_in", "raw_symbol"},
-                              {"stype_out", "instrument_id"}},
-                             TEST_DATA_DIR "/test_data.tbbo.v3.dbn.zst");
+  mock_server_.MockPostDbn("/v0/timeseries.get_range",
+                           {{"dataset", dataset::kGlbxMdp3},
+                            {"start", "2022-10-21T13:30"},
+                            {"end", "2022-10-21T20:00"},
+                            {"symbols", "CYZ2"},
+                            {"schema", "tbbo"},
+                            {"encoding", "dbn"},
+                            {"stype_in", "raw_symbol"},
+                            {"stype_out", "instrument_id"}},
+                           TEST_DATA_DIR "/test_data.tbbo.v3.dbn.zst");
   const auto port = mock_server_.ListenOnThread();
 
   databento::Historical target{&logger_, kApiKey, "localhost",
@@ -704,7 +704,7 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_NoMetadataCallback) {
 TEST_F(HistoricalTests, TestTimeseriesGetRange_BadRequest) {
   const nlohmann::json resp{
       {"detail", "Authorization failed: illegal chars in username."}};
-  mock_server_.MockBadRequest("/v0/timeseries.get_range", resp);
+  mock_server_.MockBadPostRequest("/v0/timeseries.get_range", resp);
   const auto port = mock_server_.ListenOnThread();
 
   databento::Historical target{&logger_, kApiKey, "localhost",
@@ -729,8 +729,8 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_BadRequest) {
 }
 
 TEST_F(HistoricalTests, TestTimeseriesGetRange_CallbackException) {
-  mock_server_.MockStreamDbn("/v0/timeseries.get_range", {},
-                             TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst");
+  mock_server_.MockPostDbn("/v0/timeseries.get_range", {},
+                           TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst");
   const auto port = mock_server_.ListenOnThread();
 
   databento::Historical target{&logger_, kApiKey, "localhost",
@@ -747,8 +747,8 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_CallbackException) {
 }
 
 TEST_F(HistoricalTests, TestTimeseriesGetRange_Cancellation) {
-  mock_server_.MockStreamDbn("/v0/timeseries.get_range", {},
-                             TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst");
+  mock_server_.MockPostDbn("/v0/timeseries.get_range", {},
+                           TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst");
   const auto port = mock_server_.ListenOnThread();
 
   databento::Historical target{&logger_, kApiKey, "localhost",
@@ -777,9 +777,9 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_LargeChunks) {
                    10005,
                    {}}};
   constexpr auto kRecordCount = 50'000;
-  mock_server_.MockStreamDbn("/v0/timeseries.get_range",
-                             {{"dataset", ToString(Dataset::IfusImpact)}},
-                             Record{&mbp1.hd}, kRecordCount, 75'000);
+  mock_server_.MockPostDbn("/v0/timeseries.get_range",
+                           {{"dataset", ToString(Dataset::IfusImpact)}},
+                           Record{&mbp1.hd}, kRecordCount, 75'000);
   const auto port = mock_server_.ListenOnThread();
 
   databento::Historical target{&logger_, kApiKey, "localhost",
@@ -804,9 +804,9 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_UnreadBytes) {
                    10005,
                    {}}};
   constexpr auto kRecordCount = 1'000;
-  mock_server_.MockStreamDbn("/v0/timeseries.get_range",
-                             {{"dataset", ToString(Dataset::IfusImpact)}},
-                             Record{&mbp1.hd}, kRecordCount, 20, 75'000);
+  mock_server_.MockPostDbn("/v0/timeseries.get_range",
+                           {{"dataset", ToString(Dataset::IfusImpact)}},
+                           Record{&mbp1.hd}, kRecordCount, 20, 75'000);
   const auto port = mock_server_.ListenOnThread();
 
   logger_ = mock::MockLogReceiver{[](auto count, LogLevel level,
@@ -830,16 +830,16 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_UnreadBytes) {
 }
 
 TEST_F(HistoricalTests, TestTimeseriesGetRangeToFile) {
-  mock_server_.MockStreamDbn("/v0/timeseries.get_range",
-                             {{"dataset", dataset::kGlbxMdp3},
-                              {"start", "2022-10-21T13:30"},
-                              {"end", "2022-10-21T20:00"},
-                              {"symbols", "CYZ2"},
-                              {"schema", "tbbo"},
-                              {"encoding", "dbn"},
-                              {"stype_in", "raw_symbol"},
-                              {"stype_out", "instrument_id"}},
-                             TEST_DATA_DIR "/test_data.tbbo.v3.dbn.zst");
+  mock_server_.MockPostDbn("/v0/timeseries.get_range",
+                           {{"dataset", dataset::kGlbxMdp3},
+                            {"start", "2022-10-21T13:30"},
+                            {"end", "2022-10-21T20:00"},
+                            {"symbols", "CYZ2"},
+                            {"schema", "tbbo"},
+                            {"encoding", "dbn"},
+                            {"stype_in", "raw_symbol"},
+                            {"stype_out", "instrument_id"}},
+                           TEST_DATA_DIR "/test_data.tbbo.v3.dbn.zst");
   const auto port = mock_server_.ListenOnThread();
 
   databento::Historical target{&logger_, kApiKey, "localhost",
diff --git a/tests/src/mock_http_server.cpp b/tests/src/mock_http_server.cpp
