MOD: Switch C++ client to use POST requests

Author: threecgreen

CHANGELOG.md

@@ -6,6 +6,9 @@
 - Added preliminary support for Windows
 - Added `LiveThreaded::BlockForStop` to make it easier to wait for one or more records
   before closing the session
+- Switch `BatchSubmitJob` to use form data to avoid query param length limit
+- Switch `SymbologyResolve` to use POST request with form data to avoid query param
+  length limit
 
 ## 0.9.1 - 2023-07-11
 

src/detail/http_client.cpp

@@ -41,10 +41,9 @@ nlohmann::json HttpClient::GetJson(const std::string& path,
 }
 
 nlohmann::json HttpClient::PostJson(const std::string& path,
-                                    const httplib::Params& params) {
-  // need to fully specify, otherwise sent as application/x-www-form-urlencoded
-  const std::string full_path = httplib::append_query_params(path, params);
-  httplib::Result res = client_.Post(full_path);
+                                    const httplib::Params& form_params) {
+  // params will be encoded as form data
+  httplib::Result res = client_.Post(path, {}, form_params);
   return HttpClient::CheckAndParseResponse(path, std::move(res));
 }
 

src/historical.cpp

@@ -846,7 +846,7 @@ databento::SymbologyResolution Historical::SymbologyResolve(
                          {"stype_out", ToString(stype_out)}};
   detail::SetIfNotEmpty(&params, "end_date", date_range.end);
   detail::SetIfNotEmpty(&params, "default_value", default_value);
-  const nlohmann::json json = client_.GetJson(kPath, params);
+  const nlohmann::json json = client_.PostJson(kPath, params);
   if (!json.is_object()) {
     throw JsonResponseError::TypeMismatch(kEndpoint, "object", json);
   }

test/include/mock/mock_http_server.hpp

@@ -40,6 +40,8 @@ class MockHttpServer {
  private:
   static void CheckParams(const std::map<std::string, std::string>& params,
                           const httplib::Request& req);
+  static void CheckFormParams(const std::map<std::string, std::string>& params,
+                              const httplib::Request& req);
 
   httplib::Server server_{};
   const int port_{};

test/src/historical_tests.cpp

@@ -647,16 +647,16 @@ TEST_F(HistoricalTests, TestSymbologyResolve) {
       {"status", 0},
   };
 
-  mock_server_.MockGetJson("/v0/symbology.resolve",
-                           {
-                               {"dataset", dataset::kGlbxMdp3},
-                               {"start_date", "2022-06-06"},
-                               {"end_date", "2022-06-10"},
-                               {"symbols", "ESM2"},
-                               {"stype_in", "raw_symbol"},
-                               {"stype_out", "instrument_id"},
-                           },
-                           kResp);
+  mock_server_.MockPostJson("/v0/symbology.resolve",
+                            {
+                                {"dataset", dataset::kGlbxMdp3},
+                                {"start_date", "2022-06-06"},
+                                {"end_date", "2022-06-10"},
+                                {"symbols", "ESM2"},
+                                {"stype_in", "raw_symbol"},
+                                {"stype_out", "instrument_id"},
+                            },
+                            kResp);
   const auto port = mock_server_.ListenOnThread();
 
   databento::Historical target{logger_.get(), kApiKey, "localhost",

test/src/mock_http_server.cpp

@@ -1,10 +1,12 @@
 #include "mock/mock_http_server.hpp"
 
-#include <fstream>    // ifstream
-#include <ios>        // streamsize
-#include <iostream>   // cerr
-#include <sstream>    // ostringstream
-#include <stdexcept>  // runtime_error
+#include <gtest/gtest.h>  // EXPECT_*
+#include <httplib.h>
+
+#include <fstream>   // ifstream
+#include <ios>       // streamsize
+#include <iostream>  // cerr
+#include <sstream>   // ostringstream
 #include <vector>
 
 using databento::mock::MockHttpServer;
@@ -54,16 +56,17 @@ void MockHttpServer::MockGetJson(
 }
 
 void MockHttpServer::MockPostJson(
-    const std::string& path, const std::map<std::string, std::string>& params,
+    const std::string& path,
+    const std::map<std::string, std::string>& form_params,
     const nlohmann::json& json) {
-  server_.Post(path, [json, params](const httplib::Request& req,
-                                    httplib::Response& resp) {
+  server_.Post(path, [json, form_params](const httplib::Request& req,
+                                         httplib::Response& resp) {
     if (!req.has_header("Authorization")) {
       resp.status = 401;
       return;
     }
     auto _auth = req.get_header_value("Authorization");
-    CheckParams(params, req);
+    CheckFormParams(form_params, req);
     resp.set_content(json.dump(), "application/json");
     resp.status = 200;
   });
@@ -110,19 +113,31 @@ void MockHttpServer::CheckParams(
     const std::map<std::string, std::string>& params,
     const httplib::Request& req) {
   for (const auto& param : params) {
-    if (!req.has_param(param.first)) {
-      std::ostringstream err_msg;
-      err_msg << "Missing query param " << param.first;
-      std::cerr << err_msg.str() << '\n';
-      throw std::runtime_error{err_msg.str()};
-    }
-    if (req.get_param_value(param.first) != param.second) {
-      std::ostringstream err_msg;
-      err_msg << "Incorrect query param value for " << param.first
-              << ". Expected " << param.second << ", found "
-              << req.get_param_value(param.first);
-      std::cerr << err_msg.str() << '\n';
-      throw std::runtime_error{err_msg.str()};
+    EXPECT_TRUE(req.has_param(param.first))
+        << "Missing query param " << param.first;
+    EXPECT_EQ(req.get_param_value(param.first), param.second)
+        << "Incorrect query param value for " << param.first << ". Expected "
+        << param.second << ", found " << req.get_param_value(param.first);
+  }
+}
+
+void MockHttpServer::CheckFormParams(
+    const std::map<std::string, std::string>& params,
+    const httplib::Request& req) {
+  EXPECT_EQ(req.get_header_value("content-type"),
+            "application/x-www-form-urlencoded")
+      << "Request body is not form data";
+  httplib::Params form_params;
+  httplib::detail::parse_query_text(req.body, form_params);
+  for (const auto& param : params) {
+    const auto param_it = form_params.find(param.first);
+    if (param_it == form_params.end()) {
+      EXPECT_NE(param_it, form_params.end())
+          << "Missing for mparam " << param.first;
+    } else {
+      EXPECT_EQ(param_it->second, param.second)
+          << "Incorrect form param value for " << param.first << ". Expected "
+          << param.second << ", found " << param_it->second;
     }
   }
 }