ADD: Add support for warning header to C++ client

Author: threecgreen

CHANGELOG.md

@@ -5,6 +5,7 @@
 - Added optional `exception_callback` argument to `LiveThreaded::Start` to improve
   error handling options
 - Added batch download support data files (`condition.json` and `symbology.json`)
+- Added support for logging warnings from Historical API
 - Changed `use_ts_out` default to `false`
 - Fixed missing definition for `operator==` for `ImbalanceMsg`
 

cmake/SourcesAndHeaders.cmake

@@ -22,6 +22,7 @@ set(headers
   include/databento/with_ts_out.hpp
   include/databento/detail/file_stream.hpp
   include/databento/detail/http_client.hpp
+  include/databento/detail/json_helpers.hpp
   include/databento/detail/scoped_fd.hpp
   include/databento/detail/scoped_thread.hpp
   include/databento/detail/shared_channel.hpp
@@ -49,6 +50,7 @@ set(sources
   src/symbology.cpp
   src/detail/file_stream.cpp
   src/detail/http_client.cpp
+  src/detail/json_helpers.cpp
   src/detail/scoped_fd.cpp
   src/detail/shared_channel.cpp
   src/detail/tcp_client.cpp

cmake/StandardSettings.cmake

@@ -11,17 +11,6 @@ option(${PROJECT_NAME_UPPERCASE}_USE_EXTERNAL_GTEST "Use an external google test
 #
 
 option(${PROJECT_NAME_UPPERCASE}_WARNINGS_AS_ERRORS "Treat compiler warnings as errors." ${IS_MAIN_PROJECT})
-option(${PROJECT_NAME_UPPERCASE}_FORCE_COLOR_OUTPUT "Always produce ANSI-colored output" OFF)
-
-if(${PROJECT_NAME_UPPERCASE}_FORCE_COLOR_OUTPUT)
-  if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
-    add_compile_options(-fdiagnostics-color=always)
-  elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-    add_compile_options(-fcolor-diagnostics)
-  else()
-    message(WARNING "Couldn't force color output with unsupported compiler: ${CMAKE_CXX_COMPILER_ID}")
-  endif()
-endif()
 
 #
 # Unit testing

include/databento/detail/http_client.hpp

@@ -12,12 +12,14 @@
 #include <string>
 
 namespace databento {
+class ILogReceiver;
 namespace detail {
 class HttpClient {
  public:
-  HttpClient(const std::string& key, const std::string& gateway);
-  HttpClient(const std::string& key, const std::string& gateway,
-             std::uint16_t port);
+  HttpClient(ILogReceiver* log_receiver, const std::string& key,
+             const std::string& gateway);
+  HttpClient(ILogReceiver* log_receiver, const std::string& key,
+             const std::string& gateway, std::uint16_t port);
 
   nlohmann::json GetJson(const std::string& path,
                          const httplib::Params& params);
@@ -27,12 +29,14 @@ class HttpClient {
                     const httplib::ContentReceiver& callback);
 
  private:
-  static nlohmann::json CheckAndParseResponse(const std::string& path,
-                                              httplib::Result&& res);
+  nlohmann::json CheckAndParseResponse(const std::string& path,
+                                       httplib::Result&& res) const;
+  void CheckWarnings(const httplib::Response& response) const;
   static bool IsErrorStatus(int status_code);
 
   static const httplib::Headers kHeaders;
 
+  ILogReceiver* log_receiver_;
   httplib::Client client_;
 };
 }  // namespace detail

include/databento/detail/json_helpers.hpp

@@ -0,0 +1,91 @@
+#pragma once
+
+#include <nlohmann/json.hpp>
+
+#include <map>  // multimap
+#include <string>
+#include <vector>
+
+#include "databento/datetime.hpp"    // UnixNanos
+#include "databento/enums.hpp"       // FromString
+#include "databento/exceptions.hpp"  // JsonResponseError
+
+namespace httplib {
+using Params = std::multimap<std::string, std::string>;
+}
+
+namespace databento {
+namespace detail {
+void SetIfNotEmpty(httplib::Params* params, const std::string& key,
+                   const std::string& value);
+void SetIfNotEmpty(httplib::Params* params, const std::string& key,
+                   const std::vector<databento::JobState>& states);
+
+template <typename T>
+void SetIfPositive(httplib::Params* params, const std::string& key,
+                   const T value) {
+  if (value > 0) {
+    params->emplace(key, std::to_string(value));
+  }
+}
+
+template <>
+inline void SetIfPositive<databento::UnixNanos>(
+    httplib::Params* params, const std::string& key,
+    const databento::UnixNanos value) {
+  if (value.time_since_epoch().count()) {
+    params->emplace(key, databento::ToString(value));
+  }
+}
+
+const nlohmann::json& CheckedAt(const std::string& endpoint,
+                                const nlohmann::json& json,
+                                const std::string& key);
+
+template <typename T>
+T FromCheckedAtString(const std::string& endpoint, const nlohmann::json& json,
+                      const std::string& key) {
+  const auto& val_json = CheckedAt(endpoint, json, key);
+  if (!val_json.is_string()) {
+    throw JsonResponseError::TypeMismatch(endpoint, key + " string", val_json);
+  }
+  return databento::FromString<T>(val_json);
+}
+
+template <typename T>
+T FromCheckedAtStringOrNull(const std::string& endpoint,
+                            const nlohmann::json& json, const std::string& key,
+                            T null_value) {
+  const auto& val_json = CheckedAt(endpoint, json, key);
+  if (val_json.is_null()) {
+    return null_value;
+  }
+  if (val_json.is_string()) {
+    return databento::FromString<T>(val_json);
+  }
+  throw JsonResponseError::TypeMismatch(endpoint, key + " null or string",
+                                        val_json);
+}
+
+template <typename T>
+T ParseAt(const std::string& endpoint, const nlohmann::json& json,
+          const std::string& key);
+template <>
+bool ParseAt(const std::string& endpoint, const nlohmann::json& json,
+             const std::string& key);
+template <>
+std::string ParseAt(const std::string& endpoint, const nlohmann::json& json,
+                    const std::string& key);
+template <>
+std::size_t ParseAt(const std::string& endpoint, const nlohmann::json& json,
+                    const std::string& key);
+template <>
+double ParseAt(const std::string& endpoint, const nlohmann::json& json,
+               const std::string& key);
+template <>
+std::vector<std::string> ParseAt(const std::string& endpoint,
+                                 const nlohmann::json& json,
+                                 const std::string& key);
+
+}  // namespace detail
+}  // namespace databento

include/databento/historical.hpp

@@ -16,12 +16,16 @@
 #include "databento/timeseries.hpp"  // KeepGoing, MetadataCallback, RecordCallback
 
 namespace databento {
+class ILogReceiver;
+
 // A client for interfacing with Databento's historical market data API.
 class Historical {
  public:
-  Historical(std::string key, HistoricalGateway gateway);
+  Historical(ILogReceiver* log_receiver, std::string key,
+             HistoricalGateway gateway);
   // Primarily for unit tests
-  Historical(std::string key, std::string gateway, std::uint16_t port);
+  Historical(ILogReceiver* log_receiver, std::string key, std::string gateway,
+             std::uint16_t port);
 
   /*
    * Getters
@@ -248,6 +252,7 @@ class Historical {
   DbnFileStore TimeseriesGetRangeToFile(const HttplibParams& params,
                                         const std::string& file_path);
 
+  ILogReceiver* log_receiver_;
   const std::string key_;
   const std::string gateway_;
   detail::HttpClient client_;
@@ -265,11 +270,14 @@ class HistoricalBuilder {
   HistoricalBuilder& SetKeyFromEnv();
   HistoricalBuilder& SetKey(std::string key);
   HistoricalBuilder& SetGateway(HistoricalGateway gateway);
+  // Sets the receiver of the logs to be used by the client.
+  HistoricalBuilder& SetLogReceiver(ILogReceiver* log_receiver);
   // Attempts to construct an instance of Historical or throws an exception if
   // no key has been set.
   Historical Build();
 
  private:
+  ILogReceiver* log_receiver_{};
   std::string key_;
   HistoricalGateway gateway_{HistoricalGateway::Bo1};
 };

src/detail/http_client.cpp

@@ -1,8 +1,10 @@
 #include "databento/detail/http_client.hpp"
 
-#include <chrono>  // seconds
+#include <chrono>   // seconds
+#include <sstream>  // ostringstream
 
 #include "databento/exceptions.hpp"  // HttpResponseError, HttpRequestError, JsonResponseError
+#include "databento/log.hpp"      // ILogReceiver, LogLevel
 #include "databento/version.hpp"  // DATABENTO_VERSION
 
 using databento::detail::HttpClient;
@@ -13,17 +15,19 @@ const httplib::Headers HttpClient::kHeaders{
     {"user-agent", "Databento/" DATABENTO_VERSION " C++"},
 };
 
-HttpClient::HttpClient(const std::string& key, const std::string& gateway)
-    : client_{gateway} {
+HttpClient::HttpClient(databento::ILogReceiver* log_receiver,
+                       const std::string& key, const std::string& gateway)
+    : log_receiver_{log_receiver}, client_{gateway} {
   client_.set_default_headers(HttpClient::kHeaders);
   client_.set_basic_auth(key, "");
   client_.set_read_timeout(kTimeout);
   client_.set_write_timeout(kTimeout);
 }
 
-HttpClient::HttpClient(const std::string& key, const std::string& gateway,
+HttpClient::HttpClient(databento::ILogReceiver* log_receiver,
+                       const std::string& key, const std::string& gateway,
                        std::uint16_t port)
-    : client_{gateway, port} {
+    : log_receiver_{log_receiver}, client_{gateway, port} {
   client_.set_default_headers(HttpClient::kHeaders);
   client_.set_basic_auth(key, "");
   client_.set_read_timeout(kTimeout);
@@ -79,19 +83,45 @@ void HttpClient::GetRawStream(const std::string& path,
 }
 
 nlohmann::json HttpClient::CheckAndParseResponse(const std::string& path,
-                                                 httplib::Result&& res) {
+                                                 httplib::Result&& res) const {
   if (res.error() != httplib::Error::Success) {
     throw HttpRequestError{path, res.error()};
   }
-  const auto status_code = res.value().status;
+  auto& response = res.value();
+  const auto status_code = response.status;
   if (HttpClient::IsErrorStatus(status_code)) {
-    throw HttpResponseError{path, status_code, std::move(res.value().body)};
+    throw HttpResponseError{path, status_code, std::move(response.body)};
   }
+  CheckWarnings(response);
   try {
-    return nlohmann::json::parse(std::move(res.value().body));
+    return nlohmann::json::parse(std::move(response.body));
   } catch (const nlohmann::json::parse_error& parse_err) {
     throw JsonResponseError::ParseError(path, parse_err);
   }
 }
 
+void HttpClient::CheckWarnings(const httplib::Response& response) const {
+  // Returns empty string if not found. `get_header_value` is case insensitive
+  const auto raw = response.get_header_value("X-Warning");
+  if (!raw.empty()) {
+    try {
+      const auto json = nlohmann::json::parse(raw);
+      if (json.is_array()) {
+        for (const auto& warning_json : json.items()) {
+          const std::string warning = warning_json.value();
+          std::ostringstream msg;
+          msg << __PRETTY_FUNCTION__ << " Server " << warning;
+          log_receiver_->Receive(LogLevel::Warning, msg.str());
+        }
+        return;
+      }
+    } catch (const std::exception&) {
+    }
+    std::ostringstream msg;
+    msg << __PRETTY_FUNCTION__
+        << " Failed to parse warnings from HTTP header. Raw contents: " << raw;
+    log_receiver_->Receive(LogLevel::Warning, msg.str());
+  }
+}
+
 bool HttpClient::IsErrorStatus(int status_code) { return status_code >= 400; }