From 9826c1ba8a38168f6cafd92852be9ef11529f4b2 Mon Sep 17 00:00:00 2001 From: karel rehor Date: Wed, 26 Feb 2025 16:47:35 +0100 Subject: [PATCH 01/12] feat: (WIP) refactor query api to use options class --- influxdb_client_3/__init__.py | 13 +++- influxdb_client_3/query/query_api.py | 60 +++++++++++++++- tests/test_query.py | 104 +++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 3 deletions(-) diff --git a/influxdb_client_3/__init__.py b/influxdb_client_3/__init__.py index 40ab8a4..07e2e95 100644 --- a/influxdb_client_3/__init__.py +++ b/influxdb_client_3/__init__.py @@ -2,7 +2,7 @@ import pyarrow as pa import importlib.util -from influxdb_client_3.query.query_api import QueryApi as _QueryApi +from influxdb_client_3.query.query_api import QueryApi as _QueryApi, QueryApiOptionsBuilder from influxdb_client_3.read_file import UploadFile from influxdb_client_3.write_client import InfluxDBClient as _InfluxDBClient, WriteOptions, Point from influxdb_client_3.write_client.client.exceptions import InfluxDBError @@ -165,9 +165,18 @@ def __init__( connection_string = f"grpc+tls://{hostname}:{port}" else: connection_string = f"grpc+tcp://{hostname}:{port}" + + print(f"\nDEBUG kwargs.keys {kwargs.keys()}") + q_opts_builder = QueryApiOptionsBuilder() + if kwargs.keys().__contains__('ssl_ca_cert'): + q_opts_builder.root_certs(kwargs.get('ssl_ca_cert', None)) + if kwargs.keys().__contains__('verify_ssl'): + q_opts_builder.tls_verify(kwargs.get('verify_ssl', True)) + if kwargs.keys().__contains__('proxy'): + q_opts_builder.proxy(kwargs.get('proxy', None)) self._query_api = _QueryApi(connection_string=connection_string, token=token, flight_client_options=flight_client_options, - proxy=kwargs.get("proxy", None)) + proxy=kwargs.get("proxy", None), options=q_opts_builder.build()) def write(self, record=None, database=None, **kwargs): """ diff --git a/influxdb_client_3/query/query_api.py b/influxdb_client_3/query/query_api.py index 7d03a95..c91896d 100644 --- a/influxdb_client_3/query/query_api.py +++ b/influxdb_client_3/query/query_api.py @@ -6,6 +6,55 @@ from pyarrow.flight import FlightClient, Ticket, FlightCallOptions, FlightStreamReader from influxdb_client_3.version import USER_AGENT +class QueryApiOptions(object): + tls_root_certs = None + tls_verify = True + proxy = None + flight_client_options = None + + def __init__(self, root_certs_path, verify, proxy, flight_client_options): + if root_certs_path: + self.tls_root_certs = self._read_certs(root_certs_path) + self.tls_verify = verify + self.proxy = proxy + self.flight_client_options = flight_client_options + + def _read_certs(self, path): + with open(path, "rb") as certs_file: + return certs_file.read() + + +class QueryApiOptionsBuilder(object): + + _root_certs_path = None + _tls_verify = True + _proxy = None + _flight_client_options = None + + def root_certs(self, path): + self._root_certs_path = path + return self + + def tls_verify(self, verify): + self._tls_verify = verify + return self + + def proxy(self, proxy): + self._proxy = proxy + return self + + def flight_client_options(self, flight_client_options): + self._flight_client_options = flight_client_options + return self + + def build(self): + return QueryApiOptions( + root_certs_path=self._root_certs_path, + verify=self._tls_verify, + proxy=self._proxy, + flight_client_options=self._flight_client_options + ) + class QueryApi(object): """ @@ -26,7 +75,7 @@ def __init__(self, connection_string, token, flight_client_options, - proxy=None) -> None: + proxy=None, options=None) -> None: """ Initialize defaults. @@ -37,6 +86,15 @@ def __init__(self, self._token = token self._flight_client_options = flight_client_options or {} self._proxy = proxy + print(f"\nDEBUG options {options}") + if options: + if options.flight_client_options: + self._flight_client_options = options.flight_client_options + if options.tls_root_certs: + self._flight_client_options["tls_root_certs"] = options.tls_root_certs + if options.proxy: + self._proxy = options.proxy + self._flight_client_options["disable_server_verification"] = not options.tls_verify self._flight_client_options["generic_options"] = [ ("grpc.secondary_user_agent", USER_AGENT) ] diff --git a/tests/test_query.py b/tests/test_query.py index 1c6ab43..e1e1c48 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1,5 +1,6 @@ import unittest import struct +import os from unittest.mock import Mock, ANY from pyarrow import ( @@ -18,6 +19,7 @@ ) from influxdb_client_3 import InfluxDBClient3 +from influxdb_client_3.query.query_api import QueryApiOptionsBuilder, QueryApi from influxdb_client_3.version import USER_AGENT @@ -115,6 +117,27 @@ def test_influx_default_query_headers(): class TestQuery(unittest.TestCase): + sample_cert = """-----BEGIN CERTIFICATE----- +MIIDUzCCAjugAwIBAgIUZB55ULutbc9gy6xLp1BkTQU7siowDQYJKoZIhvcNAQEL +BQAwNjE0MDIGA1UEAwwraW5mbHV4ZGIzLWNsdXN0ZXJlZC1zd2FuLmJyYW1ib3Jh +LnpvbmEtYi5ldTAeFw0yNTAyMTgxNTIyMTJaFw0yNjAyMTgxNTIyMTJaMDYxNDAy +BgNVBAMMK2luZmx1eGRiMy1jbHVzdGVyZWQtc3dhbi5icmFtYm9yYS56b25hLWIu +ZXUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCugeNrx0ZfyyP8H4e0 +zDSkKWnEXlVdjMi+ZSHhMbjvvqMkUQGLc/W59AEmMJ0Uiljka9d+F7jdu+oqDq9p +4kGPhO3Oh7zIG0IGbncj8AwIXMGDNkNyL8s7C1+LoYotlSWDpWwkEKXUeAzdqS63 +CSJFqSJM2dss8qe9BpM6zHWJAKS1I30QT3SXQFEsF5m2F62dXCEEI6pO7jlik8/w +aI47dTM20QyimVzea48SC/ELO/T4AjbmMeBGlTyCm39KOElOKRTJvB4KESEWaL3r +EvPZbTh+72PUyrjxiDa56+RmtDPo7EN3uxuRVFX/HWiNnFk7orQLKZg5Kr8wE46R +KmVvAgMBAAGjWTBXMDYGA1UdEQQvMC2CK2luZmx1eGRiMy1jbHVzdGVyZWQtc3dh +bi5icmFtYm9yYS56b25hLWIuZXUwHQYDVR0OBBYEFH8et6JCzGD7Ny84aNRtq5Nj +hvS/MA0GCSqGSIb3DQEBCwUAA4IBAQCuDwARea/Xr3+hmte9A0H+XB8wMPAJ64e8 +QA0qi0oy0gGdLfQHhsBWWmKSYLv7HygTNzb+7uFOTtq1UPLt18F+POPeLIj74QZV +z89Pbo1TwUMzQ2pgbu0yRvraXIpqXGrPm5GWYp5mopX0rBWKdimbmEMkhZA0sVeH +IdKIRUY6EyIVG+Z/nbuVqUlgnIWOMp0yg4RRC91zHy3Xvykf3Vai25H/jQpa6cbU +//MIodzUIqT8Tja5cHXE51bLdUkO1rtNKdM7TUdjzkZ+bAOpqKl+c0FlYZI+F7Ly ++MdCcNgKFc8o8jGiyP6uyAJeg+tSICpFDw00LyuKmU62c7VKuyo7 +-----END CERTIFICATE-----""" + def setUp(self): self.client = InfluxDBClient3( host="localhost", @@ -164,3 +187,84 @@ def test_query_proxy_base_client(self): assert client._query_api._proxy == test_proxy assert ('grpc.http_proxy', test_proxy) in\ client._query_api._flight_client_options.get('generic_options') + + def create_cert_file(self, file_name): + f = open(file_name, "w") + f.write(self.sample_cert) + f.close() + + def remove_cert_file(self, file_name): + os.remove(file_name) + + def test_query_api_options_builder(self): + proxy_name = "http://my.proxy.org" + cert_file = "cert_test.pem" + self.create_cert_file(cert_file) + builder = QueryApiOptionsBuilder() + options = builder.proxy(proxy_name)\ + .root_certs(cert_file)\ + .tls_verify(False)\ + .build() + + print(f"\nDEBUG options {vars(options)}") + try: + assert(options.tls_root_certs.decode('utf-8') == self.sample_cert) + assert(options.tls_verify == False) + assert(options.proxy == proxy_name) + finally: + self.remove_cert_file(cert_file) + + def test_query_client_with_options(self): + connection = "grpc+tls://localhost:9999" + token = "my_token" + proxy_name = "http://my.proxy.org" + cert_file = "cert_test.pem" + self.create_cert_file(cert_file) + options = QueryApiOptionsBuilder()\ + .proxy(proxy_name) \ + .root_certs(cert_file) \ + .tls_verify(False) \ + .build() + + client = QueryApi(connection, + token, + None, + None, + options + ) + + print(f"\nDEBUG client {vars(client)}") + try: + assert(client._token == token) + assert(client._flight_client_options['tls_root_certs'].decode('utf-8') == self.sample_cert) + assert(client._proxy == proxy_name) + # print(f"DEBUG client._flight_client_options['generic_options'] {dict(client._flight_client_options['generic_options'])['grpc.secondary_user_agent']}") + assert(dict(client._flight_client_options['generic_options'])['grpc.secondary_user_agent'].startswith('influxdb3-python/')) + assert(dict(client._flight_client_options['generic_options'])['grpc.http_proxy'] == proxy_name) + finally: + self.remove_cert_file(cert_file) + + def test_client_with_ssl_args(self): + cert_name = "cert-test.pem" + self.create_cert_file(cert_name) + proxy = "http://localhost:9999" + local_client = InfluxDBClient3( + host="localhost", + org="my_org", + database="my_db", + token="my_token", + proxy=proxy, + ssl_ca_cert = cert_name, + verify_ssl = False + ) + + try: + qapi = local_client._query_api + fc_opts = qapi._flight_client_options + assert(qapi._proxy == proxy) + assert(fc_opts['tls_root_certs'].decode('utf-8') == self.sample_cert) + assert(fc_opts['disable_server_verification'] == True) + assert(dict(fc_opts['generic_options'])['grpc.secondary_user_agent'].startswith('influxdb3-python/')) + assert(dict(fc_opts['generic_options'])['grpc.http_proxy'] == proxy) + finally: + self.remove_cert_file(cert_name) From fde3793957540175863cb03e0f3a2121ef512ed4 Mon Sep 17 00:00:00 2001 From: karel rehor Date: Wed, 26 Feb 2025 16:53:05 +0100 Subject: [PATCH 02/12] chore: fix flake issue --- influxdb_client_3/query/query_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/influxdb_client_3/query/query_api.py b/influxdb_client_3/query/query_api.py index c91896d..d3102c4 100644 --- a/influxdb_client_3/query/query_api.py +++ b/influxdb_client_3/query/query_api.py @@ -6,6 +6,7 @@ from pyarrow.flight import FlightClient, Ticket, FlightCallOptions, FlightStreamReader from influxdb_client_3.version import USER_AGENT + class QueryApiOptions(object): tls_root_certs = None tls_verify = True From 2eb06faf45d9057c845529770e7847451d3348d8 Mon Sep 17 00:00:00 2001 From: karel rehor Date: Wed, 26 Feb 2025 17:02:07 +0100 Subject: [PATCH 03/12] chore: clean up flake8 issues in tests --- tests/test_query.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/test_query.py b/tests/test_query.py index e1e1c48..d9c8caf 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -208,9 +208,9 @@ def test_query_api_options_builder(self): print(f"\nDEBUG options {vars(options)}") try: - assert(options.tls_root_certs.decode('utf-8') == self.sample_cert) - assert(options.tls_verify == False) - assert(options.proxy == proxy_name) + assert options.tls_root_certs.decode('utf-8') == self.sample_cert + assert not options.tls_verify + assert options.proxy == proxy_name finally: self.remove_cert_file(cert_file) @@ -235,12 +235,12 @@ def test_query_client_with_options(self): print(f"\nDEBUG client {vars(client)}") try: - assert(client._token == token) - assert(client._flight_client_options['tls_root_certs'].decode('utf-8') == self.sample_cert) - assert(client._proxy == proxy_name) - # print(f"DEBUG client._flight_client_options['generic_options'] {dict(client._flight_client_options['generic_options'])['grpc.secondary_user_agent']}") - assert(dict(client._flight_client_options['generic_options'])['grpc.secondary_user_agent'].startswith('influxdb3-python/')) - assert(dict(client._flight_client_options['generic_options'])['grpc.http_proxy'] == proxy_name) + assert client._token == token + assert client._flight_client_options['tls_root_certs'].decode('utf-8') == self.sample_cert + assert client._proxy == proxy_name + fc_opts = client._flight_client_options + assert dict(fc_opts['generic_options'])['grpc.secondary_user_agent'].startswith('influxdb3-python/') + assert dict(fc_opts['generic_options'])['grpc.http_proxy'] == proxy_name finally: self.remove_cert_file(cert_file) @@ -254,17 +254,17 @@ def test_client_with_ssl_args(self): database="my_db", token="my_token", proxy=proxy, - ssl_ca_cert = cert_name, - verify_ssl = False + ssl_ca_cert=cert_name, + verify_ssl=False ) try: qapi = local_client._query_api fc_opts = qapi._flight_client_options - assert(qapi._proxy == proxy) - assert(fc_opts['tls_root_certs'].decode('utf-8') == self.sample_cert) - assert(fc_opts['disable_server_verification'] == True) - assert(dict(fc_opts['generic_options'])['grpc.secondary_user_agent'].startswith('influxdb3-python/')) - assert(dict(fc_opts['generic_options'])['grpc.http_proxy'] == proxy) + assert qapi._proxy == proxy + assert fc_opts['tls_root_certs'].decode('utf-8') == self.sample_cert + assert fc_opts['disable_server_verification'] + assert dict(fc_opts['generic_options'])['grpc.secondary_user_agent'].startswith('influxdb3-python/') + assert dict(fc_opts['generic_options'])['grpc.http_proxy'] == proxy finally: self.remove_cert_file(cert_name) From 3e4206518857173ff3f03db254242a9024191a88 Mon Sep 17 00:00:00 2001 From: karel rehor Date: Thu, 27 Feb 2025 13:25:58 +0100 Subject: [PATCH 04/12] test: adds integration test for ssl arguments --- tests/test_influxdb_client_3_integration.py | 73 +++++++++++++++++++-- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/tests/test_influxdb_client_3_integration.py b/tests/test_influxdb_client_3_integration.py index 1fb5747..3a51cc6 100644 --- a/tests/test_influxdb_client_3_integration.py +++ b/tests/test_influxdb_client_3_integration.py @@ -7,6 +7,7 @@ import pyarrow import pytest +from pyarrow._flight import FlightError from influxdb_client_3 import InfluxDBClient3, InfluxDBError, write_client_options, WriteOptions @@ -122,7 +123,6 @@ def error(conf, data, exception: InfluxDBError): database=self.database, token=self.token, write_client_options=wc_opts) as w_client: - for i in range(0, data_set_size): w_client.write(f'{measurement},location=harfa val={i}i {now - (i * 1_000_000_000)}') @@ -134,7 +134,6 @@ def error(conf, data, exception: InfluxDBError): database=self.database, token=self.token, write_client_options=wc_opts) as r_client: - query = f"SELECT * FROM \"{measurement}\" WHERE time >= now() - interval '3 minute'" reader: pyarrow.Table = r_client.query(query) list_results = reader.to_pylist() @@ -165,7 +164,6 @@ def test_batch_write_closed(self): token=self.token, write_client_options=wc_opts, debug=True) as w_client: - for i in range(0, data_size): w_client.write(f'{measurement},location=harfa val={i}i {now - (i * 1_000_000_000)}') @@ -177,10 +175,77 @@ def test_batch_write_closed(self): database=self.database, token=self.token, write_client_options=wc_opts) as r_client: - logging.info("PREPARING QUERY") query = f"SELECT * FROM \"{measurement}\" WHERE time >= now() - interval '3 hours'" reader: pyarrow.Table = r_client.query(query, mode="") list_results = reader.to_pylist() self.assertEqual(data_size, len(list_results)) + + test_cert = """-----BEGIN CERTIFICATE----- +MIIDUzCCAjugAwIBAgIUZB55ULutbc9gy6xLp1BkTQU7siowDQYJKoZIhvcNAQEL +BQAwNjE0MDIGA1UEAwwraW5mbHV4ZGIzLWNsdXN0ZXJlZC1zd2FuLmJyYW1ib3Jh +LnpvbmEtYi5ldTAeFw0yNTAyMTgxNTIyMTJaFw0yNjAyMTgxNTIyMTJaMDYxNDAy +BgNVBAMMK2luZmx1eGRiMy1jbHVzdGVyZWQtc3dhbi5icmFtYm9yYS56b25hLWIu +ZXUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCugeNrx0ZfyyP8H4e0 +zDSkKWnEXlVdjMi+ZSHhMbjvvqMkUQGLc/W59AEmMJ0Uiljka9d+F7jdu+oqDq9p +4kGPhO3Oh7zIG0IGbncj8AwIXMGDNkNyL8s7C1+LoYotlSWDpWwkEKXUeAzdqS63 +CSJFqSJM2dss8qe9BpM6zHWJAKS1I30QT3SXQFEsF5m2F62dXCEEI6pO7jlik8/w +aI47dTM20QyimVzea48SC/ELO/T4AjbmMeBGlTyCm39KOElOKRTJvB4KESEWaL3r +EvPZbTh+72PUyrjxiDa56+RmtDPo7EN3uxuRVFX/HWiNnFk7orQLKZg5Kr8wE46R +KmVvAgMBAAGjWTBXMDYGA1UdEQQvMC2CK2luZmx1eGRiMy1jbHVzdGVyZWQtc3dh +bi5icmFtYm9yYS56b25hLWIuZXUwHQYDVR0OBBYEFH8et6JCzGD7Ny84aNRtq5Nj +hvS/MA0GCSqGSIb3DQEBCwUAA4IBAQCuDwARea/Xr3+hmte9A0H+XB8wMPAJ64e8 +QA0qi0oy0gGdLfQHhsBWWmKSYLv7HygTNzb+7uFOTtq1UPLt18F+POPeLIj74QZV +z89Pbo1TwUMzQ2pgbu0yRvraXIpqXGrPm5GWYp5mopX0rBWKdimbmEMkhZA0sVeH +IdKIRUY6EyIVG+Z/nbuVqUlgnIWOMp0yg4RRC91zHy3Xvykf3Vai25H/jQpa6cbU +//MIodzUIqT8Tja5cHXE51bLdUkO1rtNKdM7TUdjzkZ+bAOpqKl+c0FlYZI+F7Ly ++MdCcNgKFc8o8jGiyP6uyAJeg+tSICpFDw00LyuKmU62c7VKuyo7 +-----END CERTIFICATE-----""" + + def create_test_cert(self, cert_file): + f = open(cert_file, "w") + f.write(self.test_cert) + f.close() + + def remove_test_cert(self, cert_file): + os.remove(cert_file) + + def test_queries_w_bad_cert(self): + cert_file = "test_cert.pem" + self.create_test_cert(cert_file) + with InfluxDBClient3(host=self.host, + database=self.database, + token=self.token, + verify_ssl=True, + ssl_ca_cert=cert_file, + debug=True) as client: + try: + query = "SELECT table_name FROM information_schema.tables" + client.query(query, mode="") + assert False, "query should throw SSL_ERROR" + except FlightError as fe: + assert str(fe).__contains__('SSL_ERROR_SSL') + finally: + self.remove_test_cert(cert_file) + + def test_verify_ssl_false(self): + cert_file = "test_cert.pem" + self.create_test_cert(cert_file) + measurement = f'test{random_hex(6)}' + + with InfluxDBClient3(host=self.host, + database=self.database, + token=self.token, + verify_ssl=False, + ssl_ca_cert=cert_file, + debug=True) as client: + try: + now = time.time_ns() + client.write(f'{measurement},location=harfa val=42i {now - 1_000_000_000}') + query = f"SELECT * FROM \"{measurement}\"" + reader: pyarrow.Table = client.query(query, mode="") + list_results = reader.to_pylist() + assert len(list_results) > 0 + finally: + self.remove_test_cert(cert_file) From f59cfb9708a83ded16719dd435979dfd0bdc77e6 Mon Sep 17 00:00:00 2001 From: karel rehor Date: Fri, 28 Feb 2025 14:58:16 +0100 Subject: [PATCH 05/12] chore: replace cert in tests with anonymized --- tests/test_query.py | 46 +++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/tests/test_query.py b/tests/test_query.py index d9c8caf..253f34d 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -118,24 +118,34 @@ def test_influx_default_query_headers(): class TestQuery(unittest.TestCase): sample_cert = """-----BEGIN CERTIFICATE----- -MIIDUzCCAjugAwIBAgIUZB55ULutbc9gy6xLp1BkTQU7siowDQYJKoZIhvcNAQEL -BQAwNjE0MDIGA1UEAwwraW5mbHV4ZGIzLWNsdXN0ZXJlZC1zd2FuLmJyYW1ib3Jh -LnpvbmEtYi5ldTAeFw0yNTAyMTgxNTIyMTJaFw0yNjAyMTgxNTIyMTJaMDYxNDAy -BgNVBAMMK2luZmx1eGRiMy1jbHVzdGVyZWQtc3dhbi5icmFtYm9yYS56b25hLWIu -ZXUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCugeNrx0ZfyyP8H4e0 -zDSkKWnEXlVdjMi+ZSHhMbjvvqMkUQGLc/W59AEmMJ0Uiljka9d+F7jdu+oqDq9p -4kGPhO3Oh7zIG0IGbncj8AwIXMGDNkNyL8s7C1+LoYotlSWDpWwkEKXUeAzdqS63 -CSJFqSJM2dss8qe9BpM6zHWJAKS1I30QT3SXQFEsF5m2F62dXCEEI6pO7jlik8/w -aI47dTM20QyimVzea48SC/ELO/T4AjbmMeBGlTyCm39KOElOKRTJvB4KESEWaL3r -EvPZbTh+72PUyrjxiDa56+RmtDPo7EN3uxuRVFX/HWiNnFk7orQLKZg5Kr8wE46R -KmVvAgMBAAGjWTBXMDYGA1UdEQQvMC2CK2luZmx1eGRiMy1jbHVzdGVyZWQtc3dh -bi5icmFtYm9yYS56b25hLWIuZXUwHQYDVR0OBBYEFH8et6JCzGD7Ny84aNRtq5Nj -hvS/MA0GCSqGSIb3DQEBCwUAA4IBAQCuDwARea/Xr3+hmte9A0H+XB8wMPAJ64e8 -QA0qi0oy0gGdLfQHhsBWWmKSYLv7HygTNzb+7uFOTtq1UPLt18F+POPeLIj74QZV -z89Pbo1TwUMzQ2pgbu0yRvraXIpqXGrPm5GWYp5mopX0rBWKdimbmEMkhZA0sVeH -IdKIRUY6EyIVG+Z/nbuVqUlgnIWOMp0yg4RRC91zHy3Xvykf3Vai25H/jQpa6cbU -//MIodzUIqT8Tja5cHXE51bLdUkO1rtNKdM7TUdjzkZ+bAOpqKl+c0FlYZI+F7Ly -+MdCcNgKFc8o8jGiyP6uyAJeg+tSICpFDw00LyuKmU62c7VKuyo7 +MIIFDTCCAvWgAwIBAgIUYzpfisy9xLrhiZd+D9vOdzC3+iswDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAwwLdGVzdGhvc3QuaW8wHhcNMjUwMjI4MTM1NTMyWhcNMzUw +MjI2MTM1NTMyWjAWMRQwEgYDVQQDDAt0ZXN0aG9zdC5pbzCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBAN1lwqXYP8UMvjb56SpUEj2OpoEDRfLeWrEiHkOl +xoymvJGaXZNEpDXo2TTdysCoYWEjz9IY6GlqSo2Yssf5BZkQwMOw7MdyRwCigzrh +OAKbyCfsvEgfNFrXEdSDpaxW++5SToeErudYXc+sBfnI1NB4W3GBGqqIvx8fqaB3 +1EU9ql2sKKxI0oYIQD/If9rQEyLFKeWdD8iT6YST1Vugkvd34NPmaqV5+pjdSb4z +a8olavwUoslqFUeILqIq+WZZbOlgCcJYKcBAmELRnsxGaABRtMwMZx+0D+oKo4Kl +QQtOcER+RHkBHyYFghZIBnzudfbP9NadknOz3AilJbJolXfXJqeQhRD8Ob49kkhe +OwjAppHnaZGWjYZMLIfnwwXBwkS7bSwF16Wot83cpL46Xvg6xcl12An4JaoF798Q +cXyYrWCgvbqjVR7694gxqLGzk138AKTDSbER1h1rfqCqkk7soE0oWCs7jiCk2XvD +49qVfHtd50KYJ4/yP1XL0PmLL0Hw1kvOxLVkFENc1zkoYXJRt2Ec6j9dajmGlsFn +0bLLap6UIlIGQFuvcLf4bvsIi9FICy2jBjaIdM4UAWbReG+52+180HEleAwi5bAN +HY61WVXc4X+N0E2y8HWc1QaRioU7R4XZ5HXKs7OTWkKFZUU2JDFHAKdiiAU78qLU +7GApAgMBAAGjUzBRMB0GA1UdDgQWBBT2vPFo0mzh9ls4xJUiAgSK+B5LpTAfBgNV +HSMEGDAWgBT2vPFo0mzh9ls4xJUiAgSK+B5LpTAPBgNVHRMBAf8EBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4ICAQC4TJNPx476qhiMi8anISv9lo9cnLju+qNhcz7wupBH +3Go6bVQ7TCbSt2QpAyY64mdnRqHsXeGvZXCnabOpeKRDeAPBtRjc6yNKuXybqFtn +W3PZEs/OYc659TUA+MoBzSXYStN9yiiYXyVFqVn+Rw6kM9tKh0GgAU7f5P+8IGuR +gXJbCjkbdJO7JUiVGEEmkjUHyqFxMHaZ8V6uazs52qIFyt7OYQTeV9HdoW8D9vAt +GfzYwzRDzbsZeIJqqDzLe7NOyxEyqZHCbtNpGcOyaLOl7ZBS52WsqaUZtL+9PjqD +2TWj4WUFkOWQpTvWKHqM6//Buv4GjnTBShQKm+h+rxcGkdRMF6/sKwxPbr39P3RJ +TMfJA3u5UuowT44VaA2jkQzqIbxH9+3EA+0qPbqPJchOSr0pHSncqvR9FYcr7ayN +b6UDFnjeliyEqqksUO0arbvaO9FfB0kH8lU1NOKaQNO++Xj69GZMC6s721cNdad0 +qqcdtyXWeOBBchguYDrSUIgLnUTHEwwzOmcNQ36hO5eX282BJy3ZLT3JU6MJopjz +vkbDDAxSrpZMcaoAWSrxgJAETeYiO4YbfORIzPkwdUkEIr6XY02Pi7MdkDGQ5hiB +TavA8+oXRa4b9BR3bCWcg8S/t4uOTTLkeTcQbONPh5A5IRySLCU+CwqB+/+VlO8X +Aw== -----END CERTIFICATE-----""" def setUp(self): From 5b343ddf616701027b24fc818a6f59cedc5a2d0d Mon Sep 17 00:00:00 2001 From: karel rehor Date: Fri, 28 Feb 2025 17:18:23 +0100 Subject: [PATCH 06/12] chore: (WIP) adds basic_ssl_example.py --- Examples/basic_ssl_example.py | 101 ++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 Examples/basic_ssl_example.py diff --git a/Examples/basic_ssl_example.py b/Examples/basic_ssl_example.py new file mode 100644 index 0000000..fe0d716 --- /dev/null +++ b/Examples/basic_ssl_example.py @@ -0,0 +1,101 @@ +import os +import ssl +import time + +import pyarrow + +from config import Config +from influxdb_client_3 import InfluxDBClient3 + +bad_cert = """-----BEGIN CERTIFICATE----- +MIIFDTCCAvWgAwIBAgIUYzpfisy9xLrhiZd+D9vOdzC3+iswDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAwwLdGVzdGhvc3QuaW8wHhcNMjUwMjI4MTM1NTMyWhcNMzUw +MjI2MTM1NTMyWjAWMRQwEgYDVQQDDAt0ZXN0aG9zdC5pbzCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBAN1lwqXYP8UMvjb56SpUEj2OpoEDRfLeWrEiHkOl +xoymvJGaXZNEpDXo2TTdysCoYWEjz9IY6GlqSo2Yssf5BZkQwMOw7MdyRwCigzrh +OAKbyCfsvEgfNFrXEdSDpaxW++5SToeErudYXc+sBfnI1NB4W3GBGqqIvx8fqaB3 +1EU9ql2sKKxI0oYIQD/If9rQEyLFKeWdD8iT6YST1Vugkvd34NPmaqV5+pjdSb4z +a8olavwUoslqFUeILqIq+WZZbOlgCcJYKcBAmELRnsxGaABRtMwMZx+0D+oKo4Kl +QQtOcER+RHkBHyYFghZIBnzudfbP9NadknOz3AilJbJolXfXJqeQhRD8Ob49kkhe +OwjAppHnaZGWjYZMLIfnwwXBwkS7bSwF16Wot83cpL46Xvg6xcl12An4JaoF798Q +cXyYrWCgvbqjVR7694gxqLGzk138AKTDSbER1h1rfqCqkk7soE0oWCs7jiCk2XvD +49qVfHtd50KYJ4/yP1XL0PmLL0Hw1kvOxLVkFENc1zkoYXJRt2Ec6j9dajmGlsFn +0bLLap6UIlIGQFuvcLf4bvsIi9FICy2jBjaIdM4UAWbReG+52+180HEleAwi5bAN +HY61WVXc4X+N0E2y8HWc1QaRioU7R4XZ5HXKs7OTWkKFZUU2JDFHAKdiiAU78qLU +7GApAgMBAAGjUzBRMB0GA1UdDgQWBBT2vPFo0mzh9ls4xJUiAgSK+B5LpTAfBgNV +HSMEGDAWgBT2vPFo0mzh9ls4xJUiAgSK+B5LpTAPBgNVHRMBAf8EBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4ICAQC4TJNPx476qhiMi8anISv9lo9cnLju+qNhcz7wupBH +3Go6bVQ7TCbSt2QpAyY64mdnRqHsXeGvZXCnabOpeKRDeAPBtRjc6yNKuXybqFtn +W3PZEs/OYc659TUA+MoBzSXYStN9yiiYXyVFqVn+Rw6kM9tKh0GgAU7f5P+8IGuR +gXJbCjkbdJO7JUiVGEEmkjUHyqFxMHaZ8V6uazs52qIFyt7OYQTeV9HdoW8D9vAt +GfzYwzRDzbsZeIJqqDzLe7NOyxEyqZHCbtNpGcOyaLOl7ZBS52WsqaUZtL+9PjqD +2TWj4WUFkOWQpTvWKHqM6//Buv4GjnTBShQKm+h+rxcGkdRMF6/sKwxPbr39P3RJ +TMfJA3u5UuowT44VaA2jkQzqIbxH9+3EA+0qPbqPJchOSr0pHSncqvR9FYcr7ayN +b6UDFnjeliyEqqksUO0arbvaO9FfB0kH8lU1NOKaQNO++Xj69GZMC6s721cNdad0 +qqcdtyXWeOBBchguYDrSUIgLnUTHEwwzOmcNQ36hO5eX282BJy3ZLT3JU6MJopjz +vkbDDAxSrpZMcaoAWSrxgJAETeYiO4YbfORIzPkwdUkEIr6XY02Pi7MdkDGQ5hiB +TavA8+oXRa4b9BR3bCWcg8S/t4uOTTLkeTcQbONPh5A5IRySLCU+CwqB+/+VlO8X +Aw== +-----END CERTIFICATE-----""" + + +def write_cert(cert, file_name): + f = open(file_name, "w") + f.write(cert) + f.close() + + +def remove_cert(file_name): + os.remove(file_name) + + +def main() -> None: + print("Main") + temp_cert_file = "temp_cert.pem" + conf = Config() + + write_and_query_with_explicit_sys_cert(conf) + + write_cert(bad_cert, temp_cert_file) + query_with_verify_ssl_off(conf, temp_cert_file) + remove_cert(temp_cert_file) + + +def write_and_query_with_explicit_sys_cert(conf): + print("write and query with typical linux system cert") + with InfluxDBClient3(token=conf.token, + host=conf.host, + org=conf.org, + database=conf.database, + ssl_ca_cert="/etc/ssl/certs/ca-certificates.crt", + verify_ssl=True) as _client: + now = time.time_ns() + lp = f"escooter,id=zx80 speed=3.14,ticks=42i {now - (10 * 1_000_000_000)}" + _client.write(lp) + + query = "SELECT * FROM \"escooter\"" + reader: pyarrow.Table = _client.query(query, mode="") + list_results = reader.to_pylist() + print(list_results) + + +def query_with_verify_ssl_off(conf, cert): + print("querying with verify_ssl off") + + # Note that the passed root cert above is bad + # Switch verify_ssl to True to throw SSL_ERROR_SSL + with InfluxDBClient3(token=conf.token, + host=conf.host, + org=conf.org, + database=conf.database, + ssl_ca_cert=cert, + verify_ssl=False) as _client: + + query = "SELECT * FROM \"escooter\"" + reader: pyarrow.Table = _client.query(query, mode="") + list_results = reader.to_pylist() + print(list_results) + + +if __name__ == "__main__": + main() From 3a604f57ce9b4986767e2262e39edc833e40d400 Mon Sep 17 00:00:00 2001 From: karel rehor Date: Fri, 28 Feb 2025 17:19:19 +0100 Subject: [PATCH 07/12] chore: remove unused import --- Examples/basic_ssl_example.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Examples/basic_ssl_example.py b/Examples/basic_ssl_example.py index fe0d716..b678351 100644 --- a/Examples/basic_ssl_example.py +++ b/Examples/basic_ssl_example.py @@ -1,5 +1,4 @@ import os -import ssl import time import pyarrow From d6a4d3c6a038c308f33ba186b33556c793610ba5 Mon Sep 17 00:00:00 2001 From: karel rehor Date: Mon, 3 Mar 2025 10:35:35 +0100 Subject: [PATCH 08/12] chore: remove debug strings and update example output --- Examples/basic_ssl_example.py | 15 ++++++++++----- influxdb_client_3/__init__.py | 1 - influxdb_client_3/query/query_api.py | 1 - 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Examples/basic_ssl_example.py b/Examples/basic_ssl_example.py index b678351..8657e02 100644 --- a/Examples/basic_ssl_example.py +++ b/Examples/basic_ssl_example.py @@ -48,6 +48,12 @@ def remove_cert(file_name): os.remove(file_name) +def print_results(results: list): + print("%-6s%-6s%-6s%-24s" % ("id", "speed", "ticks", "time")) + for result in results: + print("%-6s%-6.2f%-6i%-24s" % (result['id'], result['speed'], result['ticks'], result['time'])) + + def main() -> None: print("Main") temp_cert_file = "temp_cert.pem" @@ -61,7 +67,7 @@ def main() -> None: def write_and_query_with_explicit_sys_cert(conf): - print("write and query with typical linux system cert") + print("\nwrite and query with typical linux system cert\n") with InfluxDBClient3(token=conf.token, host=conf.host, org=conf.org, @@ -75,11 +81,11 @@ def write_and_query_with_explicit_sys_cert(conf): query = "SELECT * FROM \"escooter\"" reader: pyarrow.Table = _client.query(query, mode="") list_results = reader.to_pylist() - print(list_results) + print_results(reader.to_pylist()) def query_with_verify_ssl_off(conf, cert): - print("querying with verify_ssl off") + print("\nquerying with verify_ssl off\n") # Note that the passed root cert above is bad # Switch verify_ssl to True to throw SSL_ERROR_SSL @@ -92,8 +98,7 @@ def query_with_verify_ssl_off(conf, cert): query = "SELECT * FROM \"escooter\"" reader: pyarrow.Table = _client.query(query, mode="") - list_results = reader.to_pylist() - print(list_results) + print_results(reader.to_pylist()) if __name__ == "__main__": diff --git a/influxdb_client_3/__init__.py b/influxdb_client_3/__init__.py index 07e2e95..c257208 100644 --- a/influxdb_client_3/__init__.py +++ b/influxdb_client_3/__init__.py @@ -166,7 +166,6 @@ def __init__( else: connection_string = f"grpc+tcp://{hostname}:{port}" - print(f"\nDEBUG kwargs.keys {kwargs.keys()}") q_opts_builder = QueryApiOptionsBuilder() if kwargs.keys().__contains__('ssl_ca_cert'): q_opts_builder.root_certs(kwargs.get('ssl_ca_cert', None)) diff --git a/influxdb_client_3/query/query_api.py b/influxdb_client_3/query/query_api.py index d3102c4..c9ca4dc 100644 --- a/influxdb_client_3/query/query_api.py +++ b/influxdb_client_3/query/query_api.py @@ -87,7 +87,6 @@ def __init__(self, self._token = token self._flight_client_options = flight_client_options or {} self._proxy = proxy - print(f"\nDEBUG options {options}") if options: if options.flight_client_options: self._flight_client_options = options.flight_client_options From b0298e88bb65b041c741c6dd837bb83de70d79b0 Mon Sep 17 00:00:00 2001 From: karel rehor Date: Mon, 3 Mar 2025 10:37:29 +0100 Subject: [PATCH 09/12] chore: linted new ssl example --- Examples/basic_ssl_example.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Examples/basic_ssl_example.py b/Examples/basic_ssl_example.py index 8657e02..c71c96c 100644 --- a/Examples/basic_ssl_example.py +++ b/Examples/basic_ssl_example.py @@ -80,7 +80,6 @@ def write_and_query_with_explicit_sys_cert(conf): query = "SELECT * FROM \"escooter\"" reader: pyarrow.Table = _client.query(query, mode="") - list_results = reader.to_pylist() print_results(reader.to_pylist()) From 6c631c072a9122e0eac6800725a8945f90dee6ac Mon Sep 17 00:00:00 2001 From: karel rehor Date: Mon, 3 Mar 2025 11:02:25 +0100 Subject: [PATCH 10/12] chore: call kwargs.keys() only once in init; initialize QueryApiOptions.tls_verify to None --- Examples/basic_ssl_example.py | 4 ++-- influxdb_client_3/__init__.py | 7 ++++--- influxdb_client_3/query/query_api.py | 5 +++-- tests/test_query.py | 2 -- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Examples/basic_ssl_example.py b/Examples/basic_ssl_example.py index c71c96c..98096c9 100644 --- a/Examples/basic_ssl_example.py +++ b/Examples/basic_ssl_example.py @@ -78,7 +78,7 @@ def write_and_query_with_explicit_sys_cert(conf): lp = f"escooter,id=zx80 speed=3.14,ticks=42i {now - (10 * 1_000_000_000)}" _client.write(lp) - query = "SELECT * FROM \"escooter\"" + query = "SELECT * FROM \"escooter\" ORDER BY time DESC" reader: pyarrow.Table = _client.query(query, mode="") print_results(reader.to_pylist()) @@ -95,7 +95,7 @@ def query_with_verify_ssl_off(conf, cert): ssl_ca_cert=cert, verify_ssl=False) as _client: - query = "SELECT * FROM \"escooter\"" + query = "SELECT * FROM \"escooter\" ORDER BY time DESC" reader: pyarrow.Table = _client.query(query, mode="") print_results(reader.to_pylist()) diff --git a/influxdb_client_3/__init__.py b/influxdb_client_3/__init__.py index c257208..6f736d2 100644 --- a/influxdb_client_3/__init__.py +++ b/influxdb_client_3/__init__.py @@ -167,11 +167,12 @@ def __init__( connection_string = f"grpc+tcp://{hostname}:{port}" q_opts_builder = QueryApiOptionsBuilder() - if kwargs.keys().__contains__('ssl_ca_cert'): + kw_keys = kwargs.keys() + if kw_keys.__contains__('ssl_ca_cert'): q_opts_builder.root_certs(kwargs.get('ssl_ca_cert', None)) - if kwargs.keys().__contains__('verify_ssl'): + if kw_keys.__contains__('verify_ssl'): q_opts_builder.tls_verify(kwargs.get('verify_ssl', True)) - if kwargs.keys().__contains__('proxy'): + if kw_keys.__contains__('proxy'): q_opts_builder.proxy(kwargs.get('proxy', None)) self._query_api = _QueryApi(connection_string=connection_string, token=token, flight_client_options=flight_client_options, diff --git a/influxdb_client_3/query/query_api.py b/influxdb_client_3/query/query_api.py index c9ca4dc..e217346 100644 --- a/influxdb_client_3/query/query_api.py +++ b/influxdb_client_3/query/query_api.py @@ -9,7 +9,7 @@ class QueryApiOptions(object): tls_root_certs = None - tls_verify = True + tls_verify = None proxy = None flight_client_options = None @@ -94,7 +94,8 @@ def __init__(self, self._flight_client_options["tls_root_certs"] = options.tls_root_certs if options.proxy: self._proxy = options.proxy - self._flight_client_options["disable_server_verification"] = not options.tls_verify + if options.tls_verify is not None: + self._flight_client_options["disable_server_verification"] = not options.tls_verify self._flight_client_options["generic_options"] = [ ("grpc.secondary_user_agent", USER_AGENT) ] diff --git a/tests/test_query.py b/tests/test_query.py index 253f34d..72d81df 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -216,7 +216,6 @@ def test_query_api_options_builder(self): .tls_verify(False)\ .build() - print(f"\nDEBUG options {vars(options)}") try: assert options.tls_root_certs.decode('utf-8') == self.sample_cert assert not options.tls_verify @@ -243,7 +242,6 @@ def test_query_client_with_options(self): options ) - print(f"\nDEBUG client {vars(client)}") try: assert client._token == token assert client._flight_client_options['tls_root_certs'].decode('utf-8') == self.sample_cert From 85c7b39c9f9f3954135fa0526b07e5187ffe67c0 Mon Sep 17 00:00:00 2001 From: karel rehor Date: Mon, 3 Mar 2025 14:29:58 +0100 Subject: [PATCH 11/12] chore: add API documentation, option types and assert for flight_client_options --- influxdb_client_3/query/query_api.py | 43 +++++++++++++++++++++++++--- tests/test_query.py | 6 ++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/influxdb_client_3/query/query_api.py b/influxdb_client_3/query/query_api.py index e217346..ee0725f 100644 --- a/influxdb_client_3/query/query_api.py +++ b/influxdb_client_3/query/query_api.py @@ -8,12 +8,31 @@ class QueryApiOptions(object): - tls_root_certs = None - tls_verify = None - proxy = None - flight_client_options = None + """ + Structure for encapsulating options for the QueryApi + + Attributes + ---------- + tls_root_certs (bytes): contents of an SSL root certificate or chain read as bytes + tls_verify (bool): whether to verify SSL certificates or not + proxy (str): URL to a proxy server + flight_client_options (dict): base set of flight client options passed to internal pyarrow.flight.FlightClient + """ + tls_root_certs: bytes = None + tls_verify: bool = None + proxy: str = None + flight_client_options: dict = None def __init__(self, root_certs_path, verify, proxy, flight_client_options): + """ + Initialize a set of QueryApiOptions + + :param root_certs_path: path to a certificate .pem file. + :param verify: whether to verify SSL certificates or not. + :param proxy: URL of a proxy server, if required. + :param flight_client_options: set of flight_client_options + to be passed to internal pyarrow.flight.FlightClient. + """ if root_certs_path: self.tls_root_certs = self._read_certs(root_certs_path) self.tls_verify = verify @@ -26,7 +45,21 @@ def _read_certs(self, path): class QueryApiOptionsBuilder(object): + """ + Helper class to make adding QueryApiOptions more dynamic. + Example: + + .. code-block:: python + + options = QueryApiOptionsBuilder()\ + .proxy("http://internal.tunnel.proxy:8080") \ + .root_certs("/home/fred/.etc/ssl/alt_certs.pem") \ + .tls_verify(True) \ + .build() + + client = QueryApi(connection, token, None, None, options) + """ _root_certs_path = None _tls_verify = True _proxy = None @@ -49,6 +82,7 @@ def flight_client_options(self, flight_client_options): return self def build(self): + """Build a QueryApiOptions object with previously set values""" return QueryApiOptions( root_certs_path=self._root_certs_path, verify=self._tls_verify, @@ -62,6 +96,7 @@ class QueryApi(object): Implementation for '/api/v2/query' endpoint. Example: + .. code-block:: python from influxdb_client import InfluxDBClient diff --git a/tests/test_query.py b/tests/test_query.py index 72d81df..d529ffb 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -228,11 +228,15 @@ def test_query_client_with_options(self): token = "my_token" proxy_name = "http://my.proxy.org" cert_file = "cert_test.pem" + private_key = 'our_key' + cert_chain = 'mTLS_explicit_chain' self.create_cert_file(cert_file) + test_flight_client_options = {'private_key': private_key, 'cert_chain': cert_chain} options = QueryApiOptionsBuilder()\ .proxy(proxy_name) \ .root_certs(cert_file) \ .tls_verify(False) \ + .flight_client_options(test_flight_client_options) \ .build() client = QueryApi(connection, @@ -245,6 +249,8 @@ def test_query_client_with_options(self): try: assert client._token == token assert client._flight_client_options['tls_root_certs'].decode('utf-8') == self.sample_cert + assert client._flight_client_options['private_key'] == private_key + assert client._flight_client_options['cert_chain'] == cert_chain assert client._proxy == proxy_name fc_opts = client._flight_client_options assert dict(fc_opts['generic_options'])['grpc.secondary_user_agent'].startswith('influxdb3-python/') From d6ae2ba4bf337d390729fe79710fc8669feba89e Mon Sep 17 00:00:00 2001 From: karel rehor Date: Mon, 3 Mar 2025 14:54:22 +0100 Subject: [PATCH 12/12] docs: update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ca15cf..ab0df23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## 0.12.0 [unreleased] +### Bug Fixes + +1. [#121](https://github.com/InfluxCommunity/influxdb3-python/pull/121): Fix use of arguments `verify_ssl` and `ssl_ca_cert` in `QueryApi`. + ## 0.11.0 [2025-02-27] ### Bug Fixes