Thrift-backed new Python client

This PR completely removes all gRPC functionality from the Python SQL client and replaces the backend with Thrift only. It does not add any new features, just brings in line with the previous functionality of the client.

* New unit tests which tests the behaviour of the backend on a message-by-message basis
* Manual test with test image

  - Did you add usage logs or metrics? Please mention them here.
  - Create dashboards or monitoring notebooks? Please link them here.
  - See http://go/obs/user for docs on our observability tools.

Author: NiallEgan

cmdexec/clients/python/setup.py

@@ -5,11 +5,6 @@
     version="0.0.0",
     package_dir={"": "src"},
     packages=setuptools.find_packages(where="src"),
-    install_requires=[
-        "grpcio",  # TODO: Minimum versions
-        "pyarrow",
-        "protobuf",
-        "cryptography",
-    ],
+    install_requires=["pyarrow", 'thrift>=0.10.0'],
     author="Databricks",
 )

cmdexec/clients/python/src/databricks/sql/__init__.py

@@ -1,4 +1,4 @@
-from databricks.sql.errors import *
+from databricks.sql.exc import *
 
 
 class _DBAPITypeObject(object):
@@ -9,7 +9,7 @@ def __eq__(self, other):
         return other in self.values
 
     def __repr__(self):
-        return "DBAPITypeObject(%s)" % self.values
+        return "DBAPITypeObject({})".format(self.values)
 
 
 STRING = _DBAPITypeObject('string')
@@ -20,7 +20,7 @@ def __repr__(self):
 DATE = _DBAPITypeObject('date')
 ROWID = _DBAPITypeObject()
 
-__version__ = "1.0.0"
+__version__ = "2.0.0-rc1"
 USER_AGENT_NAME = "PyDatabricksSqlConnector"
 
 

cmdexec/clients/python/src/databricks/sql/client.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python
+#
+# Autogenerated by Thrift Compiler (0.15.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+import sys
+import pprint
+if sys.version_info[0] > 2:
+    from urllib.parse import urlparse
+else:
+    from urlparse import urlparse
+from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient
+from thrift.protocol.TBinaryProtocol import TBinaryProtocol
+
+from TCLIService import TCLIService
+from TCLIService.ttypes import *
+
+if len(sys.argv) <= 1 or sys.argv[1] == '--help':
+    print('')
+    print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]')
+    print('')
+    print('Functions:')
+    print('  TOpenSessionResp OpenSession(TOpenSessionReq req)')
+    print('  TCloseSessionResp CloseSession(TCloseSessionReq req)')
+    print('  TGetInfoResp GetInfo(TGetInfoReq req)')
+    print('  TExecuteStatementResp ExecuteStatement(TExecuteStatementReq req)')
+    print('  TGetTypeInfoResp GetTypeInfo(TGetTypeInfoReq req)')
+    print('  TGetCatalogsResp GetCatalogs(TGetCatalogsReq req)')
+    print('  TGetSchemasResp GetSchemas(TGetSchemasReq req)')
+    print('  TGetTablesResp GetTables(TGetTablesReq req)')
+    print('  TGetTableTypesResp GetTableTypes(TGetTableTypesReq req)')
+    print('  TGetColumnsResp GetColumns(TGetColumnsReq req)')
+    print('  TGetFunctionsResp GetFunctions(TGetFunctionsReq req)')
+    print('  TGetPrimaryKeysResp GetPrimaryKeys(TGetPrimaryKeysReq req)')
+    print('  TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req)')
+    print('  TGetOperationStatusResp GetOperationStatus(TGetOperationStatusReq req)')
+    print('  TCancelOperationResp CancelOperation(TCancelOperationReq req)')
+    print('  TCloseOperationResp CloseOperation(TCloseOperationReq req)')
+    print('  TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq req)')
+    print('  TFetchResultsResp FetchResults(TFetchResultsReq req)')
+    print('  TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req)')
+    print('  TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req)')
+    print('  TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req)')
+    print('')
+    sys.exit(0)
+
+pp = pprint.PrettyPrinter(indent=2)
+host = 'localhost'
+port = 9090
+uri = ''
+framed = False
+ssl = False
+validate = True
+ca_certs = None
+keyfile = None
+certfile = None
+http = False
+argi = 1
+
+if sys.argv[argi] == '-h':
+    parts = sys.argv[argi + 1].split(':')
+    host = parts[0]
+    if len(parts) > 1:
+        port = int(parts[1])
+    argi += 2
+
+if sys.argv[argi] == '-u':
+    url = urlparse(sys.argv[argi + 1])
+    parts = url[1].split(':')
+    host = parts[0]
+    if len(parts) > 1:
+        port = int(parts[1])
+    else:
+        port = 80
+    uri = url[2]
+    if url[4]:
+        uri += '?%s' % url[4]
+    http = True
+    argi += 2
+
+if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed':
+    framed = True
+    argi += 1
+
+if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl':
+    ssl = True
+    argi += 1
+
+if sys.argv[argi] == '-novalidate':
+    validate = False
+    argi += 1
+
+if sys.argv[argi] == '-ca_certs':
+    ca_certs = sys.argv[argi+1]
+    argi += 2
+
+if sys.argv[argi] == '-keyfile':
+    keyfile = sys.argv[argi+1]
+    argi += 2
+
+if sys.argv[argi] == '-certfile':
+    certfile = sys.argv[argi+1]
+    argi += 2
+
+cmd = sys.argv[argi]
+args = sys.argv[argi + 1:]
+
+if http:
+    transport = THttpClient.THttpClient(host, port, uri)
+else:
+    if ssl:
+        socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile)
+    else:
+        socket = TSocket.TSocket(host, port)
+    if framed:
+        transport = TTransport.TFramedTransport(socket)
+    else:
+        transport = TTransport.TBufferedTransport(socket)
+protocol = TBinaryProtocol(transport)
+client = TCLIService.Client(protocol)
+transport.open()
+
+if cmd == 'OpenSession':
+    if len(args) != 1:
+        print('OpenSession requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.OpenSession(eval(args[0]),))
+
+elif cmd == 'CloseSession':
+    if len(args) != 1:
+        print('CloseSession requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.CloseSession(eval(args[0]),))
+
+elif cmd == 'GetInfo':
+    if len(args) != 1:
+        print('GetInfo requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetInfo(eval(args[0]),))
+
+elif cmd == 'ExecuteStatement':
+    if len(args) != 1:
+        print('ExecuteStatement requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.ExecuteStatement(eval(args[0]),))
+
+elif cmd == 'GetTypeInfo':
+    if len(args) != 1:
+        print('GetTypeInfo requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetTypeInfo(eval(args[0]),))
+
+elif cmd == 'GetCatalogs':
+    if len(args) != 1:
+        print('GetCatalogs requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetCatalogs(eval(args[0]),))
+
+elif cmd == 'GetSchemas':
+    if len(args) != 1:
+        print('GetSchemas requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetSchemas(eval(args[0]),))
+
+elif cmd == 'GetTables':
+    if len(args) != 1:
+        print('GetTables requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetTables(eval(args[0]),))
+
+elif cmd == 'GetTableTypes':
+    if len(args) != 1:
+        print('GetTableTypes requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetTableTypes(eval(args[0]),))
+
+elif cmd == 'GetColumns':
+    if len(args) != 1:
+        print('GetColumns requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetColumns(eval(args[0]),))
+
+elif cmd == 'GetFunctions':
+    if len(args) != 1:
+        print('GetFunctions requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetFunctions(eval(args[0]),))
+
+elif cmd == 'GetPrimaryKeys':
+    if len(args) != 1:
+        print('GetPrimaryKeys requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetPrimaryKeys(eval(args[0]),))
+
+elif cmd == 'GetCrossReference':
+    if len(args) != 1:
+        print('GetCrossReference requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetCrossReference(eval(args[0]),))
+
+elif cmd == 'GetOperationStatus':
+    if len(args) != 1:
+        print('GetOperationStatus requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetOperationStatus(eval(args[0]),))
+
+elif cmd == 'CancelOperation':
+    if len(args) != 1:
+        print('CancelOperation requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.CancelOperation(eval(args[0]),))
+
+elif cmd == 'CloseOperation':
+    if len(args) != 1:
+        print('CloseOperation requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.CloseOperation(eval(args[0]),))
+
+elif cmd == 'GetResultSetMetadata':
+    if len(args) != 1:
+        print('GetResultSetMetadata requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetResultSetMetadata(eval(args[0]),))
+
+elif cmd == 'FetchResults':
+    if len(args) != 1:
+        print('FetchResults requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.FetchResults(eval(args[0]),))
+
+elif cmd == 'GetDelegationToken':
+    if len(args) != 1:
+        print('GetDelegationToken requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.GetDelegationToken(eval(args[0]),))
+
+elif cmd == 'CancelDelegationToken':
+    if len(args) != 1:
+        print('CancelDelegationToken requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.CancelDelegationToken(eval(args[0]),))
+
+elif cmd == 'RenewDelegationToken':
+    if len(args) != 1:
+        print('RenewDelegationToken requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.RenewDelegationToken(eval(args[0]),))
+
+else:
+    print('Unrecognized method %s' % cmd)
+    sys.exit(1)
+
+transport.close()