Implement Kerberos authentication support for Hive Catalog

Author: yothinix

mkdocs/docs/configuration.md

@@ -227,19 +227,19 @@ catalog:
 catalog:
   default:
     uri: thrift://localhost:9083
-    s3.endpoint: http://localhost:9000
-    s3.access-key-id: admin
-    s3.secret-access-key: password
+    hive:
+      hive2-compatible: true
+      use-kerberos: true
 ```
 
-When using Hive 2.x, make sure to set the compatibility flag:
+<!-- markdown-link-check-disable -->
 
-```yaml
-catalog:
-  default:
-...
-    hive.hive2-compatible: true
-```
+| Key                   | Example             | Description                                      |
+| --------------------- | ------------------- | ------------------------------------------------ |
+| hive.hive2-compatible | true                | Using Hive 2.x compatibility mode                |
+| hive.use-kerberos     | true                | Using authentication via Kerberos                |
+
+<!-- markdown-link-check-enable-->
 
 ## Glue Catalog
 

mkdocs/docs/index.md

@@ -40,22 +40,23 @@ pip install "pyiceberg[s3fs,hive]"
 
 You can mix and match optional dependencies depending on your needs:
 
-| Key          | Description:                                                         |
-| ------------ | -------------------------------------------------------------------- |
-| hive         | Support for the Hive metastore                                       |
-| glue         | Support for AWS Glue                                                 |
-| dynamodb     | Support for AWS DynamoDB                                             |
-| sql-postgres | Support for SQL Catalog backed by Postgresql                         |
-| sql-sqlite   | Support for SQL Catalog backed by SQLite                             |
-| pyarrow      | PyArrow as a FileIO implementation to interact with the object store |
-| pandas       | Installs both PyArrow and Pandas                                     |
-| duckdb       | Installs both PyArrow and DuckDB                                     |
-| ray          | Installs PyArrow, Pandas, and Ray                                    |
-| daft         | Installs Daft                                                        |
-| s3fs         | S3FS as a FileIO implementation to interact with the object store    |
-| adlfs        | ADLFS as a FileIO implementation to interact with the object store   |
-| snappy       | Support for snappy Avro compression                                  |
-| gcsfs        | GCSFS as a FileIO implementation to interact with the object store   |
+| Key           | Description:                                                         |
+| ------------- | -------------------------------------------------------------------- |
+| hive          | Support for the Hive metastore                                       |
+| hive-kerberos | Support for Hive metastore in Kerberos environment                   |
+| glue          | Support for AWS Glue                                                 |
+| dynamodb      | Support for AWS DynamoDB                                             |
+| sql-postgres  | Support for SQL Catalog backed by Postgresql                         |
+| sql-sqlite    | Support for SQL Catalog backed by SQLite                             |
+| pyarrow       | PyArrow as a FileIO implementation to interact with the object store |
+| pandas        | Installs both PyArrow and Pandas                                     |
+| duckdb        | Installs both PyArrow and DuckDB                                     |
+| ray           | Installs PyArrow, Pandas, and Ray                                    |
+| daft          | Installs Daft                                                        |
+| s3fs          | S3FS as a FileIO implementation to interact with the object store    |
+| adlfs         | ADLFS as a FileIO implementation to interact with the object store   |
+| snappy        | Support for snappy Avro compression                                  |
+| gcsfs         | GCSFS as a FileIO implementation to interact with the object store   |
 
 You either need to install `s3fs`, `adlfs`, `gcsfs`, or `pyarrow` to be able to fetch files from an object store.
 

poetry.lock

@@ -122,6 +122,9 @@
 HIVE2_COMPATIBLE = "hive.hive2-compatible"
 HIVE2_COMPATIBLE_DEFAULT = False
 
+HIVE_KERBEROS_AUTH = "hive.use-kerberos"
+HIVE_KERBEROS_AUTH_DEFAULT = False
+
 LOCK_CHECK_MIN_WAIT_TIME = "lock-check-min-wait-time"
 LOCK_CHECK_MAX_WAIT_TIME = "lock-check-max-wait-time"
 LOCK_CHECK_RETRIES = "lock-check-retries"
@@ -139,11 +142,26 @@ class _HiveClient:
     _client: Client
     _ugi: Optional[List[str]]
 
-    def __init__(self, uri: str, ugi: Optional[str] = None):
+    def __init__(
+            self,
+            uri: str,
+            ugi: Optional[str] = None,
+            use_kerberos: Optional[bool] = HIVE_KERBEROS_AUTH_DEFAULT
+    ):
         url_parts = urlparse(uri)
+
         transport = TSocket.TSocket(url_parts.hostname, url_parts.port)
-        self._transport = TTransport.TBufferedTransport(transport)
-        protocol = TBinaryProtocol.TBinaryProtocol(transport)
+
+        if not use_kerberos:
+            self._transport = TTransport.TBufferedTransport(transport)
+        else:
+            self._transport = TTransport.TSaslClientTransport(
+                transport,
+                host=url_parts.hostname,
+                service="hive"
+            )
+
+        protocol = TBinaryProtocol.TBinaryProtocol(self._transport)
 
         self._client = Client(protocol)
         self._ugi = ugi.split(':') if ugi else None
@@ -258,7 +276,11 @@ class HiveCatalog(MetastoreCatalog):
 
     def __init__(self, name: str, **properties: str):
         super().__init__(name, **properties)
-        self._client = _HiveClient(properties["uri"], properties.get("ugi"))
+        self._client = _HiveClient(
+            properties["uri"],
+            properties.get("ugi"),
+            properties.get(HIVE_KERBEROS_AUTH, HIVE_KERBEROS_AUTH_DEFAULT)
+        )
 
         self._lock_check_min_wait_time = PropertyUtil.property_as_float(
             properties, LOCK_CHECK_MIN_WAIT_TIME, DEFAULT_LOCK_CHECK_MIN_WAIT_TIME

pyiceberg/catalog/hive.py

@@ -72,6 +72,8 @@ gcsfs = { version = ">=2023.1.0,<2024.1.0", optional = true }
 psycopg2-binary = { version = ">=2.9.6", optional = true }
 sqlalchemy = { version = "^2.0.18", optional = true }
 getdaft = { version = ">=0.2.12", optional = true }
+thrift-sasl = { version = ">=0.4.3", optional = true }
+kerberos = { version = "1.3.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pytest = "7.4.4"
@@ -580,6 +582,7 @@ ray = ["ray", "pyarrow", "pandas"]
 daft = ["getdaft"]
 snappy = ["python-snappy"]
 hive = ["thrift"]
+hive-kerberos = ["thrift", "thrift_sasl", "kerberos"]
 s3fs = ["s3fs"]
 glue = ["boto3", "mypy-boto3-glue"]
 adlfs = ["adlfs"]