enable hf filesystem

Author: lhoestq

pyiceberg/io/__init__.py

@@ -92,6 +92,8 @@
 GCS_SERVICE_HOST = "gcs.service.host"
 GCS_DEFAULT_LOCATION = "gcs.default-bucket-location"
 GCS_VERSION_AWARE = "gcs.version-aware"
+HF_ENDPOINT = "hf.endpoint"
+HF_TOKEN = "hf.token"
 PYARROW_USE_LARGE_TYPES_ON_READ = "pyarrow.use-large-types-on-read"
 
 
@@ -306,6 +308,7 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None:
     "viewfs": [ARROW_FILE_IO],
     "abfs": [FSSPEC_FILE_IO],
     "abfss": [FSSPEC_FILE_IO],
+    "hf": [FSSPEC_FILE_IO],
 }
 
 

pyiceberg/io/fsspec.py

@@ -59,6 +59,8 @@
     GCS_SESSION_KWARGS,
     GCS_TOKEN,
     GCS_VERSION_AWARE,
+    HF_ENDPOINT,
+    HF_TOKEN,
     S3_ACCESS_KEY_ID,
     S3_CONNECT_TIMEOUT,
     S3_ENDPOINT,
@@ -208,6 +210,15 @@ def _adls(properties: Properties) -> AbstractFileSystem:
     )
 
 
+def _hf(properties: Properties) -> AbstractFileSystem:
+    from huggingface_hub import HfFileSystem
+
+    return HfFileSystem(
+        endpoint=properties.get(HF_ENDPOINT),
+        token=properties.get(HF_TOKEN),
+    )
+
+
 SCHEME_TO_FS = {
     "": _file,
     "file": _file,
@@ -218,6 +229,7 @@ def _adls(properties: Properties) -> AbstractFileSystem:
     "abfss": _adls,
     "gs": _gs,
     "gcs": _gs,
+    "hf": _hf,
 }