add ns parsing support

Author: sungwy

pyiceberg/transforms.py

@@ -330,6 +330,8 @@ def hash_func(v: Any) -> int:
         elif isinstance(source, (TimestampNanoType, TimestamptzNanoType)):
 
             def hash_func(v: Any) -> int:
+                # In order to bucket TimestampNano the same as Timestamp
+                # convert to micros before hashing.
                 if isinstance(v, py_datetime.datetime):
                     v = datetime.datetime_to_micros(v)
                 else:

pyiceberg/types.py

@@ -33,6 +33,7 @@
 from __future__ import annotations
 
 import re
+from enum import IntEnum
 from functools import cached_property
 from typing import (
     Any,
@@ -62,6 +63,12 @@
 FIXED_PARSER = ParseNumberFromBrackets(FIXED)
 
 
+class TableVersion(IntEnum):
+    ONE = 1
+    TWO = 2
+    THREE = 3
+
+
 def transform_dict_value_to_str(dict: Dict[str, Any]) -> Dict[str, str]:
     """Transform all values in the dictionary to string. Raise an error if any value is None."""
     for key, value in dict.items():
@@ -181,9 +188,9 @@ def is_primitive(self) -> bool:
     def is_struct(self) -> bool:
         return isinstance(self, StructType)
 
-    def minimum_format_version(self) -> int:
+    def minimum_format_version(self) -> TableVersion:
         """Minimum Iceberg format version after which this type is supported."""
-        return 1
+        return TableVersion.ONE
 
 
 class PrimitiveType(Singleton, IcebergRootModel[str], IcebergType):
@@ -726,8 +733,8 @@ class TimestampNanoType(PrimitiveType):
 
     root: Literal["timestamp_ns"] = Field(default="timestamp_ns")
 
-    def minimum_format_version(self) -> int:
-        return 3
+    def minimum_format_version(self) -> TableVersion:
+        return TableVersion.THREE
 
 
 class TimestamptzNanoType(PrimitiveType):
@@ -745,8 +752,8 @@ class TimestamptzNanoType(PrimitiveType):
 
     root: Literal["timestamptz_ns"] = Field(default="timestamptz_ns")
 
-    def minimum_format_version(self) -> int:
-        return 3
+    def minimum_format_version(self) -> TableVersion:
+        return TableVersion.THREE
 
 
 class StringType(PrimitiveType):

pyiceberg/utils/datetime.py

@@ -29,8 +29,10 @@
 EPOCH_DATE = date.fromisoformat("1970-01-01")
 EPOCH_TIMESTAMP = datetime.fromisoformat("1970-01-01T00:00:00.000000")
 ISO_TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,6})?")
+ISO_TIMESTAMP_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?")
 EPOCH_TIMESTAMPTZ = datetime.fromisoformat("1970-01-01T00:00:00.000000+00:00")
 ISO_TIMESTAMPTZ = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,6})?[-+]\d{2}:\d{2}")
+ISO_TIMESTAMPTZ_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?([-+]\d{2}:\d{2})")
 
 
 def micros_to_days(timestamp: int) -> int:
@@ -115,28 +117,30 @@ def datetime_to_nanos(dt: datetime) -> int:
 
 
 def timestamp_to_nanos(timestamp_str: str) -> int:
-    """Convert an ISO-9601 formatted timestamp without zone to microseconds from 1970-01-01T00:00:00.000000.
-
-    Currently only microsecond precision timestamp_str is supported as python datetime does not have
-    nanoseconds support.
-    """
-    if ISO_TIMESTAMP.fullmatch(timestamp_str):
-        return datetime_to_nanos(datetime.fromisoformat(timestamp_str))
-    if ISO_TIMESTAMPTZ.fullmatch(timestamp_str):
+    """Convert an ISO-9601 formatted timestamp without zone to nanoseconds from 1970-01-01T00:00:00.000000000."""
+    if match := ISO_TIMESTAMP_NANO.fullmatch(timestamp_str):
+        # Python datetime does not have native nanoseconds support
+        # Hence we need to extract nanoseconds timestamp manually
+        ns_str = match.group(3) or "0"
+        ms_str = match.group(2) if match.group(2) else ""
+        timestamp_str_without_ns_str = match.group(1) + ms_str
+        return datetime_to_nanos(datetime.fromisoformat(timestamp_str_without_ns_str)) + int(ns_str)
+    if ISO_TIMESTAMPTZ_NANO.fullmatch(timestamp_str):
         # When we can match a timestamp without a zone, we can give a more specific error
         raise ValueError(f"Zone offset provided, but not expected: {timestamp_str}")
     raise ValueError(f"Invalid timestamp without zone: {timestamp_str} (must be ISO-8601)")
 
 
 def timestamptz_to_nanos(timestamptz_str: str) -> int:
-    """Convert an ISO-8601 formatted timestamp with zone to microseconds from 1970-01-01T00:00:00.000000+00:00.
-
-    Currently only microsecond precision timestamp_str is supported as python datetime does not have
-    nanoseconds support.
-    """
-    if ISO_TIMESTAMPTZ.fullmatch(timestamptz_str):
-        return datetime_to_nanos(datetime.fromisoformat(timestamptz_str))
-    if ISO_TIMESTAMP.fullmatch(timestamptz_str):
+    """Convert an ISO-8601 formatted timestamp with zone to nanoseconds from 1970-01-01T00:00:00.000000000+00:00."""
+    if match := ISO_TIMESTAMPTZ_NANO.fullmatch(timestamptz_str):
+        # Python datetime does not have native nanoseconds support
+        # Hence we need to extract nanoseconds timestamp manually
+        ns_str = match.group(3) or "0"
+        ms_str = match.group(2) if match.group(2) else ""
+        timestamptz_str_without_ns_str = match.group(1) + ms_str + match.group(4)
+        return datetime_to_nanos(datetime.fromisoformat(timestamptz_str_without_ns_str)) + int(ns_str)
+    if ISO_TIMESTAMPTZ_NANO.fullmatch(timestamptz_str):
         # When we can match a timestamp without a zone, we can give a more specific error
         raise ValueError(f"Missing zone offset: {timestamptz_str} (must be ISO-8601)")
     raise ValueError(f"Invalid timestamp with zone: {timestamptz_str} (must be ISO-8601)")

tests/test_transforms.py

@@ -152,11 +152,21 @@
             TimestampNanoType(),
             -1207196810,
         ),
+        (
+            timestamp_to_nanos("2017-11-16T22:31:08.000001001"),
+            TimestampNanoType(),
+            -1207196810,
+        ),
         (
             timestamptz_to_nanos("2017-11-16T14:31:08.000001-08:00"),
             TimestamptzNanoType(),
             -1207196810,
         ),
+        (
+            timestamptz_to_nanos("2017-11-16T14:31:08.000001001-08:00"),
+            TimestamptzNanoType(),
+            -1207196810,
+        ),
     ],
 )
 def test_bucket_hash_values(test_input: Any, test_type: PrimitiveType, expected: Any) -> None:

tests/utils/test_datetime.py

@@ -105,13 +105,25 @@ def test_datetime_to_nanos(datetime_: datetime, nanos: int) -> None:
     assert nanos == datetime_to_nanos(datetime_)
 
 
-@pytest.mark.parametrize("timestamp, nanos", [("1970-01-01T00:00:00", 0), ("2025-02-23T20:21:44.375612", 1740342104375612000)])
+@pytest.mark.parametrize(
+    "timestamp, nanos",
+    [
+        ("1970-01-01T00:00:00", 0),
+        ("2025-02-23T20:21:44.375612", 1740342104375612000),
+        ("2025-02-23T20:21:44.375612001", 1740342104375612001),
+    ],
+)
 def test_timestamp_to_nanos(timestamp: str, nanos: int) -> None:
     assert nanos == timestamp_to_nanos(timestamp)
 
 
 @pytest.mark.parametrize(
-    "timestamp, nanos", [("1970-01-01T00:00:00+00:00", 0), ("2025-02-23T16:21:44.375612-04:00", 1740342104375612000)]
+    "timestamp, nanos",
+    [
+        ("1970-01-01T00:00:00+00:00", 0),
+        ("2025-02-23T16:21:44.375612-04:00", 1740342104375612000),
+        ("2025-02-23T16:21:44.375612001-04:00", 1740342104375612001),
+    ],
 )
 def test_timestamptz_to_nanos(timestamp: str, nanos: int) -> None:
     assert nanos == timestamptz_to_nanos(timestamp)