allow older_than to be None

Author: jayceslesar

pyiceberg/table/maintenance.py

@@ -19,7 +19,7 @@
 import logging
 from datetime import datetime, timedelta, timezone
 from functools import reduce
-from typing import TYPE_CHECKING, Set
+from typing import TYPE_CHECKING, Optional, Set
 
 from pyiceberg.utils.concurrent import ExecutorFactory
 
@@ -41,7 +41,7 @@ def __init__(self, tbl: Table) -> None:
         except ModuleNotFoundError as e:
             raise ModuleNotFoundError("For metadata operations PyArrow needs to be installed") from e
 
-    def _orphaned_files(self, location: str, older_than: timedelta = timedelta(days=3)) -> Set[str]:
+    def _orphaned_files(self, location: str, older_than: Optional[timedelta] = None) -> Set[str]:
         """Get all files which are not referenced in any metadata files of an Iceberg table and can thus be considered "orphaned".
 
         Args:
@@ -69,15 +69,18 @@ def _orphaned_files(self, location: str, older_than: timedelta = timedelta(days=
 
         _, _, path = pyarrow_io.parse_location(location)
         selector = FileSelector(path, recursive=True)
+
         # filter to just files as it may return directories, and filter on time
+        if older_than is None:
+            older_than = timedelta(0)
         as_of = datetime.now(timezone.utc) - older_than
         all_files = [f.path for f in fs.get_file_info(selector) if f.type == FileType.File and f.mtime < as_of]
 
         orphaned_files = set(all_files).difference(flat_known_files)
 
         return orphaned_files
 
-    def remove_orphaned_files(self, older_than: timedelta = timedelta(days=3), dry_run: bool = False) -> None:
+    def remove_orphaned_files(self, older_than: Optional[timedelta] = None, dry_run: bool = False) -> None:
         """Remove files which are not referenced in any metadata files of an Iceberg table and can thus be considered "orphaned".
 
         Args:

tests/table/test_remove_orphans.py

@@ -72,10 +72,6 @@ def test_remove_orphaned_files(catalog: Catalog) -> None:
     tbl.maintenance.remove_orphaned_files(dry_run=True)
     assert orphaned_file.exists()
 
-    # should not delete because it was just created...
-    tbl.maintenance.remove_orphaned_files()
-    assert orphaned_file.exists()
-
     # modify creation date to be older than 3 days
     five_days_ago = (datetime.now() - timedelta(days=5)).timestamp()
     os.utime(orphaned_file, (five_days_ago, five_days_ago))