From a89bc19e3592d53d9027a9f851fad56bc1ea5c6b Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 7 Nov 2025 19:38:21 +0000 Subject: [PATCH 1/3] mypy: Ignore missing annotations globally Weirdly, this also showed some new issues: ``` mypy.....................................................................Failed - hook id: mypy - exit code: 1 mkdocs/gen_doc_stubs.py:20: error: Unused "type: ignore" comment [unused-ignore] mkdocs/gen_doc_stubs.py:21: error: Unused "type: ignore" comment [unused-ignore] tests/utils/test_manifest.py:51: error: "Callable[[FileIO, str], tuple[ManifestFile, ...]]" has no attribute "cache_clear" [attr-defined] tests/conftest.py:101: error: Unused "type: ignore" comment [unused-ignore] Found 4 errors in 3 files (checked 166 source files) ``` --- mkdocs/gen_doc_stubs.py | 4 +- pyiceberg/manifest.py | 6 +- pyproject.toml | 343 ----------------------------------- tests/conftest.py | 2 +- tests/utils/test_manifest.py | 4 +- 5 files changed, 10 insertions(+), 349 deletions(-) diff --git a/mkdocs/gen_doc_stubs.py b/mkdocs/gen_doc_stubs.py index bd3b128101..a804d65db7 100644 --- a/mkdocs/gen_doc_stubs.py +++ b/mkdocs/gen_doc_stubs.py @@ -17,8 +17,8 @@ from pathlib import Path -import griffe # type: ignore -import mkdocs_gen_files # type: ignore +import griffe +import mkdocs_gen_files nav = mkdocs_gen_files.Nav() diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index eafb2b7c03..c8c5eb3366 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -876,7 +876,11 @@ def __hash__(self) -> int: return hash(self.manifest_path) -@cached(cache=LRUCache(maxsize=128), key=lambda io, manifest_list: hashkey(manifest_list), lock=threading.RLock()) +# Global cache for manifest lists +_manifest_cache: LRUCache[Any, Tuple[ManifestFile, ...]] = LRUCache(maxsize=128) + + +@cached(cache=_manifest_cache, key=lambda io, manifest_list: hashkey(manifest_list), lock=threading.RLock()) def _manifests(io: FileIO, manifest_list: str) -> Tuple[ManifestFile, ...]: """Read and cache manifests from the given manifest list, returning a tuple to prevent modification.""" file = io.new_input(manifest_list) diff --git a/pyproject.toml b/pyproject.toml index ab09140b78..2b8572be2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,178 +119,6 @@ mkdocs-material = "9.6.22" mkdocs-material-extensions = "1.3.1" mkdocs-section-index = "0.3.10" -[[tool.mypy.overrides]] -module = "pytest_mock.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pyarrow.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "google.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pandas.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "snappy.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "zstandard.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pydantic.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pydantic_core.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pytest.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "fastavro.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "mmh3.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "hive_metastore.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "thrift.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "requests_mock.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "click.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "rich.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "fsspec.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "s3fs.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "azure.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "adlfs.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "gcsfs.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "packaging.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "tests.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "boto3" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "botocore.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "mypy_boto3_glue.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "mypy_boto3_dynamodb.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "moto" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "aiobotocore.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "aiohttp.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "duckdb.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "ray.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "daft.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pyparsing.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pyspark.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "strictyaml.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "sortedcontainers.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "sqlalchemy.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "Cython.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "setuptools.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "tenacity.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pyiceberg_core.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "google.*" -ignore_missing_imports = true - [tool.poetry.scripts] pyiceberg = "pyiceberg.cli.console:run" @@ -362,177 +190,6 @@ warn_unused_ignores = true disallow_any_generics = true disallow_untyped_defs = true implicit_reexport = false - -[[tool.mypy.overrides]] -module = "pyarrow.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pandas.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "snappy.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "zstandard.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pydantic.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pydantic_core.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pytest.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "fastavro.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "mmh3.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "hive_metastore.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "thrift.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "requests_mock.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "click.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "rich.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "fsspec.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "s3fs.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "azure.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "adlfs.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "gcsfs.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "huggingface_hub.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "packaging.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "tests.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "boto3" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "botocore.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "mypy_boto3_glue.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "moto" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "aiobotocore.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "aiohttp.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "duckdb.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "ray.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "daft.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "bodo.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pyparsing.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pyspark.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "strictyaml.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "sortedcontainers.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "sqlalchemy.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "Cython.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "setuptools.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "tenacity.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "polars.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "datafusion.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pyroaring.*" ignore_missing_imports = true [tool.coverage.run] diff --git a/tests/conftest.py b/tests/conftest.py index 6734932993..b14c4eef8e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -98,7 +98,7 @@ if TYPE_CHECKING: import pyarrow as pa - from moto.server import ThreadedMotoServer # type: ignore + from moto.server import ThreadedMotoServer from pyspark.sql import SparkSession from pyiceberg.io.pyarrow import PyArrowFileIO diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index a5d5a6fefb..314170adb8 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -33,7 +33,7 @@ ManifestEntryStatus, ManifestFile, PartitionFieldSummary, - _manifests, + _manifest_cache, read_manifest_list, write_manifest, write_manifest_list, @@ -48,7 +48,7 @@ @pytest.fixture(autouse=True) def clear_global_manifests_cache() -> None: # Clear the global cache before each test - _manifests.cache_clear() + _manifest_cache.clear() def _verify_metadata_with_fastavro(avro_file: str, expected_metadata: Dict[str, str]) -> None: From 0d2280ee0803410d89719bee41c1098a150bed63 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 7 Nov 2025 11:46:07 -0800 Subject: [PATCH 2/3] remove last ignore_missing_imports --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2b8572be2e..490753be79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -190,7 +190,6 @@ warn_unused_ignores = true disallow_any_generics = true disallow_untyped_defs = true implicit_reexport = false -ignore_missing_imports = true [tool.coverage.run] source = ['pyiceberg/'] From 0bda1c4e40430338fdb26705c2577dff02a02db6 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 7 Nov 2025 11:47:39 -0800 Subject: [PATCH 3/3] Revert "remove last ignore_missing_imports" This reverts commit 0d2280ee0803410d89719bee41c1098a150bed63. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 490753be79..2b8572be2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -190,6 +190,7 @@ warn_unused_ignores = true disallow_any_generics = true disallow_untyped_defs = true implicit_reexport = false +ignore_missing_imports = true [tool.coverage.run] source = ['pyiceberg/']