infra: upgrade pytest and use pytest_lazy_fixtures (#2978)

<!--
Thanks for opening a pull request!
-->

<!-- In the case this PR will resolve an issue, please replace
${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
<!-- Closes #${GITHUB_ISSUE_ID} -->

# Rationale for this change
Closes #2810, #393
Relates to #2743

This PR upgrades `pytest` to `9.0.2`. In doing so, we also had to switch
`pytest-lazy-fixture` to `pytest-lazy-fixtures` (notice the extra s)
which is a maintained version
https://github.com/dev-petrov/pytest-lazy-fixtures

Changed all references of `pytest_lazyfixture` to the new
`pytest_lazy_fixtures`

## Are these changes tested?

## Are there any user-facing changes?

<!-- In the case of user-facing changes, please add the changelog label.
-->

Author: kevinjqliu

pyproject.toml

@@ -100,10 +100,10 @@ entra-auth = ["azure-identity>=1.25.1"]
 
 [dependency-groups]
 dev = [
-    "pytest==7.4.4",
+    "pytest==9.0.2",
     "pytest-checkdocs==2.14.0",
     "prek>=0.2.1,<0.4",
-    "pytest-lazy-fixture==0.6.3",
+    "pytest-lazy-fixtures==1.4.0",
     "fastavro==1.12.1",
     "coverage[toml]>=7.4.2,<8",
     "requests-mock==1.12.1",

tests/catalog/test_catalog_behaviors.py

@@ -26,7 +26,7 @@
 import pyarrow as pa
 import pytest
 from pydantic_core import ValidationError
-from pytest_lazyfixture import lazy_fixture
+from pytest_lazy_fixtures import lf
 from sqlalchemy.exc import IntegrityError
 
 from pyiceberg.catalog import Catalog
@@ -323,10 +323,10 @@ def test_write_pyarrow_schema(catalog: Catalog, test_table_identifier: Identifie
 @pytest.mark.parametrize(
     "schema,expected",
     [
-        (lazy_fixture("pyarrow_schema_simple_without_ids"), lazy_fixture("iceberg_schema_simple_no_ids")),
-        (lazy_fixture("table_schema_simple"), lazy_fixture("table_schema_simple")),
-        (lazy_fixture("table_schema_nested"), lazy_fixture("table_schema_nested")),
-        (lazy_fixture("pyarrow_schema_nested_without_ids"), lazy_fixture("iceberg_schema_nested_no_ids")),
+        (lf("pyarrow_schema_simple_without_ids"), lf("iceberg_schema_simple_no_ids")),
+        (lf("table_schema_simple"), lf("table_schema_simple")),
+        (lf("table_schema_nested"), lf("table_schema_nested")),
+        (lf("pyarrow_schema_nested_without_ids"), lf("iceberg_schema_nested_no_ids")),
     ],
 )
 def test_convert_schema_if_needed(

tests/conftest.py

@@ -45,7 +45,7 @@
 import pytest
 from moto import mock_aws
 from pydantic_core import to_json
-from pytest_lazyfixture import lazy_fixture
+from pytest_lazy_fixtures import lf
 
 from pyiceberg.catalog import Catalog, load_catalog
 from pyiceberg.catalog.memory import InMemoryCatalog
@@ -3080,23 +3080,21 @@ def fixed_test_table_namespace() -> Identifier:
 
 
 @pytest.fixture(
-    scope="session",
     params=[
-        lazy_fixture("fixed_test_table_identifier"),
-        lazy_fixture("random_table_identifier"),
-        lazy_fixture("random_hierarchical_identifier"),
+        lf("fixed_test_table_identifier"),
+        lf("random_table_identifier"),
+        lf("random_hierarchical_identifier"),
     ],
 )
 def test_table_identifier(request: pytest.FixtureRequest) -> Identifier:
     return request.param
 
 
 @pytest.fixture(
-    scope="session",
     params=[
-        lazy_fixture("another_fixed_test_table_identifier"),
-        lazy_fixture("another_random_table_identifier"),
-        lazy_fixture("another_random_hierarchical_identifier"),
+        lf("another_fixed_test_table_identifier"),
+        lf("another_random_table_identifier"),
+        lf("another_random_hierarchical_identifier"),
     ],
 )
 def another_table_identifier(request: pytest.FixtureRequest) -> Identifier:
@@ -3105,9 +3103,9 @@ def another_table_identifier(request: pytest.FixtureRequest) -> Identifier:
 
 @pytest.fixture(
     params=[
-        lazy_fixture("database_name"),
-        lazy_fixture("hierarchical_namespace_name"),
-        lazy_fixture("fixed_test_table_namespace"),
+        lf("database_name"),
+        lf("hierarchical_namespace_name"),
+        lf("fixed_test_table_namespace"),
     ],
 )
 def test_namespace(request: pytest.FixtureRequest) -> Identifier:

tests/integration/test_catalog.py

@@ -21,6 +21,7 @@
 from pathlib import Path, PosixPath
 
 import pytest
+from pytest_lazy_fixtures import lf
 
 from pyiceberg.catalog import Catalog, MetastoreCatalog, load_catalog
 from pyiceberg.catalog.hive import HiveCatalog
@@ -109,12 +110,12 @@ def hive_catalog() -> Generator[Catalog, None, None]:
 
 
 CATALOGS = [
-    pytest.lazy_fixture("memory_catalog"),
-    pytest.lazy_fixture("sqlite_catalog_memory"),
-    pytest.lazy_fixture("sqlite_catalog_file"),
-    pytest.lazy_fixture("rest_catalog"),
-    pytest.lazy_fixture("hive_catalog"),
-    pytest.lazy_fixture("rest_test_catalog"),
+    lf("memory_catalog"),
+    lf("sqlite_catalog_memory"),
+    lf("sqlite_catalog_file"),
+    lf("rest_catalog"),
+    lf("hive_catalog"),
+    lf("rest_test_catalog"),
 ]
 
 

tests/integration/test_inspect_table.py

@@ -24,6 +24,7 @@
 import pytest
 import pytz
 from pyspark.sql import DataFrame, SparkSession
+from pytest_lazy_fixtures import lf
 
 from pyiceberg.catalog import Catalog
 from pyiceberg.exceptions import NoSuchTableError
@@ -672,7 +673,7 @@ def test_inspect_partitions_partitioned_with_filter(spark: SparkSession, session
 
 
 @pytest.mark.integration
-@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")])
+@pytest.mark.parametrize("catalog", [lf("session_catalog")])
 def test_inspect_partitions_partitioned_transform_with_filter(spark: SparkSession, catalog: Catalog) -> None:
     for table_name, predicate, partition_predicate in [
         ("test_partitioned_by_identity", "ts >= '2023-03-05T00:00:00+00:00'", "ts >= '2023-03-05T00:00:00+00:00'"),