Run Catalog integration tests against REST Catalog impls

mkdocs/docs/contributing.md

@@ -155,6 +155,34 @@ make test-integration-rebuild
 
 To rebuild the containers from scratch.
 
+#### Running Integration Tests against REST Catalogs
+
+!!! warning "Do not run against production catalogs"
+    The integration tests will delete data throughout the entirety of your catalog. Running these integration tests against production catalogs will result in data loss.
+
+PyIceberg supports the ability to run our catalog tests against an arbitrary REST Catalog.
+
+In order to run the test catalog, you will need to specify which REST catalog to run against with the `PYICEBERG_TEST_CATALOG` environment variable
+
+```sh
+export PYICEBERG_TEST_CATALOG=test_catalog
+```
+
+The catalog in question can be configured either through the ~/.pyiceberg.yaml file or through environment variables.
+
+```yaml
+catalog:
+  test_catalog:
+    uri: http://rest-catalog/ws/
+    credential: t-1234:secret
+```
+
+```sh
+export PYICEBERG_CATALOG__TEST_CATALOG__URI=thrift://localhost:9083
+export PYICEBERG_CATALOG__TEST_CATALOG__ACCESS_KEY_ID=username
+export PYICEBERG_CATALOG__TEST_CATALOG__SECRET_ACCESS_KEY=password
+```
+
 ## Code standards
 
 Below are the formalized conventions that we adhere to in the PyIceberg project. The goal of this is to have a common agreement on how to evolve the codebase, but also using it as guidelines for newcomers to the project.

tests/integration/test_catalog.py

@@ -15,12 +15,13 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
+import os
 from pathlib import Path, PosixPath
 from typing import Generator, List
 
 import pytest
 
-from pyiceberg.catalog import Catalog, MetastoreCatalog
+from pyiceberg.catalog import Catalog, MetastoreCatalog, load_catalog
 from pyiceberg.catalog.hive import HiveCatalog
 from pyiceberg.catalog.memory import InMemoryCatalog
 from pyiceberg.catalog.rest import RestCatalog
@@ -74,6 +75,16 @@ def rest_catalog() -> Generator[Catalog, None, None]:
     clean_up(test_catalog)
 
 
+@pytest.fixture(scope="function")
+def rest_test_catalog() -> Generator[Catalog, None, None]:
+    if test_catalog_name := os.environ.get("PYICEBERG_TEST_CATALOG"):
+        test_catalog = load_catalog(test_catalog_name)
+        yield test_catalog
+        clean_up(test_catalog)
+    else:
+        pytest.skip("PYICEBERG_TEST_CATALOG environment variables not set")
+
+
 @pytest.fixture(scope="function")
 def hive_catalog() -> Generator[Catalog, None, None]:
     test_catalog = HiveCatalog(
@@ -95,6 +106,7 @@ def hive_catalog() -> Generator[Catalog, None, None]:
     pytest.lazy_fixture("sqlite_catalog_file"),
     pytest.lazy_fixture("rest_catalog"),
     pytest.lazy_fixture("hive_catalog"),
+    pytest.lazy_fixture("rest_test_catalog"),
 ]