diff --git a/mkdocs/docs/contributing.md b/mkdocs/docs/contributing.md index 4ff3541b54..b9e4e5c50c 100644 --- a/mkdocs/docs/contributing.md +++ b/mkdocs/docs/contributing.md @@ -155,6 +155,34 @@ make test-integration-rebuild To rebuild the containers from scratch. +#### Running Integration Tests against REST Catalogs + +!!! warning "Do not run against production catalogs" + The integration tests will delete data throughout the entirety of your catalog. Running these integration tests against production catalogs will result in data loss. + +PyIceberg supports the ability to run our catalog tests against an arbitrary REST Catalog. + +In order to run the test catalog, you will need to specify which REST catalog to run against with the `PYICEBERG_TEST_CATALOG` environment variable + +```sh +export PYICEBERG_TEST_CATALOG=test_catalog +``` + +The catalog in question can be configured either through the ~/.pyiceberg.yaml file or through environment variables. + +```yaml +catalog: + test_catalog: + uri: http://rest-catalog/ws/ + credential: t-1234:secret +``` + +```sh +export PYICEBERG_CATALOG__TEST_CATALOG__URI=thrift://localhost:9083 +export PYICEBERG_CATALOG__TEST_CATALOG__ACCESS_KEY_ID=username +export PYICEBERG_CATALOG__TEST_CATALOG__SECRET_ACCESS_KEY=password +``` + ## Code standards Below are the formalized conventions that we adhere to in the PyIceberg project. The goal of this is to have a common agreement on how to evolve the codebase, but also using it as guidelines for newcomers to the project. diff --git a/tests/integration/test_catalog.py b/tests/integration/test_catalog.py index 587c13b35b..b57ab983ba 100644 --- a/tests/integration/test_catalog.py +++ b/tests/integration/test_catalog.py @@ -15,12 +15,13 @@ # specific language governing permissions and limitations # under the License. +import os from pathlib import Path, PosixPath from typing import Generator, List import pytest -from pyiceberg.catalog import Catalog, MetastoreCatalog +from pyiceberg.catalog import Catalog, MetastoreCatalog, load_catalog from pyiceberg.catalog.hive import HiveCatalog from pyiceberg.catalog.memory import InMemoryCatalog from pyiceberg.catalog.rest import RestCatalog @@ -74,6 +75,16 @@ def rest_catalog() -> Generator[Catalog, None, None]: clean_up(test_catalog) +@pytest.fixture(scope="function") +def rest_test_catalog() -> Generator[Catalog, None, None]: + if test_catalog_name := os.environ.get("PYICEBERG_TEST_CATALOG"): + test_catalog = load_catalog(test_catalog_name) + yield test_catalog + clean_up(test_catalog) + else: + pytest.skip("PYICEBERG_TEST_CATALOG environment variables not set") + + @pytest.fixture(scope="function") def hive_catalog() -> Generator[Catalog, None, None]: test_catalog = HiveCatalog( @@ -95,6 +106,7 @@ def hive_catalog() -> Generator[Catalog, None, None]: pytest.lazy_fixture("sqlite_catalog_file"), pytest.lazy_fixture("rest_catalog"), pytest.lazy_fixture("hive_catalog"), + pytest.lazy_fixture("rest_test_catalog"), ]