From 4505f0c95e036a88f09ff0d928b7800b1a168c38 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 28 Dec 2025 10:59:44 -0800 Subject: [PATCH 1/2] add log level to CLI --- mkdocs/docs/contributing.md | 21 +++++++++++++ pyiceberg/cli/console.py | 17 ++++++++++ tests/cli/test_console.py | 62 +++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/mkdocs/docs/contributing.md b/mkdocs/docs/contributing.md index aaecab2cb0..4f13570904 100644 --- a/mkdocs/docs/contributing.md +++ b/mkdocs/docs/contributing.md @@ -258,6 +258,27 @@ Which will warn: Deprecated in 0.1.0, will be removed in 0.2.0. The old_property is deprecated. Please use the something_else property instead. ``` +### Logging + +PyIceberg uses Python's standard logging module. You can control the logging level using either: + +**CLI option:** + +```bash +pyiceberg --log-level DEBUG describe my_table +``` + +**Environment variable:** + +```bash +export PYICEBERG_LOG_LEVEL=DEBUG +pyiceberg describe my_table +``` + +Valid log levels are: `DEBUG`, `INFO`, `WARNING` (default), `ERROR`, `CRITICAL`. + +Debug logging is particularly useful for troubleshooting issues with FileIO implementations, catalog connections, and other integration points. + ### Type annotations For the type annotation the types from the `Typing` package are used. diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index 9baa813eff..21722bf31a 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # pylint: disable=broad-except,redefined-builtin,redefined-outer-name +import logging +import os from collections.abc import Callable from functools import wraps from typing import ( @@ -33,6 +35,8 @@ from pyiceberg.table.refs import SnapshotRef, SnapshotRefType from pyiceberg.utils.properties import property_as_int +DEFAULT_LOG_LEVEL = "WARNING" + def catch_exception() -> Callable: # type: ignore def decorator(func: Callable) -> Callable: # type: ignore @@ -55,6 +59,11 @@ def wrapper(*args: Any, **kwargs: Any): # type: ignore @click.option("--catalog") @click.option("--verbose", type=click.BOOL) @click.option("--output", type=click.Choice(["text", "json"]), default="text") +@click.option( + "--log-level", + type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False), + help="Set the logging level (also configurable via PYICEBERG_LOG_LEVEL environment variable)", +) @click.option("--ugi") @click.option("--uri") @click.option("--credential") @@ -64,10 +73,18 @@ def run( catalog: str | None, verbose: bool, output: str, + log_level: str | None, ugi: str | None, uri: str | None, credential: str | None, ) -> None: + # Configure logging level from CLI option or environment variable + level = log_level or os.getenv("PYICEBERG_LOG_LEVEL") or DEFAULT_LOG_LEVEL + logging.basicConfig( + level=getattr(logging, level.upper()), + format="%(levelname)s:%(name)s:%(message)s", + ) + properties = {} if ugi: properties["ugi"] = ugi diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index a0e9552236..a713975ec9 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -967,3 +967,65 @@ def test_json_properties_remove_table_does_not_exist(catalog: InMemoryCatalog) - result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.doesnotexist", "location"]) assert result.exit_code == 1 assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexist"}\n""" + + +def test_log_level_cli_option(mocker: MockFixture) -> None: + mock_basicConfig = mocker.patch("logging.basicConfig") + + runner = CliRunner() + runner.invoke(run, ["--log-level", "DEBUG", "list"]) + + # Verify logging.basicConfig was called with DEBUG level + import logging + + mock_basicConfig.assert_called_once() + call_kwargs = mock_basicConfig.call_args[1] + assert call_kwargs["level"] == logging.DEBUG + + +def test_log_level_env_variable(mocker: MockFixture) -> None: + mock_basicConfig = mocker.patch("logging.basicConfig") + mocker.patch.dict(os.environ, {"PYICEBERG_LOG_LEVEL": "INFO"}) + + runner = CliRunner() + runner.invoke(run, ["list"]) + + # Verify logging.basicConfig was called with INFO level + import logging + + mock_basicConfig.assert_called_once() + call_kwargs = mock_basicConfig.call_args[1] + assert call_kwargs["level"] == logging.INFO + + +def test_log_level_default_warning(mocker: MockFixture) -> None: + mock_basicConfig = mocker.patch("logging.basicConfig") + # Ensure PYICEBERG_LOG_LEVEL is not set + mocker.patch.dict(os.environ, {}, clear=False) + if "PYICEBERG_LOG_LEVEL" in os.environ: + del os.environ["PYICEBERG_LOG_LEVEL"] + + runner = CliRunner() + runner.invoke(run, ["list"]) + + # Verify logging.basicConfig was called with WARNING level (default) + import logging + + mock_basicConfig.assert_called_once() + call_kwargs = mock_basicConfig.call_args[1] + assert call_kwargs["level"] == logging.WARNING + + +def test_log_level_cli_overrides_env(mocker: MockFixture) -> None: + mock_basicConfig = mocker.patch("logging.basicConfig") + mocker.patch.dict(os.environ, {"PYICEBERG_LOG_LEVEL": "INFO"}) + + runner = CliRunner() + runner.invoke(run, ["--log-level", "ERROR", "list"]) + + # Verify CLI option overrides environment variable + import logging + + mock_basicConfig.assert_called_once() + call_kwargs = mock_basicConfig.call_args[1] + assert call_kwargs["level"] == logging.ERROR From 85fc6a7f48b5e772b46dd45831432eec3452e3ed Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 28 Dec 2025 13:19:15 -0800 Subject: [PATCH 2/2] thx drew --- pyiceberg/cli/console.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index 21722bf31a..c59919c5d9 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -16,7 +16,6 @@ # under the License. # pylint: disable=broad-except,redefined-builtin,redefined-outer-name import logging -import os from collections.abc import Callable from functools import wraps from typing import ( @@ -35,8 +34,6 @@ from pyiceberg.table.refs import SnapshotRef, SnapshotRefType from pyiceberg.utils.properties import property_as_int -DEFAULT_LOG_LEVEL = "WARNING" - def catch_exception() -> Callable: # type: ignore def decorator(func: Callable) -> Callable: # type: ignore @@ -62,7 +59,9 @@ def wrapper(*args: Any, **kwargs: Any): # type: ignore @click.option( "--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False), - help="Set the logging level (also configurable via PYICEBERG_LOG_LEVEL environment variable)", + default="WARNING", + envvar="PYICEBERG_LOG_LEVEL", + help="Set the logging level", ) @click.option("--ugi") @click.option("--uri") @@ -73,15 +72,13 @@ def run( catalog: str | None, verbose: bool, output: str, - log_level: str | None, + log_level: str, ugi: str | None, uri: str | None, credential: str | None, ) -> None: - # Configure logging level from CLI option or environment variable - level = log_level or os.getenv("PYICEBERG_LOG_LEVEL") or DEFAULT_LOG_LEVEL logging.basicConfig( - level=getattr(logging, level.upper()), + level=getattr(logging, log_level.upper()), format="%(levelname)s:%(name)s:%(message)s", )