From 252b606f001e9ee783e2d13d36275dbd1ef58539 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 09:53:11 +0100 Subject: [PATCH 01/19] Add addition_to_excluded_paths and excluded_paths to BaseConfig --- exasol/toolbox/config.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index d1b68bfb5..ef81bb25a 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -49,6 +49,15 @@ class BaseConfig(BaseModel): default=False, description="If true, creates also the major version tags (v*) automatically", ) + addition_to_excluded_paths: tuple[str, ...] = Field( + default=(), + description=""" + This is used to extend the default excluded_paths. If a more general path that + would be seen in other projects, like .venv, needs to be added into this + argument, please instead modify the + :meth:`exasol.toolbox.config.BaseConfig.excluded_paths` attribute. + """, + ) model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) @computed_field # type: ignore[misc] @@ -65,6 +74,23 @@ def minimum_python_version(self) -> str: index_min_version = versioned.index(min_version) return self.python_versions[index_min_version] + @computed_field + @property + def excluded_paths(self) -> tuple[str, ...]: + """ + There are certain nox sessions: + - lint:code + - lint:security + - lint:typing + - project:fix + - project:format + where it is desired restrict which Python files are considered within the + source_path, like excluding `dist`, `.eggs`. As such, this property is used to + exclude such undesired paths. + """ + default_excluded_paths = {"dist", ".eggs", "venv", ".poetry"} + return tuple(default_excluded_paths.union(set(self.addition_to_excluded_paths))) + @computed_field # type: ignore[misc] @property def pyupgrade_argument(self) -> tuple[str, ...]: From 380f3357714a3875095f27a7597ed98637b3f35a Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 10:07:09 +0100 Subject: [PATCH 02/19] Move PTB Config to using BaseConfig for excluded_paths --- exasol/toolbox/config.py | 9 ++++++++- exasol/toolbox/nox/_shared.py | 9 ++++++--- noxconfig.py | 12 ++++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index ef81bb25a..cddd36bf8 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -88,7 +88,14 @@ def excluded_paths(self) -> tuple[str, ...]: source_path, like excluding `dist`, `.eggs`. As such, this property is used to exclude such undesired paths. """ - default_excluded_paths = {"dist", ".eggs", "venv", ".poetry"} + default_excluded_paths = { + "dist", + ".eggs", + "venv", + ".poetry", + ".sonar", + ".html-documentation", + } return tuple(default_excluded_paths.union(set(self.addition_to_excluded_paths))) @computed_field # type: ignore[misc] diff --git a/exasol/toolbox/nox/_shared.py b/exasol/toolbox/nox/_shared.py index b0d4463ab..5e4e429ec 100644 --- a/exasol/toolbox/nox/_shared.py +++ b/exasol/toolbox/nox/_shared.py @@ -44,10 +44,13 @@ def python_files(project_root: Path) -> Iterable[str]: """ Returns iterable of python files after removing unwanted paths """ - deny_list = DEFAULT_PATH_FILTERS.union(set(PROJECT_CONFIG.path_filters)) - + check_for_config_attribute(config=PROJECT_CONFIG, attribute="excluded_paths") files = project_root.glob("**/*.py") - return [f"{path}" for path in files if not set(path.parts).intersection(deny_list)] + return [ + f"{path}" + for path in files + if not set(path.parts).intersection(PROJECT_CONFIG.excluded_paths) + ] def _version(session: Session, mode: Mode) -> None: diff --git a/noxconfig.py b/noxconfig.py index a71db3b6e..8629180ca 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -56,16 +56,16 @@ class Config(BaseConfig): source: Path = Path("exasol/toolbox") importlinter: Path = Path(__file__).parent / ".import_linter_config" version_file: Path = Path(__file__).parent / "exasol" / "toolbox" / "version.py" - path_filters: Iterable[str] = ( - "metrics-schema", - "project-template", - "idioms", - ".github", - ) plugins: Iterable[object] = (UpdateTemplates,) PROJECT_CONFIG = Config( + addition_to_excluded_paths=( + # The cookiecutter placeholders do not work well with checks. + # Instead, the format & linting are checked in the + # ``test.integration.project-template``. + "project-template", + ), create_major_version_tags=True, # The PTB does not have integration tests run with an Exasol DB, # so for running in the CI, we take the first element. From 0f92db15478c09ad30ebd71184531be69244202d Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 10:08:16 +0100 Subject: [PATCH 03/19] Fix title to right capitalization --- doc/github_actions/github_actions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/github_actions/github_actions.rst b/doc/github_actions/github_actions.rst index 001eeb2ef..e9139e886 100644 --- a/doc/github_actions/github_actions.rst +++ b/doc/github_actions/github_actions.rst @@ -1,6 +1,6 @@ .. _github_actions: -:octicon:`play` Github Actions +:octicon:`play` GitHub Actions =============================== .. toctree:: From 097b817da875cf6010c4014b9c08675a2fa83a55 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 11:15:28 +0100 Subject: [PATCH 04/19] Fix test to use BaseConfig provided values instead --- exasol/toolbox/nox/_shared.py | 1 - test/unit/nox/_shared_test.py | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/exasol/toolbox/nox/_shared.py b/exasol/toolbox/nox/_shared.py index 5e4e429ec..aff880c0e 100644 --- a/exasol/toolbox/nox/_shared.py +++ b/exasol/toolbox/nox/_shared.py @@ -20,7 +20,6 @@ Config, ) -DEFAULT_PATH_FILTERS = {"dist", ".eggs", "venv", ".poetry"} DOCS_OUTPUT_DIR = ".html-documentation" diff --git a/test/unit/nox/_shared_test.py b/test/unit/nox/_shared_test.py index 374e40afe..211a096fa 100644 --- a/test/unit/nox/_shared_test.py +++ b/test/unit/nox/_shared_test.py @@ -1,13 +1,12 @@ from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path +from unittest.mock import patch import pytest -import noxconfig from exasol.toolbox.config import BaseConfig from exasol.toolbox.nox._shared import ( - DEFAULT_PATH_FILTERS, check_for_config_attribute, python_files, ) @@ -30,7 +29,9 @@ def path_filter_directory(): @pytest.fixture(scope="session") def directories(package_directory, path_filter_directory): - yield DEFAULT_PATH_FILTERS.union({package_directory, path_filter_directory}) + yield set(BaseConfig().excluded_paths).union( + {package_directory, path_filter_directory} + ) @pytest.fixture(scope="session") @@ -50,12 +51,11 @@ def create_files(tmp_directory, directories): def test_python_files( tmp_directory, create_files, package_directory, path_filter_directory ): - # Use builtin object to modify attribute path_filters of frozen dataclass instance. - object.__setattr__( - noxconfig.PROJECT_CONFIG, "path_filters", (path_filter_directory,) - ) + config = BaseConfig(addition_to_excluded_paths=(path_filter_directory,)) + + with patch("exasol.toolbox.nox._shared.PROJECT_CONFIG", config): + actual = python_files(tmp_directory) - actual = python_files(tmp_directory) assert len(actual) == 1 assert "toolbox-dummy" in actual[0] From a0215d94c5eb89648000ae65412ea4835a871dd1 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 11:20:56 +0100 Subject: [PATCH 05/19] Make names more explicit as just for python files --- exasol/toolbox/config.py | 14 ++++++++------ exasol/toolbox/nox/_shared.py | 4 ++-- noxconfig.py | 2 +- test/unit/nox/_shared_test.py | 4 ++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index cddd36bf8..9ec17334c 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -49,12 +49,12 @@ class BaseConfig(BaseModel): default=False, description="If true, creates also the major version tags (v*) automatically", ) - addition_to_excluded_paths: tuple[str, ...] = Field( + add_to_excluded_python_paths: tuple[str, ...] = Field( default=(), description=""" - This is used to extend the default excluded_paths. If a more general path that - would be seen in other projects, like .venv, needs to be added into this - argument, please instead modify the + This is used to extend the default excluded_python_paths. If a more general + path that would be seen in other projects, like .venv, needs to be added into + this argument, please instead modify the :meth:`exasol.toolbox.config.BaseConfig.excluded_paths` attribute. """, ) @@ -76,7 +76,7 @@ def minimum_python_version(self) -> str: @computed_field @property - def excluded_paths(self) -> tuple[str, ...]: + def excluded_python_paths(self) -> tuple[str, ...]: """ There are certain nox sessions: - lint:code @@ -96,7 +96,9 @@ def excluded_paths(self) -> tuple[str, ...]: ".sonar", ".html-documentation", } - return tuple(default_excluded_paths.union(set(self.addition_to_excluded_paths))) + return tuple( + default_excluded_paths.union(set(self.add_to_excluded_python_paths)) + ) @computed_field # type: ignore[misc] @property diff --git a/exasol/toolbox/nox/_shared.py b/exasol/toolbox/nox/_shared.py index aff880c0e..52220fd10 100644 --- a/exasol/toolbox/nox/_shared.py +++ b/exasol/toolbox/nox/_shared.py @@ -43,12 +43,12 @@ def python_files(project_root: Path) -> Iterable[str]: """ Returns iterable of python files after removing unwanted paths """ - check_for_config_attribute(config=PROJECT_CONFIG, attribute="excluded_paths") + check_for_config_attribute(config=PROJECT_CONFIG, attribute="excluded_python_paths") files = project_root.glob("**/*.py") return [ f"{path}" for path in files - if not set(path.parts).intersection(PROJECT_CONFIG.excluded_paths) + if not set(path.parts).intersection(PROJECT_CONFIG.excluded_python_paths) ] diff --git a/noxconfig.py b/noxconfig.py index 8629180ca..dec6d539b 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -60,7 +60,7 @@ class Config(BaseConfig): PROJECT_CONFIG = Config( - addition_to_excluded_paths=( + add_to_excluded_python_paths=( # The cookiecutter placeholders do not work well with checks. # Instead, the format & linting are checked in the # ``test.integration.project-template``. diff --git a/test/unit/nox/_shared_test.py b/test/unit/nox/_shared_test.py index 211a096fa..ee0a678c7 100644 --- a/test/unit/nox/_shared_test.py +++ b/test/unit/nox/_shared_test.py @@ -29,7 +29,7 @@ def path_filter_directory(): @pytest.fixture(scope="session") def directories(package_directory, path_filter_directory): - yield set(BaseConfig().excluded_paths).union( + yield set(BaseConfig().excluded_python_paths).union( {package_directory, path_filter_directory} ) @@ -51,7 +51,7 @@ def create_files(tmp_directory, directories): def test_python_files( tmp_directory, create_files, package_directory, path_filter_directory ): - config = BaseConfig(addition_to_excluded_paths=(path_filter_directory,)) + config = BaseConfig(add_to_excluded_python_paths=(path_filter_directory,)) with patch("exasol.toolbox.nox._shared.PROJECT_CONFIG", config): actual = python_files(tmp_directory) From 2707e6f37c65b0f223b7ffb017bf77dc6407548d Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 11:25:54 +0100 Subject: [PATCH 06/19] Rename python_files to get_filtered_python_files --- exasol/toolbox/nox/_format.py | 6 +++--- exasol/toolbox/nox/_lint.py | 14 +++++++------- exasol/toolbox/nox/_shared.py | 5 ++--- exasol/toolbox/nox/tasks.py | 4 ++-- test/unit/nox/_shared_test.py | 6 +++--- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/exasol/toolbox/nox/_format.py b/exasol/toolbox/nox/_format.py index 9e2789ef9..3d8e5c2a3 100644 --- a/exasol/toolbox/nox/_format.py +++ b/exasol/toolbox/nox/_format.py @@ -9,7 +9,7 @@ Mode, _version, check_for_config_attribute, - python_files, + get_filtered_python_files, ) from noxconfig import ( PROJECT_CONFIG, @@ -45,7 +45,7 @@ def command(*args: str) -> Iterable[str]: @nox.session(name="project:fix", python=False) def fix(session: Session) -> None: """Runs all automated fixes on the code base""" - py_files = python_files(PROJECT_CONFIG.root) + py_files = get_filtered_python_files(PROJECT_CONFIG.root) _version(session, Mode.Fix) _pyupgrade(session, config=PROJECT_CONFIG, files=py_files) _ruff(session, mode=Mode.Fix, files=py_files) @@ -55,6 +55,6 @@ def fix(session: Session) -> None: @nox.session(name="project:format", python=False) def fmt_check(session: Session) -> None: """Checks the project for correct formatting""" - py_files = python_files(PROJECT_CONFIG.root) + py_files = get_filtered_python_files(PROJECT_CONFIG.root) _ruff(session, mode=Mode.Check, files=py_files) _code_format(session=session, mode=Mode.Check, files=py_files) diff --git a/exasol/toolbox/nox/_lint.py b/exasol/toolbox/nox/_lint.py index b8d74f201..838105013 100644 --- a/exasol/toolbox/nox/_lint.py +++ b/exasol/toolbox/nox/_lint.py @@ -10,7 +10,7 @@ import tomlkit from nox import Session -from exasol.toolbox.nox._shared import python_files +from exasol.toolbox.nox._shared import get_filtered_python_files from exasol.toolbox.util.dependencies.shared_models import PoetryFiles from noxconfig import PROJECT_CONFIG @@ -120,22 +120,22 @@ def report_illegal(illegal: dict[str, list[str]], console: rich.console.Console) @nox.session(name="lint:code", python=False) def lint(session: Session) -> None: """Runs the static code analyzer on the project""" - py_files = python_files(PROJECT_CONFIG.root / PROJECT_CONFIG.source) - _pylint(session, py_files) + py_files = get_filtered_python_files(PROJECT_CONFIG.root / PROJECT_CONFIG.source) + _pylint(session=session, files=py_files) @nox.session(name="lint:typing", python=False) def type_check(session: Session) -> None: """Runs the type checker on the project""" - py_files = [f"{file}" for file in python_files(PROJECT_CONFIG.root)] - _type_check(session, py_files) + py_files = get_filtered_python_files(PROJECT_CONFIG.root) + _type_check(session=session, files=py_files) @nox.session(name="lint:security", python=False) def security_lint(session: Session) -> None: """Runs the security linter on the project""" - py_files = python_files(PROJECT_CONFIG.root / PROJECT_CONFIG.source) - _security_lint(session, py_files) + py_files = get_filtered_python_files(PROJECT_CONFIG.root / PROJECT_CONFIG.source) + _security_lint(session=session, files=py_files) @nox.session(name="lint:dependencies", python=False) diff --git a/exasol/toolbox/nox/_shared.py b/exasol/toolbox/nox/_shared.py index 52220fd10..fb1989b81 100644 --- a/exasol/toolbox/nox/_shared.py +++ b/exasol/toolbox/nox/_shared.py @@ -3,7 +3,6 @@ import argparse from collections import ChainMap from collections.abc import ( - Iterable, MutableMapping, ) from enum import ( @@ -39,9 +38,9 @@ class Mode(Enum): Check = auto() -def python_files(project_root: Path) -> Iterable[str]: +def get_filtered_python_files(project_root: Path) -> list[str]: """ - Returns iterable of python files after removing unwanted paths + Returns iterable of Python files after removing excluded paths """ check_for_config_attribute(config=PROJECT_CONFIG, attribute="excluded_python_paths") files = project_root.glob("**/*.py") diff --git a/exasol/toolbox/nox/tasks.py b/exasol/toolbox/nox/tasks.py index dbb0c257b..9d04bd802 100644 --- a/exasol/toolbox/nox/tasks.py +++ b/exasol/toolbox/nox/tasks.py @@ -34,7 +34,7 @@ def check(session: Session) -> None: """Runs all available checks on the project""" context = _context(session, coverage=True) - py_files = python_files(PROJECT_CONFIG.root) + py_files = get_filtered_python_files(PROJECT_CONFIG.root) _version(session, Mode.Check) _code_format(session, Mode.Check, py_files) _pylint(session, py_files) @@ -65,7 +65,7 @@ def check(session: Session) -> None: Mode, _context, _version, - python_files, + get_filtered_python_files, ) from exasol.toolbox.nox._ci import ( diff --git a/test/unit/nox/_shared_test.py b/test/unit/nox/_shared_test.py index ee0a678c7..6331ad7f6 100644 --- a/test/unit/nox/_shared_test.py +++ b/test/unit/nox/_shared_test.py @@ -8,7 +8,7 @@ from exasol.toolbox.config import BaseConfig from exasol.toolbox.nox._shared import ( check_for_config_attribute, - python_files, + get_filtered_python_files, ) @@ -48,13 +48,13 @@ def create_files(tmp_directory, directories): yield file_list -def test_python_files( +def test_get_filtered_python_files( tmp_directory, create_files, package_directory, path_filter_directory ): config = BaseConfig(add_to_excluded_python_paths=(path_filter_directory,)) with patch("exasol.toolbox.nox._shared.PROJECT_CONFIG", config): - actual = python_files(tmp_directory) + actual = get_filtered_python_files(tmp_directory) assert len(actual) == 1 assert "toolbox-dummy" in actual[0] From d2875ba40f100c786248c725d0ecc5f1d476af37 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 11:29:15 +0100 Subject: [PATCH 07/19] Rename fixture to match more closely to attribute name --- test/unit/nox/_shared_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/unit/nox/_shared_test.py b/test/unit/nox/_shared_test.py index 6331ad7f6..8783ad399 100644 --- a/test/unit/nox/_shared_test.py +++ b/test/unit/nox/_shared_test.py @@ -23,14 +23,14 @@ def tmp_directory(tmp_path_factory): @pytest.fixture(scope="session") -def path_filter_directory(): - return "path_filter" +def excluded_python_path(): + return "excluded_python_path" @pytest.fixture(scope="session") -def directories(package_directory, path_filter_directory): +def directories(package_directory, excluded_python_path): yield set(BaseConfig().excluded_python_paths).union( - {package_directory, path_filter_directory} + {package_directory, excluded_python_path} ) @@ -49,9 +49,9 @@ def create_files(tmp_directory, directories): def test_get_filtered_python_files( - tmp_directory, create_files, package_directory, path_filter_directory + tmp_directory, create_files, package_directory, excluded_python_path ): - config = BaseConfig(add_to_excluded_python_paths=(path_filter_directory,)) + config = BaseConfig(add_to_excluded_python_paths=(excluded_python_path,)) with patch("exasol.toolbox.nox._shared.PROJECT_CONFIG", config): actual = get_filtered_python_files(tmp_directory) From 979e429e7302dc6e4e15d8c91efe429170acc326 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 11:30:59 +0100 Subject: [PATCH 08/19] Add directory found in other PTB projects --- exasol/toolbox/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 9ec17334c..d7d4790ad 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -92,6 +92,7 @@ def excluded_python_paths(self) -> tuple[str, ...]: "dist", ".eggs", "venv", + ".venv", ".poetry", ".sonar", ".html-documentation", From 773dc0cbce5e28c09ba7c7c8280385e9e96d6ab6 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 11:32:23 +0100 Subject: [PATCH 09/19] Alphabetize paths so less messy --- exasol/toolbox/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index d7d4790ad..12fc25b5b 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -89,13 +89,13 @@ def excluded_python_paths(self) -> tuple[str, ...]: exclude such undesired paths. """ default_excluded_paths = { - "dist", ".eggs", - "venv", - ".venv", + ".html-documentation", ".poetry", ".sonar", - ".html-documentation", + ".venv", + "dist", + "venv", } return tuple( default_excluded_paths.union(set(self.add_to_excluded_python_paths)) From 858099893ad500c5dec4073fa8a407f6d2865d7c Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 11:42:01 +0100 Subject: [PATCH 10/19] Add test for test_excluded_python_paths and update description --- exasol/toolbox/config.py | 23 ++++++++++++----------- test/unit/config_test.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 12fc25b5b..9e979f031 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -20,6 +20,16 @@ def valid_version_string(version_string: str) -> str: ValidVersionStr = Annotated[str, AfterValidator(valid_version_string)] +DEFAULT_EXCLUDED_PATHS = { + ".eggs", + ".html-documentation", + ".poetry", + ".sonar", + ".venv", + "dist", + "venv", +} + class BaseConfig(BaseModel): """ @@ -55,7 +65,7 @@ class BaseConfig(BaseModel): This is used to extend the default excluded_python_paths. If a more general path that would be seen in other projects, like .venv, needs to be added into this argument, please instead modify the - :meth:`exasol.toolbox.config.BaseConfig.excluded_paths` attribute. + `exasol.toolbox.config.DEFAULT_EXCLUDED_PATHS`. """, ) model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) @@ -88,17 +98,8 @@ def excluded_python_paths(self) -> tuple[str, ...]: source_path, like excluding `dist`, `.eggs`. As such, this property is used to exclude such undesired paths. """ - default_excluded_paths = { - ".eggs", - ".html-documentation", - ".poetry", - ".sonar", - ".venv", - "dist", - "venv", - } return tuple( - default_excluded_paths.union(set(self.add_to_excluded_python_paths)) + DEFAULT_EXCLUDED_PATHS.union(set(self.add_to_excluded_python_paths)) ) @computed_field # type: ignore[misc] diff --git a/test/unit/config_test.py b/test/unit/config_test.py index f65ff03a9..685e7c14e 100644 --- a/test/unit/config_test.py +++ b/test/unit/config_test.py @@ -2,6 +2,7 @@ from pydantic_core._pydantic_core import ValidationError from exasol.toolbox.config import ( + DEFAULT_EXCLUDED_PATHS, BaseConfig, valid_version_string, ) @@ -66,3 +67,22 @@ def test_minimum_python_version(): def test_pyupgrade_argument(minimum_python_version): conf = BaseConfig(python_versions=("3.11", minimum_python_version, "3.12")) assert conf.pyupgrade_argument == ("--py310-plus",) + + +@pytest.mark.parametrize( + "add_to_excluded_python_paths,expected", + [ + pytest.param((), tuple(DEFAULT_EXCLUDED_PATHS), id="no_additions"), + pytest.param( + (next(iter(DEFAULT_EXCLUDED_PATHS)),), + tuple(DEFAULT_EXCLUDED_PATHS), + id="duplicate_addition", + ), + pytest.param( + ("dummy",), tuple(DEFAULT_EXCLUDED_PATHS) + ("dummy",), id="add_a_new_entry" + ), + ], +) +def test_excluded_python_paths(add_to_excluded_python_paths, expected): + conf = BaseConfig(add_to_excluded_python_paths=add_to_excluded_python_paths) + assert sorted(conf.excluded_python_paths) == sorted(expected) From 1126324fd28c1e0572a63d228acdb27c40066c97 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 11:48:44 +0100 Subject: [PATCH 11/19] Update cookiecutter template as now in BaseConfig --- project-template/{{cookiecutter.repo_name}}/noxconfig.py | 1 - 1 file changed, 1 deletion(-) diff --git a/project-template/{{cookiecutter.repo_name}}/noxconfig.py b/project-template/{{cookiecutter.repo_name}}/noxconfig.py index cb7c86200..301a6a5a0 100644 --- a/project-template/{{cookiecutter.repo_name}}/noxconfig.py +++ b/project-template/{{cookiecutter.repo_name}}/noxconfig.py @@ -16,7 +16,6 @@ class Config(BaseConfig): / "{{cookiecutter.package_name}}" / "version.py" ) - path_filters: Iterable[str] = () plugins: Iterable[object] = () PROJECT_CONFIG = Config() From 26300223df9c69dac299f67dd298babcf938acc1 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 11:57:39 +0100 Subject: [PATCH 12/19] Add blurb in documentation for --- doc/user_guide/features/formatting_code/index.rst | 7 +++++++ .../features/formatting_code/troubleshooting.rst | 2 ++ 2 files changed, 9 insertions(+) diff --git a/doc/user_guide/features/formatting_code/index.rst b/doc/user_guide/features/formatting_code/index.rst index ded96f8d7..de916f401 100644 --- a/doc/user_guide/features/formatting_code/index.rst +++ b/doc/user_guide/features/formatting_code/index.rst @@ -18,6 +18,13 @@ experience across projects. Nox sessions ++++++++++++ +.. note:: + To prevent Python files from being formatted, you can do one of the following: + * For a single file, use a comment in the files as described in :ref:`this table `. + * If it is a directory (i.e. ``.workspace``), then you can exclude it by + adding it to the ``add_to_excluded_python_paths`` in the project's ``Config`` + defined in the ``noxconfig.py``. + For autoformatting, the following tools are used: * `black `__ - diff --git a/doc/user_guide/features/formatting_code/troubleshooting.rst b/doc/user_guide/features/formatting_code/troubleshooting.rst index a2cac89e5..b9cff201e 100644 --- a/doc/user_guide/features/formatting_code/troubleshooting.rst +++ b/doc/user_guide/features/formatting_code/troubleshooting.rst @@ -15,6 +15,8 @@ you receive an error from ``project:format`` (i.e. ``isort`` or ``black``), it i likely that you need to update your configuration to align with :ref:`formatting_configuration`. +.. _prevent_auto_format: + The automatic formatting is doing x, but we shouldn't do that because of y --------------------------------------------------------------------------- Usually, automatic formatting is helpful, but there are rare cases where a developer From 54bba16cdb0efc00a03be20e904409198552afcc Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 11:59:28 +0100 Subject: [PATCH 13/19] Add changelog entry --- doc/changes/unreleased.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 79e701b84..9a45341c8 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1 +1,5 @@ # Unreleased + +## Feature + +* #613: Replaced `path_filters` with `BaseConfig.add_to_excluded_python_paths` and `BaseConfig.excluded_python_paths` From 557b7644140938adca3fc3caadf75305f5768b72 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 12:00:32 +0100 Subject: [PATCH 14/19] Add type hint ignore --- exasol/toolbox/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/toolbox/config.py b/exasol/toolbox/config.py index 9e979f031..9d55f277e 100644 --- a/exasol/toolbox/config.py +++ b/exasol/toolbox/config.py @@ -84,7 +84,7 @@ def minimum_python_version(self) -> str: index_min_version = versioned.index(min_version) return self.python_versions[index_min_version] - @computed_field + @computed_field # type: ignore[misc] @property def excluded_python_paths(self) -> tuple[str, ...]: """ From f479d71a1b459aa40a37f3433a8d0c964f2155c0 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 12:02:42 +0100 Subject: [PATCH 15/19] Remove empty test file --- test/unit/project_test.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 test/unit/project_test.py diff --git a/test/unit/project_test.py b/test/unit/project_test.py deleted file mode 100644 index 678272b17..000000000 --- a/test/unit/project_test.py +++ /dev/null @@ -1,10 +0,0 @@ -def test_python_files_with_default_filter() -> None: - pass - - -def test_python_files_with_no_filter() -> None: - pass - - -def test_deny_filter() -> None: - pass From 838037f90b42ad5a0a741d00fb8e89c6cea85c65 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 12:03:40 +0100 Subject: [PATCH 16/19] Fix old reference in migrating.rst --- doc/user_guide/migrating.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/user_guide/migrating.rst b/doc/user_guide/migrating.rst index 8463c48b3..5df604471 100644 --- a/doc/user_guide/migrating.rst +++ b/doc/user_guide/migrating.rst @@ -72,9 +72,9 @@ For example, if test execution isn't performed in the standard way (e.g., :code: # ATTENTION: # In cases where it is reasonable to use "internal" functions, please do those imports # within the function to keep them isolated and simplify future removal or replacement. - from exasol.toolbox.nox._shared import python_files + from exasol.toolbox.nox._shared import get_filtered_python_files - py_files = [f"{file}" for file in python_files(PROJECT_CONFIG.root)] + py_files = get_filtered_python_files(PROJECT_CONFIG.root) print("The original 'project:fix' task has been taken hostage by this overwrite") print("Files:\n{files}".format(files="\n".join(py_files)) From 559950c9be7ea838e2f77192f96f932f4b9f8a0f Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 13:05:33 +0100 Subject: [PATCH 17/19] Fix changelog entry to sub-issue #614 --- doc/changes/unreleased.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 9a45341c8..09408dc71 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,4 +2,4 @@ ## Feature -* #613: Replaced `path_filters` with `BaseConfig.add_to_excluded_python_paths` and `BaseConfig.excluded_python_paths` +* #614: Replaced `path_filters` with `BaseConfig.add_to_excluded_python_paths` and `BaseConfig.excluded_python_paths` From 5c5afc57e63002cb1e09f7862e8eafe54d474764 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 13:11:06 +0100 Subject: [PATCH 18/19] Skip get_poetry.py as not maintained by us, but added for convenience --- noxconfig.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/noxconfig.py b/noxconfig.py index dec6d539b..3f7adc169 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -65,6 +65,9 @@ class Config(BaseConfig): # Instead, the format & linting are checked in the # ``test.integration.project-template``. "project-template", + # This file comes from poetry (https://install.python-poetry.org/), + # so we should not modify it. + "get_poetry.py", ), create_major_version_tags=True, # The PTB does not have integration tests run with an Exasol DB, From eec672cede85fa235935c981fca2cd90e6d9b93c Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 27 Nov 2025 14:19:28 +0100 Subject: [PATCH 19/19] Add tests for lint:security, lint:typing, lint:code --- exasol/toolbox/nox/_lint.py | 7 ++- test/unit/nox/_lint_test.py | 99 +++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 test/unit/nox/_lint_test.py diff --git a/exasol/toolbox/nox/_lint.py b/exasol/toolbox/nox/_lint.py index 838105013..6ae54eb5e 100644 --- a/exasol/toolbox/nox/_lint.py +++ b/exasol/toolbox/nox/_lint.py @@ -16,10 +16,13 @@ def _pylint(session: Session, files: Iterable[str]) -> None: + json_file = PROJECT_CONFIG.root / ".lint.json" + txt_file = PROJECT_CONFIG.root / ".lint.txt" + session.run( "pylint", "--output-format", - "colorized,json:.lint.json,text:.lint.txt", + f"colorized,json:{json_file},text:{txt_file}", *files, ) @@ -47,7 +50,7 @@ def _security_lint(session: Session, files: Iterable[str]) -> None: "--format", "json", "--output", - ".security.json", + PROJECT_CONFIG.root / ".security.json", "--exit-zero", *files, ) diff --git a/test/unit/nox/_lint_test.py b/test/unit/nox/_lint_test.py new file mode 100644 index 000000000..9cd5084c4 --- /dev/null +++ b/test/unit/nox/_lint_test.py @@ -0,0 +1,99 @@ +import json +from inspect import cleandoc +from pathlib import Path +from unittest.mock import patch + +import pytest +from nox.command import CommandFailed + +from exasol.toolbox.nox._lint import ( + lint, + security_lint, + type_check, +) + + +@pytest.fixture +def file_with_multiple_problems(tmp_path): + """ + In this file with multiple problems, it is expected that the nox + lint sessions would detect the following errors: + + * lint:code + * C0304: Final newline missing (missing-final-newline) + * C0114: Missing module docstring (missing-module-docstring) + * W1510: 'subprocess.run' used without explicitly defining the value for 'check'. (subprocess-run-check) + * lint:typing + * Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] + * lint:security + * [B404:blacklist] Consider possible security implications associated with the subprocess module. + * [B607:start_process_with_partial_path] Starting a process with a partial executable path + * [B603:subprocess_without_shell_equals_true] subprocess call - check for execution of untrusted input. + """ + + file_path = tmp_path / "dummy_file.py" + text = """ + import subprocess + + x: str = 2 + subprocess.run("ls") + """ + file_path.write_text(cleandoc(text)) + return file_path + + +def test_lint(nox_session, tmp_path, file_with_multiple_problems): + with patch("exasol.toolbox.nox._lint.PROJECT_CONFIG") as config: + config.root = tmp_path + config.source = Path("") + with pytest.raises(CommandFailed, match="Returned code 20"): + lint(session=nox_session) + + json_file = tmp_path / ".lint.json" + txt_file = tmp_path / ".lint.txt" + + assert json_file.exists() + assert txt_file.exists() + + contents = json_file.read_text() + errors = {row["message-id"] for row in json.loads(contents)} + assert {"C0114", "C0304", "W1510"}.issubset(errors) + + +def test_type_check(nox_session, tmp_path, file_with_multiple_problems, caplog): + with patch("exasol.toolbox.nox._lint.PROJECT_CONFIG") as config: + config.root = tmp_path + config.source = Path("") + with pytest.raises(CommandFailed, match="Returned code 1"): + type_check(session=nox_session) + + assert caplog.messages[1] == ( + "Command mypy --explicit-package-bases --namespace-packages --show-error-codes " + "--pretty --show-column-numbers --show-error-context --scripts-are-modules " + f"{file_with_multiple_problems} failed with exit code 1" + ) + + +def test_security_lint(nox_session, tmp_path, file_with_multiple_problems): + with patch("exasol.toolbox.nox._lint.PROJECT_CONFIG") as config: + config.root = tmp_path + config.source = Path("") + security_lint(session=nox_session) + + output_file = tmp_path / ".security.json" + assert output_file.exists() + + contents = output_file.read_text() + assert json.loads(contents)["metrics"]["_totals"] == { + "CONFIDENCE.HIGH": 3, + "CONFIDENCE.LOW": 0, + "CONFIDENCE.MEDIUM": 0, + "CONFIDENCE.UNDEFINED": 0, + "SEVERITY.HIGH": 0, + "SEVERITY.LOW": 3, + "SEVERITY.MEDIUM": 0, + "SEVERITY.UNDEFINED": 0, + "loc": 3, + "nosec": 0, + "skipped_tests": 0, + }