From 70b3edd51c1465b5c23c877e3fd4cf036a8c98d1 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Thu, 20 Mar 2025 11:47:27 +0100 Subject: [PATCH 1/2] [Python] Add tests for recursive imports --- .../python/databricks/bundles/core/_load.py | 2 +- .../databricks_tests/core/test_import.py | 61 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 experimental/python/databricks_tests/core/test_import.py diff --git a/experimental/python/databricks/bundles/core/_load.py b/experimental/python/databricks/bundles/core/_load.py index 8eb5ebe5d0..e643dc4b1b 100644 --- a/experimental/python/databricks/bundles/core/_load.py +++ b/experimental/python/databricks/bundles/core/_load.py @@ -7,7 +7,7 @@ from types import ModuleType from typing import Any, Iterable -from databricks.bundles.core import Diagnostics +from databricks.bundles.core._diagnostics import Diagnostics from databricks.bundles.core._location import Location from databricks.bundles.core._resource import Resource from databricks.bundles.core._resources import Resources diff --git a/experimental/python/databricks_tests/core/test_import.py b/experimental/python/databricks_tests/core/test_import.py new file mode 100644 index 0000000000..f584eaa5fd --- /dev/null +++ b/experimental/python/databricks_tests/core/test_import.py @@ -0,0 +1,61 @@ +import sys +from types import ModuleType + +import pytest + + +@pytest.fixture(scope="function") +def reset_sys_modules(): + """ + Reset sys.module before test and restore it after test. + """ + + original_modules = sys.modules.copy() + try: + import databricks.bundles + + for module in original_modules.keys(): + if is_subpackage(module, databricks.bundles): + del sys.modules[module] + + del sys.modules[databricks.bundles.__package__] + + yield + finally: + # Clear any modifications and restore the original modules + sys.modules.clear() + sys.modules.update(original_modules) + + +def test_import_regression(reset_sys_modules): + """ + We don't want importing databricks.bundles.core to import any of resources. + + We refer to resources in type signatures (e.g. in Resources or @job_mutator), + unless we use forward references (e.g. "Job" instead of Job) we can create a + cycling dependency resulting in a runtime error. + + We want every package with resources (e.g., databricks.bundles.jobs) to be loaded + only when needed. + """ + + import databricks.bundles.core + + assert databricks.bundles.core + + import databricks.bundles + + for module in sys.modules.keys(): + if is_subpackage(module, databricks.bundles): + is_core = module == databricks.bundles.core.__package__ + is_core_subpackage = is_subpackage(module, databricks.bundles.core) + + assert is_core or is_core_subpackage, ( + f"Unexpected loaded module AFTER loading core: {module}" + ) + + +def is_subpackage(module: str, parent: ModuleType) -> bool: + assert parent.__package__ + + return module.startswith(parent.__package__ + ".") From b111e4b6a179caa7417057a7dcbf2f91af77b539 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Thu, 27 Mar 2025 16:59:20 +0100 Subject: [PATCH 2/2] Fix lint --- experimental/python/databricks_tests/core/test_import.py | 1 + 1 file changed, 1 insertion(+) diff --git a/experimental/python/databricks_tests/core/test_import.py b/experimental/python/databricks_tests/core/test_import.py index f584eaa5fd..732c52a7cb 100644 --- a/experimental/python/databricks_tests/core/test_import.py +++ b/experimental/python/databricks_tests/core/test_import.py @@ -18,6 +18,7 @@ def reset_sys_modules(): if is_subpackage(module, databricks.bundles): del sys.modules[module] + assert databricks.bundles.__package__ del sys.modules[databricks.bundles.__package__] yield