diff --git a/pyproject.toml b/pyproject.toml index 0a23b0a3f..a6f38afb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,3 +23,6 @@ repair-wheel-command = "" [tool.cibuildwheel.windows] archs = ["AMD64"] + +[tool.check-manifest] +ignore = ["src/snowflake/connector/minicore/*"] diff --git a/setup.cfg b/setup.cfg index 216a3939d..2f349b9bf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,6 +72,12 @@ where = src exclude = snowflake.connector.cpp* include = snowflake.* +[options.package_data] +snowflake.connector.minicore = + *.so + *.dll + *.dylib + [options.entry_points] console_scripts = snowflake-dump-ocsp-response = snowflake.connector.tool.dump_ocsp_response:main diff --git a/src/snowflake/connector/__init__.py b/src/snowflake/connector/__init__.py index 41b5288ac..e23ee0b12 100644 --- a/src/snowflake/connector/__init__.py +++ b/src/snowflake/connector/__init__.py @@ -5,6 +5,8 @@ from functools import wraps +from ._utils import _core_loader + apilevel = "2.0" threadsafety = 2 paramstyle = "pyformat" @@ -45,6 +47,14 @@ from .log_configuration import EasyLoggingConfigPython from .version import VERSION +# Load the core library - failures are captured in core_loader and don't prevent module loading +try: + _core_loader.load() +except Exception: + # Silently continue if core loading fails - the error is already captured in core_loader + # This ensures the connector module loads even if the minicore library is unavailable + pass + logging.getLogger(__name__).addHandler(NullHandler()) setup_external_libraries() diff --git a/src/snowflake/connector/_utils.py b/src/snowflake/connector/_utils.py index dbdd2bc57..c95d7d174 100644 --- a/src/snowflake/connector/_utils.py +++ b/src/snowflake/connector/_utils.py @@ -1,6 +1,11 @@ from __future__ import annotations +import ctypes +import importlib +import os import string +import sys +import threading from enum import Enum from inspect import stack from random import choice @@ -96,3 +101,74 @@ def get_application_path() -> str: return outermost_frame.filename except Exception: return "unknown" + + +class _CoreLoader: + def __init__(self): + self._version: bytes | None = None + self._error: Exception | None = None + + @staticmethod + def _get_core_path(): + # Define the file name for each platform + if sys.platform.startswith("win"): + lib_name = "libsf_mini_core.dll" + elif sys.platform.startswith("darwin"): + lib_name = "libsf_mini_core.dylib" + else: + lib_name = "libsf_mini_core.so" + + files = importlib.resources.files("snowflake.connector.minicore") + return files.joinpath(lib_name) + + @staticmethod + def _register_functions(core: ctypes.CDLL): + core.sf_core_full_version.argtypes = [] + core.sf_core_full_version.restype = ctypes.c_char_p + + @staticmethod + def _load_minicore(path: str) -> ctypes.CDLL: + # This context manager is the safe way to get a + # file path from importlib.resources. It handles cases + # where the file is inside a zip and needs to be extracted + # to a temporary location. + with importlib.resources.as_file(path) as lib_path: + core = ctypes.CDLL(str(lib_path)) + return core + + def _is_core_disabled(self) -> bool: + value = str(os.getenv("SNOWFLAKE_DISABLE_MINICORE", None)).lower() + return value in ["1", "true"] + + def _load(self) -> None: + try: + path = self._get_core_path() + core = self._load_minicore(path) + self._register_functions(core) + self._version = core.sf_core_full_version() + self._error = None + except Exception as err: + self._error = err + + def load(self): + """Spawn a separate thread to load the minicore library (non-blocking).""" + if self._is_core_disabled(): + self._error = "mini-core-disabled" + return + self._error = "still-loading" + thread = threading.Thread(target=self._load, daemon=True) + thread.start() + + def get_load_error(self) -> str: + return str(self._error) + + def get_core_version(self) -> str | None: + if self._version: + try: + return self._version.decode("utf-8") + except Exception: + pass + return None + + +_core_loader = _CoreLoader() diff --git a/src/snowflake/connector/connection.py b/src/snowflake/connector/connection.py index 06c829982..f8101ea77 100644 --- a/src/snowflake/connector/connection.py +++ b/src/snowflake/connector/connection.py @@ -42,6 +42,7 @@ from ._utils import ( _DEFAULT_VALUE_SERVER_DOP_CAP_FOR_FILE_TRANSFER, _VARIABLE_NAME_SERVER_DOP_CAP_FOR_FILE_TRANSFER, + _core_loader, ) from .auth import ( FIRST_PARTY_AUTHENTICATORS, @@ -677,6 +678,7 @@ def __init__( # get the imported modules from sys.modules self._log_telemetry_imported_packages() + self._log_minicore_import() # check SNOW-1218851 for long term improvement plan to refactor ocsp code atexit.register(self._close_at_exit) @@ -2488,6 +2490,22 @@ def async_query_check_helper( return not found_unfinished_query + def _log_minicore_import(self): + ts = get_time_millis() + self._log_telemetry( + TelemetryData.from_telemetry_data_dict( + from_dict={ + TelemetryField.KEY_TYPE.value: TelemetryField.CORE_IMPORT.value, + TelemetryField.KEY_VALUE: { + "CORE_LOAD_ERROR": _core_loader.get_load_error(), + "CORE_VERSION": _core_loader.get_core_version(), + }, + }, + timestamp=ts, + connection=self, + ) + ) + def _log_telemetry_imported_packages(self) -> None: if self._log_imported_packages_in_telemetry: # filter out duplicates caused by submodules diff --git a/src/snowflake/connector/minicore/__init__.py b/src/snowflake/connector/minicore/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/snowflake/connector/minicore/sf_mini_core.h b/src/snowflake/connector/minicore/sf_mini_core.h new file mode 100644 index 000000000..715bf1f79 --- /dev/null +++ b/src/snowflake/connector/minicore/sf_mini_core.h @@ -0,0 +1,54 @@ +/** + * @file sf_mini_core.h + * @brief C API for sf_mini_core library + * + * This header file provides the C API for the sf_mini_core Rust library. + * This file is auto-generated by cbindgen. Do not edit manually! + */ + +#ifndef SF_MINI_CORE_H +#define SF_MINI_CORE_H + +/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. + */ + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +/** + * Returns the full version string for sf_core. + * + * This function returns a pointer to a static null-terminated string + * containing the version of sf_core. + * + * @return A pointer to a static string containing the version. + * The caller must NOT free this pointer. + * The returned string is valid for the lifetime of the program. + * + * @note Thread-safe: Yes + * @note This function never returns NULL + * + * Example usage: + * @code + * const char* version = sf_core_full_version(); + * printf("Version: %s\n", version); + * @endcode + * + * # Safety + * + * The returned pointer points to a static string that is valid for the lifetime + * of the program. The caller must not free the returned pointer. + */ +const char *sf_core_full_version(void); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif /* SF_MINI_CORE_H */ diff --git a/src/snowflake/connector/telemetry.py b/src/snowflake/connector/telemetry.py index 37edd3fd4..380e5c6de 100644 --- a/src/snowflake/connector/telemetry.py +++ b/src/snowflake/connector/telemetry.py @@ -39,6 +39,8 @@ class TelemetryField(Enum): PANDAS_WRITE = "client_write_pandas" # imported packages along with client IMPORTED_PACKAGES = "client_imported_packages" + # Core import + CORE_IMPORT = "mini_core_import" # multi-statement usage MULTI_STATEMENT = "client_multi_statement_query" # Keys for telemetry data sent through either in-band or out-of-band telemetry diff --git a/test/integ/test_connection.py b/test/integ/test_connection.py index b5d490d34..5fa7c9c79 100644 --- a/test/integ/test_connection.py +++ b/test/integ/test_connection.py @@ -1230,6 +1230,36 @@ def check_packages(message: str, expected_packages: list[str]) -> bool: assert len(telemetry_test.records) == 0 +@pytest.mark.skipolddriver +def test_minicore_import_telemetry(conn_cnx, capture_sf_telemetry): + """Test that minicore import telemetry (CORE_LOAD_ERROR and CORE_VERSION) is logged.""" + with ( + conn_cnx() as conn, + capture_sf_telemetry.patch_connection(conn, False) as telemetry_test, + ): + conn._log_minicore_import() + assert len(telemetry_test.records) > 0 + # Check that the telemetry record contains the proper structure + found_minicore_telemetry = False + for t in telemetry_test.records: + if ( + t.message.get(TelemetryField.KEY_TYPE.value) + == TelemetryField.CORE_IMPORT.value + and TelemetryField.KEY_VALUE.value in t.message + ): + found_minicore_telemetry = True + # Verify that the value contains CORE_LOAD_ERROR and CORE_VERSION + value = t.message[TelemetryField.KEY_VALUE.value] + assert ( + "CORE_LOAD_ERROR" in value + ), "CORE_LOAD_ERROR not in telemetry value" + assert "CORE_VERSION" in value, "CORE_VERSION not in telemetry value" + break + assert ( + found_minicore_telemetry + ), "Minicore telemetry not found in telemetry records" + + @pytest.mark.skipolddriver def test_disable_query_context_cache(conn_cnx) -> None: with conn_cnx(disable_query_context_cache=True) as conn: diff --git a/test/unit/test_util.py b/test/unit/test_util.py index b2862f466..60774191d 100644 --- a/test/unit/test_util.py +++ b/test/unit/test_util.py @@ -1,7 +1,13 @@ +import ctypes +import os +import sys +from importlib import reload +from unittest import mock + import pytest try: - from snowflake.connector._utils import _TrackedQueryCancellationTimer + from snowflake.connector._utils import _CoreLoader, _TrackedQueryCancellationTimer except ImportError: pass @@ -18,3 +24,327 @@ def test_timer(): timer.start() timer.cancel() assert not timer.executed + + +class TestCoreLoader: + """Tests for the _CoreLoader class.""" + + def test_e2e(self): + loader = _CoreLoader() + loader.load() + assert loader.get_load_error() == str(None) + assert loader.get_core_version() == "0.0.1" + + def test_core_loader_initialization(self): + """Test that _CoreLoader initializes with None values.""" + loader = _CoreLoader() + assert loader._version is None + assert loader._error is None + + def test_get_core_path_windows(self): + """Test _get_core_path returns correct path for Windows.""" + with mock.patch.object(sys, "platform", "win32"): + with mock.patch("importlib.resources.files") as mock_files: + mock_files_obj = mock.MagicMock() + mock_files.return_value = mock_files_obj + + _CoreLoader._get_core_path() + + mock_files.assert_called_once_with("snowflake.connector.minicore") + mock_files_obj.joinpath.assert_called_once_with("libsf_mini_core.dll") + + def test_get_core_path_darwin(self): + """Test _get_core_path returns correct path for macOS.""" + with mock.patch.object(sys, "platform", "darwin"): + with mock.patch("importlib.resources.files") as mock_files: + mock_files_obj = mock.MagicMock() + mock_files.return_value = mock_files_obj + + _CoreLoader._get_core_path() + + mock_files.assert_called_once_with("snowflake.connector.minicore") + mock_files_obj.joinpath.assert_called_once_with("libsf_mini_core.dylib") + + def test_get_core_path_linux(self): + """Test _get_core_path returns correct path for Linux.""" + with mock.patch.object(sys, "platform", "linux"): + with mock.patch("importlib.resources.files") as mock_files: + mock_files_obj = mock.MagicMock() + mock_files.return_value = mock_files_obj + + _CoreLoader._get_core_path() + + mock_files.assert_called_once_with("snowflake.connector.minicore") + mock_files_obj.joinpath.assert_called_once_with("libsf_mini_core.so") + + def test_register_functions(self): + """Test that _register_functions sets up the C library functions correctly.""" + mock_core = mock.MagicMock() + mock_core.sf_core_full_version = mock.MagicMock() + + _CoreLoader._register_functions(mock_core) + + # Verify the function signature was configured + assert mock_core.sf_core_full_version.argtypes == [] + assert mock_core.sf_core_full_version.restype == ctypes.c_char_p + + def test_load_minicore(self): + """Test that _load_minicore loads the library.""" + mock_path = mock.MagicMock() + mock_lib_path = "/path/to/libsf_mini_core.so" + + with mock.patch("importlib.resources.as_file") as mock_as_file: + with mock.patch("ctypes.CDLL") as mock_cdll: + # Setup the context manager + mock_as_file.return_value.__enter__ = mock.Mock( + return_value=mock_lib_path + ) + mock_as_file.return_value.__exit__ = mock.Mock(return_value=False) + + mock_core = mock.MagicMock() + mock_cdll.return_value = mock_core + + result = _CoreLoader._load_minicore(mock_path) + + mock_as_file.assert_called_once_with(mock_path) + mock_cdll.assert_called_once_with(str(mock_lib_path)) + assert result == mock_core + + @pytest.mark.parametrize("env_value", ["1", "true", "True", "TRUE"]) + def test_is_core_disabled_returns_true(self, env_value): + """Test that _is_core_disabled returns True when env var is '1' or 'true' (case-insensitive).""" + loader = _CoreLoader() + with mock.patch.dict(os.environ, {"SNOWFLAKE_DISABLE_MINICORE": env_value}): + assert loader._is_core_disabled() is True + + @pytest.mark.parametrize("env_value", ["0", "false", "False", "no", "other", ""]) + def test_is_core_disabled_returns_false(self, env_value): + """Test that _is_core_disabled returns False for other values.""" + loader = _CoreLoader() + with mock.patch.dict(os.environ, {"SNOWFLAKE_DISABLE_MINICORE": env_value}): + assert loader._is_core_disabled() is False + + def test_is_core_disabled_returns_false_when_not_set(self): + """Test that _is_core_disabled returns False when env var is not set.""" + loader = _CoreLoader() + with mock.patch.dict(os.environ, {}, clear=True): + # Ensure the env var is not set + os.environ.pop("SNOWFLAKE_DISABLE_CORE", None) + assert loader._is_core_disabled() is False + + def test_load_skips_loading_when_core_disabled(self): + """Test that load() returns early when core is disabled.""" + loader = _CoreLoader() + + with mock.patch.dict(os.environ, {"SNOWFLAKE_DISABLE_MINICORE": "1"}): + with mock.patch.object(loader, "_get_core_path") as mock_get_path: + loader.load() + + # Verify that _get_core_path was never called (loading was skipped) + mock_get_path.assert_not_called() + # Verify the error message is set correctly + assert loader._error == "mini-core-disabled" + assert loader._version is None + + def test_load_success(self): + """Test successful load of the core library.""" + loader = _CoreLoader() + mock_path = mock.MagicMock() + mock_core = mock.MagicMock() + mock_version = b"1.2.3" + mock_core.sf_core_full_version = mock.MagicMock(return_value=mock_version) + + with mock.patch.object(loader, "_is_core_disabled", return_value=False): + with mock.patch.object( + loader, "_get_core_path", return_value=mock_path + ) as mock_get_path: + with mock.patch.object( + loader, "_load_minicore", return_value=mock_core + ) as mock_load: + with mock.patch.object( + loader, "_register_functions" + ) as mock_register: + loader.load() + + mock_get_path.assert_called_once() + mock_load.assert_called_once_with(mock_path) + mock_register.assert_called_once_with(mock_core) + assert loader._version == mock_version + assert loader._error is None + + def test_load_failure(self): + """Test that load captures exceptions.""" + loader = _CoreLoader() + test_error = Exception("Test error loading core") + + with mock.patch.object(loader, "_is_core_disabled", return_value=False): + with mock.patch.object( + loader, "_get_core_path", side_effect=test_error + ) as mock_get_path: + loader.load() + + mock_get_path.assert_called_once() + assert loader._version is None + assert loader._error == test_error + + def test_get_load_error_with_error(self): + """Test get_load_error returns error message when error exists.""" + loader = _CoreLoader() + test_error = Exception("Test error message") + loader._error = test_error + + result = loader.get_load_error() + + assert result == "Test error message" + + def test_get_load_error_no_error(self): + """Test get_load_error returns 'None' string when no error exists.""" + loader = _CoreLoader() + + result = loader.get_load_error() + + assert result == "None" + + def test_get_core_version_with_version(self): + """Test get_core_version returns decoded version string.""" + loader = _CoreLoader() + loader._version = b"1.2.3-beta" + + result = loader.get_core_version() + + assert result == "1.2.3-beta" + + def test_get_core_version_no_version(self): + """Test get_core_version returns None when no version exists.""" + loader = _CoreLoader() + + result = loader.get_core_version() + + assert result is None + + +def test_importing_snowflake_connector_triggers_core_loader_load(): + """Test that importing snowflake.connector triggers core_loader.load().""" + # We need to test that when snowflake.connector is imported, + # core_loader.load() is called. Since snowflake.connector is already imported, + # we need to reload it and mock the load method. + + with mock.patch("snowflake.connector._utils._core_loader.load") as mock_load: + # Reload the connector module to trigger the __init__.py code again + import snowflake.connector + + reload(snowflake.connector) + + # Verify that load was called during import + mock_load.assert_called_once() + + +def test_snowflake_connector_loads_when_core_loader_fails(): + """Test that snowflake.connector loads successfully even if core_loader.load() fails.""" + # Mock core_loader.load() to raise an exception + with mock.patch( + "snowflake.connector._utils._core_loader.load", + side_effect=Exception("Simulated core loading failure"), + ): + import snowflake.connector + + # Reload the connector module - this should NOT raise an exception + try: + reload(snowflake.connector) + # If we reach here, the module loaded successfully despite core_loader.load() failing + assert True + except Exception as e: + pytest.fail( + f"snowflake.connector failed to load when core_loader.load() raised an exception: {e}" + ) + + # Verify the module has expected attributes + assert hasattr(snowflake.connector, "connect") + assert hasattr(snowflake.connector, "SnowflakeConnection") + assert hasattr(snowflake.connector, "Connect") + + +def test_snowflake_connector_usable_when_core_loader_fails(): + """Test that snowflake.connector remains usable even if core_loader.load() fails.""" + # Mock core_loader.load() to raise an exception + with mock.patch( + "snowflake.connector._utils._core_loader.load", + side_effect=RuntimeError("Core library not found"), + ): + import snowflake.connector + + # Reload the connector module + reload(snowflake.connector) + + # Verify we can access key classes and functions + assert snowflake.connector.SnowflakeConnection is not None + assert callable(snowflake.connector.connect) + assert callable(snowflake.connector.Connect) + + # Verify error classes are available + assert hasattr(snowflake.connector, "Error") + assert hasattr(snowflake.connector, "DatabaseError") + assert hasattr(snowflake.connector, "ProgrammingError") + + # Verify DBAPI constants are available + assert hasattr(snowflake.connector, "apilevel") + assert hasattr(snowflake.connector, "threadsafety") + assert hasattr(snowflake.connector, "paramstyle") + + +def test_core_loader_error_captured_when_load_fails(): + """Test that errors from core_loader.load() are captured in the loader's error attribute.""" + loader = _CoreLoader() + test_exception = FileNotFoundError("Library file not found") + + # Mock _get_core_path to raise an exception + with mock.patch.object(loader, "_is_core_disabled", return_value=False): + with mock.patch.object(loader, "_get_core_path", side_effect=test_exception): + # Call load - it should NOT raise an exception + loader.load() + + # Verify the error was captured + assert loader._error is test_exception + assert loader._version is None + assert loader.get_load_error() == "Library file not found" + assert loader.get_core_version() is None + + +def test_core_loader_fails_gracefully_on_missing_library(): + """Test that core_loader handles missing library files gracefully.""" + loader = _CoreLoader() + + # Mock importlib.resources.files to simulate missing library + with mock.patch.object(loader, "_is_core_disabled", return_value=False): + with mock.patch("importlib.resources.files") as mock_files: + mock_files.side_effect = FileNotFoundError("minicore module not found") + + # Call load - it should NOT raise an exception + loader.load() + + # Verify the error was captured + assert loader._error is not None + assert loader._version is None + assert "minicore module not found" in loader.get_load_error() + + +def test_core_loader_fails_gracefully_on_incompatible_library(): + """Test that core_loader handles incompatible library files gracefully.""" + loader = _CoreLoader() + mock_path = mock.MagicMock() + + # Mock the loading to simulate incompatible library (OSError is common for this) + with mock.patch.object(loader, "_is_core_disabled", return_value=False): + with mock.patch.object(loader, "_get_core_path", return_value=mock_path): + with mock.patch.object( + loader, + "_load_minicore", + side_effect=OSError("incompatible library version"), + ): + # Call load - it should NOT raise an exception + loader.load() + + # Verify the error was captured + assert loader._error is not None + assert loader._version is None + assert "incompatible library version" in loader.get_load_error()