feat: enhance SessionContext to support automatic registration of Python objects via session config

Author: kosiew

docs/source/user-guide/dataframe/index.rst

@@ -228,10 +228,10 @@ Core Classes
     * :py:meth:`~datafusion.SessionContext.from_pandas` - Create from Pandas DataFrame
     * :py:meth:`~datafusion.SessionContext.from_arrow` - Create from Arrow data
 
-    ``SessionContext`` automatically resolves SQL table names that match
-    in-scope Python data objects. When ``auto_register_python_objects`` is
-    enabled (the default), a query such as ``ctx.sql("SELECT * FROM pdf")``
-    will register a pandas or PyArrow object named ``pdf`` without calling
+    ``SessionContext`` can automatically resolve SQL table names that match
+    in-scope Python data objects. When automatic lookup is enabled, a query
+    such as ``ctx.sql("SELECT * FROM pdf")`` will register a pandas or
+    PyArrow object named ``pdf`` without calling
     :py:meth:`~datafusion.SessionContext.from_pandas` or
     :py:meth:`~datafusion.SessionContext.from_arrow` explicitly. This requires
     the corresponding library (``pandas`` for pandas objects, ``pyarrow`` for
@@ -242,16 +242,18 @@ Core Classes
         import pandas as pd
         from datafusion import SessionContext
 
-        ctx = SessionContext()
+        ctx = SessionContext(auto_register_python_objects=True)
         pdf = pd.DataFrame({"value": [1, 2, 3]})
 
         df = ctx.sql("SELECT SUM(value) AS total FROM pdf")
         print(df.to_pandas())  # automatically registers `pdf`
 
-    To opt out, either pass ``auto_register_python_objects=False`` when
-    constructing the session, or call
-    :py:meth:`~datafusion.SessionContext.set_python_table_lookup` with
-    ``False`` to require explicit registration.
+    Automatic lookup is disabled by default. Enable it by passing
+    ``auto_register_python_objects=True`` when constructing the session or by
+    configuring :py:class:`~datafusion.SessionConfig` with
+    :py:meth:`~datafusion.SessionConfig.with_python_table_lookup`. Use
+    :py:meth:`~datafusion.SessionContext.set_python_table_lookup` to toggle the
+    behaviour at runtime.
 
     See: :py:class:`datafusion.SessionContext`
 

python/datafusion/context.py

@@ -524,7 +524,10 @@ def __init__(
             runtime: Runtime configuration options.
             auto_register_python_objects: Automatically register referenced
                 Python objects (such as pandas or PyArrow data) when ``sql``
-                queries reference them by name.
+                queries reference them by name. When omitted, this defaults to
+                the value configured via
+                :py:meth:`~datafusion.SessionConfig.with_python_table_lookup`
+                (``False`` unless explicitly enabled).
             auto_register_python_variables: Deprecated alias for
                 ``auto_register_python_objects``. When provided, it overrides
                 the automatic registration behavior.
@@ -543,33 +546,34 @@ def __init__(
             config.config_internal if config is not None else None,
             runtime.config_internal if runtime is not None else None,
         )
+
         if auto_register_python_variables is not None:
             warnings.warn(
                 _AUTO_REGISTER_PYTHON_VARIABLES_DEPRECATED,
                 DeprecationWarning,
                 stacklevel=2,
             )
 
-        if (
-            auto_register_python_objects is not None
-            and auto_register_python_variables is not None
-            and auto_register_python_objects != auto_register_python_variables
-        ):
-            conflict_message = (
-                "auto_register_python_objects and auto_register_python_variables "
-                "were provided with conflicting values."
-            )
-            raise ValueError(conflict_message)
+        if auto_register_python_variables is not None and auto_register_python_objects is not None:
+            if auto_register_python_objects != auto_register_python_variables:
+                conflict_message = (
+                    "auto_register_python_objects and auto_register_python_variables "
+                    "were provided with conflicting values."
+                )
+                raise ValueError(conflict_message)
 
-        if auto_register_python_objects is None:
-            if auto_register_python_variables is None:
-                auto_python_table_lookup = True
-            else:
-                auto_python_table_lookup = auto_register_python_variables
-        else:
+        # Determine the final value for python table lookup
+        if auto_register_python_objects is not None:
             auto_python_table_lookup = auto_register_python_objects
+        elif auto_register_python_variables is not None:
+            auto_python_table_lookup = auto_register_python_variables
+        else:
+            # Default to session config value or False if not configured
+            auto_python_table_lookup = getattr(
+                config, "_python_table_lookup", False
+            )
 
-        self._auto_python_table_lookup = auto_python_table_lookup
+        self._auto_python_table_lookup = bool(auto_python_table_lookup)
 
     def __repr__(self) -> str:
         """Print a string representation of the Session Context."""
@@ -597,18 +601,18 @@ def enable_url_table(self) -> SessionContext:
         obj = klass.__new__(klass)
         obj.ctx = self.ctx.enable_url_table()
         obj._auto_python_table_lookup = getattr(
-            self, "_auto_python_table_lookup", True
+            self, "_auto_python_table_lookup", False
         )
         return obj
 
     def set_python_table_lookup(self, enabled: bool = True) -> SessionContext:
         """Enable or disable automatic registration of Python objects in SQL.
 
         Args:
-            enabled: When ``True`` (default), SQL queries automatically attempt
-                to resolve missing table names by looking up Python objects in
-                the caller's scope. When ``False``, missing tables will raise an
-                error unless they have been explicitly registered.
+            enabled: When ``True``, SQL queries automatically attempt to
+                resolve missing table names by looking up Python objects in the
+                caller's scope. Use ``False`` to require explicit registration
+                of any referenced tables.
 
         Returns:
             The current :py:class:`SessionContext` instance for chaining.
@@ -624,7 +628,7 @@ def auto_register_python_variables(self) -> bool:
             DeprecationWarning,
             stacklevel=2,
         )
-        return getattr(self, "_auto_python_table_lookup", True)
+        return bool(getattr(self, "_auto_python_table_lookup", False))
 
     @auto_register_python_variables.setter
     def auto_register_python_variables(self, enabled: bool) -> None:
@@ -633,7 +637,7 @@ def auto_register_python_variables(self, enabled: bool) -> None:
             DeprecationWarning,
             stacklevel=2,
         )
-        self.set_python_table_lookup(enabled)
+        self.set_python_table_lookup(bool(enabled))
 
     def register_object_store(
         self, schema: str, store: Any, host: str | None = None
@@ -709,7 +713,7 @@ def _execute_sql() -> DataFrame:
         try:
             return _execute_sql()
         except Exception as err:
-            if not getattr(self, "_auto_python_table_lookup", True):
+            if not getattr(self, "_auto_python_table_lookup", False):
                 raise
 
             missing_tables = self._extract_missing_table_names(err)

python/tests/test_context.py

@@ -772,10 +772,10 @@ def test_sql_with_options_no_statements(ctx):
         ctx.sql_with_options(sql, options=options)
 
 
-def test_sql_auto_register_pandas():
+def test_session_config_python_table_lookup_enables_auto_registration():
     pd = pytest.importorskip("pandas")
 
-    ctx = SessionContext()
+    ctx = SessionContext(config=SessionConfig().with_python_table_lookup(True))
     pdf = pd.DataFrame({"value": [1, 2, 3]})
     assert len(pdf) == 3
 
@@ -784,7 +784,7 @@ def test_sql_auto_register_pandas():
 
 
 def test_sql_auto_register_arrow():
-    ctx = SessionContext()
+    ctx = SessionContext(auto_register_python_objects=True)
     arrow_table = pa.table({"value": [1, 2, 3, 4]})
     assert arrow_table.num_rows == 4
 
@@ -795,7 +795,7 @@ def test_sql_auto_register_arrow():
 def test_sql_auto_register_disabled():
     pd = pytest.importorskip("pandas")
 
-    ctx = SessionContext(auto_register_python_objects=False)
+    ctx = SessionContext()
     pdf = pd.DataFrame({"value": [1, 2, 3]})
     assert len(pdf) == 3