Merge branch 'main' into migrate-coalesce-op-to-sqlglot

Author: jialuoo

CHANGELOG.md

@@ -4,6 +4,29 @@
 
 [1]: https://pypi.org/project/bigframes/#history
 
+## [2.28.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.27.0...v2.28.0) (2025-11-03)
+
+
+### Features
+
+* Add bigframes.bigquery.st_simplify ([#2210](https://github.com/googleapis/python-bigquery-dataframes/issues/2210)) ([ecee2bc](https://github.com/googleapis/python-bigquery-dataframes/commit/ecee2bc6ada0bc968fc56ed7194dc8c043547e93))
+* Add Series.dt.day_name ([#2218](https://github.com/googleapis/python-bigquery-dataframes/issues/2218)) ([5e006e4](https://github.com/googleapis/python-bigquery-dataframes/commit/5e006e404b65c32e5b1d342ebfcfce59ee592c8c))
+* Polars engine supports std, var ([#2215](https://github.com/googleapis/python-bigquery-dataframes/issues/2215)) ([ef5e83a](https://github.com/googleapis/python-bigquery-dataframes/commit/ef5e83acedf005cbe1e6ad174bec523ac50517d7))
+* Support INFORMATION_SCHEMA views in `read_gbq` ([#1895](https://github.com/googleapis/python-bigquery-dataframes/issues/1895)) ([d97cafc](https://github.com/googleapis/python-bigquery-dataframes/commit/d97cafcb5921fca2351b18011b0e54e2631cc53d))
+* Support some python standard lib callables in apply/combine ([#2187](https://github.com/googleapis/python-bigquery-dataframes/issues/2187)) ([86a2756](https://github.com/googleapis/python-bigquery-dataframes/commit/86a27564b48b854a32b3d11cd2105aa0fa496279))
+
+
+### Bug Fixes
+
+* Correct connection normalization in blob system tests ([#2222](https://github.com/googleapis/python-bigquery-dataframes/issues/2222)) ([a0e1e50](https://github.com/googleapis/python-bigquery-dataframes/commit/a0e1e50e47c758bdceb54d04180ed36b35cf2e35))
+* Improve error handling in blob operations ([#2194](https://github.com/googleapis/python-bigquery-dataframes/issues/2194)) ([d410046](https://github.com/googleapis/python-bigquery-dataframes/commit/d4100466612df0523d01ed01ca1e115dabd6ef45))
+* Resolve AttributeError in TableWidget and improve initialization ([#1937](https://github.com/googleapis/python-bigquery-dataframes/issues/1937)) ([4c4c9b1](https://github.com/googleapis/python-bigquery-dataframes/commit/4c4c9b14657b7cda1940ef39e7d4db20a9ff5308))
+
+
+### Documentation
+
+* Update bq_dataframes_llm_output_schema.ipynb ([#2004](https://github.com/googleapis/python-bigquery-dataframes/issues/2004)) ([316ba9f](https://github.com/googleapis/python-bigquery-dataframes/commit/316ba9f557d792117d5a7845d7567498f78dd513))
+
 ## [2.27.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.26.0...v2.27.0) (2025-10-24)
 
 

bigframes/core/blocks.py

@@ -68,6 +68,7 @@
 import bigframes.operations.aggregations as agg_ops
 from bigframes.session import dry_runs, execution_spec
 from bigframes.session import executor as executors
+from bigframes.session._io import pandas as io_pandas
 
 # Type constraint for wherever column labels are used
 Label = typing.Hashable
@@ -711,12 +712,15 @@ def to_pandas_batches(
         # To reduce the number of edge cases to consider when working with the
         # results of this, always return at least one DataFrame. See:
         # b/428918844.
-        empty_val = pd.DataFrame(
-            {
-                col: pd.Series([], dtype=self.expr.get_column_type(col))
-                for col in itertools.chain(self.value_columns, self.index_columns)
-            }
-        )
+        try:
+            empty_arrow_table = self.expr.schema.to_pyarrow().empty_table()
+        except pa.ArrowNotImplementedError:
+            # Bug with some pyarrow versions(https://github.com/apache/arrow/issues/45262),
+            # empty_table only supports base storage types, not extension types.
+            empty_arrow_table = self.expr.schema.to_pyarrow(
+                use_storage_types=True
+            ).empty_table()
+        empty_val = io_pandas.arrow_to_pandas(empty_arrow_table, self.expr.schema)
         dfs = map(
             lambda a: a[0],
             itertools.zip_longest(

bigframes/core/nodes.py

@@ -1627,7 +1627,7 @@ class ResultNode(UnaryNode):
     # TODO: CTE definitions
 
     def _validate(self):
-        for ref, name in self.output_cols:
+        for ref, _ in self.output_cols:
             assert ref.id in self.child.ids
 
     @property

bigframes/core/rewrite/identifiers.py

@@ -57,8 +57,10 @@ def remap_variables(
     new_root = root.transform_children(lambda node: remapped_children[node])
 
     # Step 3: Transform the current node using the mappings from its children.
+    # "reversed" is required for InNode so that in case of a duplicate column ID,
+    # the left child's mapping is the one that's kept.
     downstream_mappings: dict[identifiers.ColumnId, identifiers.ColumnId] = {
-        k: v for mapping in new_child_mappings for k, v in mapping.items()
+        k: v for mapping in reversed(new_child_mappings) for k, v in mapping.items()
     }
     if isinstance(new_root, nodes.InNode):
         new_root = typing.cast(nodes.InNode, new_root)

bigframes/functions/function_typing.py

@@ -60,8 +60,22 @@ class UnsupportedTypeError(ValueError):
     def __init__(self, type_, supported_types):
         self.type = type_
         self.supported_types = supported_types
+
+        types_to_format = supported_types
+        if isinstance(supported_types, dict):
+            types_to_format = supported_types.keys()
+
+        supported_types_str = ", ".join(
+            sorted(
+                [
+                    getattr(supported, "__name__", supported)
+                    for supported in types_to_format
+                ]
+            )
+        )
+
         super().__init__(
-            f"'{type_}' must be one of the supported types ({supported_types}) "
+            f"'{getattr(type_, '__name__', type_)}' must be one of the supported types ({supported_types_str}) "
             "or a list of one of those types."
         )
 

bigframes/session/loader.py

@@ -45,7 +45,6 @@
 import google.cloud.bigquery.table
 from google.cloud.bigquery_storage_v1 import types as bq_storage_types
 import pandas
-import pyarrow as pa
 
 import bigframes._tools
 import bigframes._tools.strings
@@ -1307,22 +1306,6 @@ def _transform_read_gbq_configuration(configuration: Optional[dict]) -> dict:
     return configuration
 
 
-def _has_json_arrow_type(arrow_type: pa.DataType) -> bool:
-    """
-    Searches recursively for JSON array type within a PyArrow DataType.
-    """
-    if arrow_type == bigframes.dtypes.JSON_ARROW_TYPE:
-        return True
-    if pa.types.is_list(arrow_type):
-        return _has_json_arrow_type(arrow_type.value_type)
-    if pa.types.is_struct(arrow_type):
-        for i in range(arrow_type.num_fields):
-            if _has_json_arrow_type(arrow_type.field(i).type):
-                return True
-        return False
-    return False
-
-
 def _validate_dtype_can_load(name: str, column_type: bigframes.dtypes.Dtype):
     """
     Determines whether a datatype is supported by bq load jobs.
@@ -1339,7 +1322,9 @@ def _validate_dtype_can_load(name: str, column_type: bigframes.dtypes.Dtype):
     if column_type == bigframes.dtypes.JSON_DTYPE:
         return
 
-    if isinstance(column_type, pandas.ArrowDtype) and _has_json_arrow_type(
+    if isinstance(
+        column_type, pandas.ArrowDtype
+    ) and bigframes.dtypes.contains_db_dtypes_json_arrow_type(
         column_type.pyarrow_dtype
     ):
         raise NotImplementedError(

bigframes/version.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "2.27.0"
+__version__ = "2.28.0"
 
 # {x-release-please-start-date}
-__release_date__ = "2025-10-24"
+__release_date__ = "2025-11-03"
 # {x-release-please-end}

tests/system/conftest.py

@@ -70,6 +70,23 @@ def _hash_digest_file(hasher, filepath):
             hasher.update(chunk)
 
 
+@pytest.fixture(scope="session")
+def normalize_connection_id():
+    """Normalizes the connection ID by casefolding only the LOCATION component.
+
+    Connection format: PROJECT.LOCATION.CONNECTION_NAME
+    Only LOCATION is case-insensitive; PROJECT and CONNECTION_NAME must be lowercase.
+    """
+
+    def normalize(connection_id: str) -> str:
+        parts = connection_id.split(".")
+        if len(parts) == 3:
+            return f"{parts[0]}.{parts[1].casefold()}.{parts[2]}"
+        return connection_id  # Return unchanged if invalid format
+
+    return normalize
+
+
 @pytest.fixture(scope="session")
 def tokyo_location() -> str:
     return TOKYO_LOCATION

tests/system/large/blob/test_function.py

@@ -52,6 +52,7 @@ def images_output_uris(images_output_folder: str) -> list[str]:
     ]
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_exif(
     bq_connection: str,
     session: bigframes.Session,
@@ -103,6 +104,7 @@ def test_blob_exif_verbose(
     assert content_series.dtype == dtypes.JSON_DTYPE
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_image_blur_to_series(
     images_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -136,6 +138,7 @@ def test_blob_image_blur_to_series(
     assert not actual.blob.size().isna().any()
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_image_blur_to_series_verbose(
     images_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -163,6 +166,7 @@ def test_blob_image_blur_to_series_verbose(
     assert not actual.blob.size().isna().any()
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_image_blur_to_folder(
     images_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -195,6 +199,7 @@ def test_blob_image_blur_to_folder(
     assert not actual.blob.size().isna().any()
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_image_blur_to_folder_verbose(
     images_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -254,6 +259,7 @@ def test_blob_image_blur_to_bq_verbose(images_mm_df: bpd.DataFrame, bq_connectio
     assert content_series.dtype == dtypes.BYTES_DTYPE
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_image_resize_to_series(
     images_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -291,6 +297,7 @@ def test_blob_image_resize_to_series(
     assert not actual.blob.size().isna().any()
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_image_resize_to_series_verbose(
     images_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -325,6 +332,7 @@ def test_blob_image_resize_to_series_verbose(
     assert not actual.blob.size().isna().any()
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_image_resize_to_folder(
     images_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -358,6 +366,7 @@ def test_blob_image_resize_to_folder(
     assert not actual.blob.size().isna().any()
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_image_resize_to_folder_verbose(
     images_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -420,6 +429,7 @@ def test_blob_image_resize_to_bq_verbose(
     assert content_series.dtype == dtypes.BYTES_DTYPE
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_image_normalize_to_series(
     images_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -492,6 +502,7 @@ def test_blob_image_normalize_to_series_verbose(
     assert hasattr(content_series, "blob")
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_image_normalize_to_folder(
     images_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -598,6 +609,7 @@ def test_blob_image_normalize_to_bq_verbose(
     assert content_series.dtype == dtypes.BYTES_DTYPE
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_pdf_extract(
     pdf_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -633,6 +645,7 @@ def test_blob_pdf_extract(
         ), f"Item (verbose=False): Expected keyword '{keyword}' not found in extracted text. "
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_pdf_extract_verbose(
     pdf_mm_df: bpd.DataFrame,
     bq_connection: str,
@@ -670,6 +683,7 @@ def test_blob_pdf_extract_verbose(
         ), f"Item (verbose=True): Expected keyword '{keyword}' not found in extracted text. "
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_pdf_chunk(pdf_mm_df: bpd.DataFrame, bq_connection: str):
     actual = (
         pdf_mm_df["pdf"]
@@ -709,6 +723,7 @@ def test_blob_pdf_chunk(pdf_mm_df: bpd.DataFrame, bq_connection: str):
         ), f"Item (verbose=False): Expected keyword '{keyword}' not found in extracted text. "
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_blob_pdf_chunk_verbose(pdf_mm_df: bpd.DataFrame, bq_connection: str):
     actual = (
         pdf_mm_df["pdf"]

tests/system/small/bigquery/test_ai.py

@@ -273,6 +273,7 @@ def test_ai_if(session):
     assert result.dtype == dtypes.BOOL_DTYPE
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_ai_if_multi_model(session):
     df = session.from_glob_path(
         "gs://bigframes-dev-testing/a_multimodel/images/*", name="image"
@@ -293,6 +294,7 @@ def test_ai_classify(session):
     assert result.dtype == dtypes.STRING_DTYPE
 
 
+@pytest.mark.skip(reason="b/457416070")
 def test_ai_classify_multi_model(session):
     df = session.from_glob_path(
         "gs://bigframes-dev-testing/a_multimodel/images/*", name="image"