Merge branch 'main' into tswast-log

Author: tswast

bigframes/core/compile/polars/__init__.py

@@ -25,6 +25,7 @@
 # isn't installed.
 import bigframes.core.compile.polars.operations.generic_ops  # noqa: F401
 import bigframes.core.compile.polars.operations.numeric_ops  # noqa: F401
+import bigframes.core.compile.polars.operations.struct_ops  # noqa: F401
 
 try:
     import bigframes._importing

bigframes/core/compile/polars/operations/struct_ops.py

@@ -0,0 +1,48 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+BigFrames -> Polars compilation for the operations in bigframes.operations.generic_ops.
+
+Please keep implementations in sequential order by op name.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import bigframes_vendored.constants
+
+import bigframes.core.compile.polars.compiler as polars_compiler
+from bigframes.operations import struct_ops
+
+if TYPE_CHECKING:
+    import polars as pl
+
+
+@polars_compiler.register_op(struct_ops.StructFieldOp)
+def struct_field_op_impl(
+    compiler: polars_compiler.PolarsExpressionCompiler,
+    op: struct_ops.StructFieldOp,  # type: ignore
+    input: pl.Expr,
+) -> pl.Expr:
+    if isinstance(op.name_or_index, str):
+        name = op.name_or_index
+    else:
+        raise NotImplementedError(
+            "Referencing a struct field by number not implemented in polars compiler. "
+            f"{bigframes_vendored.constants.FEEDBACK_LINK}"
+        )
+
+    return input.struct.field(name)

bigframes/core/compile/sqlglot/expressions/generic_ops.py

@@ -23,6 +23,7 @@
 import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler
 
 register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op
+register_ternary_op = scalar_compiler.scalar_op_compiler.register_ternary_op
 
 
 @register_unary_op(ops.AsTypeOp, pass_op=True)
@@ -66,6 +67,18 @@ def _(expr: TypedExpr, op: ops.AsTypeOp) -> sge.Expression:
     return _cast(sg_expr, sg_to_type, op.safe)
 
 
+@register_ternary_op(ops.clip_op)
+def _(
+    original: TypedExpr,
+    lower: TypedExpr,
+    upper: TypedExpr,
+) -> sge.Expression:
+    return sge.Greatest(
+        this=sge.Least(this=original.expr, expressions=[upper.expr]),
+        expressions=[lower.expr],
+    )
+
+
 @register_unary_op(ops.hash_op)
 def _(expr: TypedExpr) -> sge.Expression:
     return sge.func("FARM_FINGERPRINT", expr.expr)
@@ -94,6 +107,13 @@ def _(expr: TypedExpr) -> sge.Expression:
     return sge.Not(this=sge.Is(this=expr.expr, expression=sge.Null()))
 
 
+@register_ternary_op(ops.where_op)
+def _(
+    original: TypedExpr, condition: TypedExpr, replacement: TypedExpr
+) -> sge.Expression:
+    return sge.If(this=condition.expr, true=original.expr, false=replacement.expr)
+
+
 # Helper functions
 def _cast_to_json(expr: TypedExpr, op: ops.AsTypeOp) -> sge.Expression:
     from_type = expr.dtype

bigframes/operations/structs.py

@@ -17,9 +17,8 @@
 import bigframes_vendored.pandas.core.arrays.arrow.accessors as vendoracessors
 import pandas as pd
 
-from bigframes.core import log_adapter
+from bigframes.core import backports, log_adapter
 import bigframes.dataframe
-import bigframes.dtypes
 import bigframes.operations
 import bigframes.operations.base
 import bigframes.series
@@ -45,17 +44,24 @@ def explode(self) -> bigframes.dataframe.DataFrame:
 
         pa_type = self._dtype.pyarrow_dtype
         return bigframes.pandas.concat(
-            [self.field(i) for i in range(pa_type.num_fields)], axis="columns"
+            [
+                self.field(field.name)
+                for field in backports.pyarrow_struct_type_fields(pa_type)
+            ],
+            axis="columns",
         )
 
+    @property
     def dtypes(self) -> pd.Series:
         pa_type = self._dtype.pyarrow_dtype
         return pd.Series(
             data=[
-                bigframes.dtypes.arrow_dtype_to_bigframes_dtype(pa_type.field(i).type)
-                for i in range(pa_type.num_fields)
+                pd.ArrowDtype(field.type)
+                for field in backports.pyarrow_struct_type_fields(pa_type)
+            ],
+            index=[
+                field.name for field in backports.pyarrow_struct_type_fields(pa_type)
             ],
-            index=[pa_type.field(i).name for i in range(pa_type.num_fields)],
         )
 
 

notebooks/data_types/struct.ipynb

@@ -211,11 +211,11 @@
     {
      "data": {
       "text/plain": [
-       "0    [{'tables': {'score': 0.9349926710128784, 'val...\n",
-       "1    [{'tables': {'score': 0.9690881371498108, 'val...\n",
-       "2    [{'tables': {'score': 0.8667634129524231, 'val...\n",
-       "3    [{'tables': {'score': 0.9351968765258789, 'val...\n",
-       "4    [{'tables': {'score': 0.8572560548782349, 'val...\n",
+       "0    [{'tables': {'score': 0.8667634129524231, 'val...\n",
+       "1    [{'tables': {'score': 0.9351968765258789, 'val...\n",
+       "2    [{'tables': {'score': 0.8572560548782349, 'val...\n",
+       "3    [{'tables': {'score': 0.9690881371498108, 'val...\n",
+       "4    [{'tables': {'score': 0.9349926710128784, 'val...\n",
        "Name: predicted_default_payment_next_month, dtype: list<item: struct<tables: struct<score: double, value: string>>>[pyarrow]"
       ]
      },
@@ -267,7 +267,7 @@
     }
    ],
    "source": [
-    "df['Address'].struct.dtypes()"
+    "df['Address'].struct.dtypes"
    ]
   },
   {
@@ -461,7 +461,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "venv",
    "language": "python",
    "name": "python3"
   },
@@ -475,7 +475,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.1"
+   "version": "3.12.9"
   }
  },
  "nbformat": 4,

tests/system/small/engines/test_generic_ops.py

@@ -314,7 +314,7 @@ def test_engines_astype_timedelta(scalars_array_value: array_value.ArrayValue, e
     assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine)
 
 
-@pytest.mark.parametrize("engine", ["polars", "bq"], indirect=True)
+@pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True)
 def test_engines_where_op(scalars_array_value: array_value.ArrayValue, engine):
     arr, _ = scalars_array_value.compute_values(
         [

tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_clip/out.sql

@@ -0,0 +1,15 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `int64_col` AS `bfcol_0`,
+    `int64_too` AS `bfcol_1`,
+    `rowindex` AS `bfcol_2`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    GREATEST(LEAST(`bfcol_2`, `bfcol_1`), `bfcol_0`) AS `bfcol_3`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_3` AS `result_col`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_where/out.sql

@@ -0,0 +1,15 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `bool_col` AS `bfcol_0`,
+    `int64_col` AS `bfcol_1`,
+    `float64_col` AS `bfcol_2`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    IF(`bfcol_0`, `bfcol_1`, `bfcol_2`) AS `bfcol_3`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_3` AS `result_col`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/expressions/test_generic_ops.py

@@ -168,6 +168,22 @@ def test_astype_json_invalid(
         )
 
 
+def test_clip(scalar_types_df: bpd.DataFrame, snapshot):
+    op_expr = ops.clip_op.as_expr("rowindex", "int64_col", "int64_too")
+
+    array_value = scalar_types_df._block.expr
+    result, col_ids = array_value.compute_values([op_expr])
+
+    # Rename columns for deterministic golden SQL results.
+    assert len(col_ids) == 1
+    result = result.rename_columns({col_ids[0]: "result_col"}).select_columns(
+        ["result_col"]
+    )
+
+    sql = result.session._executor.to_sql(result, enable_cache=False)
+    snapshot.assert_match(sql, "out.sql")
+
+
 def test_hash(scalar_types_df: bpd.DataFrame, snapshot):
     col_name = "string_col"
     bf_df = scalar_types_df[[col_name]]
@@ -202,3 +218,19 @@ def test_map(scalar_types_df: bpd.DataFrame, snapshot):
     )
 
     snapshot.assert_match(sql, "out.sql")
+
+
+def test_where(scalar_types_df: bpd.DataFrame, snapshot):
+    op_expr = ops.where_op.as_expr("int64_col", "bool_col", "float64_col")
+
+    array_value = scalar_types_df._block.expr
+    result, col_ids = array_value.compute_values([op_expr])
+
+    # Rename columns for deterministic golden SQL results.
+    assert len(col_ids) == 1
+    result = result.rename_columns({col_ids[0]: "result_col"}).select_columns(
+        ["result_col"]
+    )
+
+    sql = result.session._executor.to_sql(result, enable_cache=False)
+    snapshot.assert_match(sql, "out.sql")