Merge remote-tracking branch 'github/main' into python_fn_mappings

Author: TrevorBergeron

CHANGELOG.md

@@ -4,6 +4,24 @@
 
 [1]: https://pypi.org/project/bigframes/#history
 
+## [2.27.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.26.0...v2.27.0) (2025-10-24)
+
+
+### Features
+
+* Add __abs__ to dataframe ([#2186](https://github.com/googleapis/python-bigquery-dataframes/issues/2186)) ([c331dfe](https://github.com/googleapis/python-bigquery-dataframes/commit/c331dfed59174962fbdc8ace175dd00fcc3d5d50))
+* Add df.groupby().corr()/cov() support ([#2190](https://github.com/googleapis/python-bigquery-dataframes/issues/2190)) ([ccd7c07](https://github.com/googleapis/python-bigquery-dataframes/commit/ccd7c0774a65d09e6cf31d2b62d0bc64bd7c4248))
+* Add str accessor to index ([#2179](https://github.com/googleapis/python-bigquery-dataframes/issues/2179)) ([cd87ce0](https://github.com/googleapis/python-bigquery-dataframes/commit/cd87ce0d504747f44d1b5a55f869a2e0fca6df17))
+* Add support for `np.isnan` and `np.isfinite` ufuncs ([#2188](https://github.com/googleapis/python-bigquery-dataframes/issues/2188)) ([68723bc](https://github.com/googleapis/python-bigquery-dataframes/commit/68723bc1f08013e43a8b11752f908bf8fd6d51f5))
+* Include local data bytes in the dry run report when available ([#2185](https://github.com/googleapis/python-bigquery-dataframes/issues/2185)) ([ee2c40c](https://github.com/googleapis/python-bigquery-dataframes/commit/ee2c40c6789535e259fb6a9774831d6913d16212))
+* Support len() on Groupby objects ([#2183](https://github.com/googleapis/python-bigquery-dataframes/issues/2183)) ([4191821](https://github.com/googleapis/python-bigquery-dataframes/commit/4191821b0976281a96c8965336ef51f061b0c481))
+* Support pa.json_(pa.string()) in struct/list if available ([#2180](https://github.com/googleapis/python-bigquery-dataframes/issues/2180)) ([5ec3cc0](https://github.com/googleapis/python-bigquery-dataframes/commit/5ec3cc0298c7a6195d5bd12a08d996e7df57fc5f))
+
+
+### Documentation
+
+* Update AI operators deprecation notice ([#2182](https://github.com/googleapis/python-bigquery-dataframes/issues/2182)) ([2c50310](https://github.com/googleapis/python-bigquery-dataframes/commit/2c503107e17c59232b14b0d7bc40c350bb087d6f))
+
 ## [2.26.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.25.0...v2.26.0) (2025-10-17)
 
 

bigframes/core/compile/ibis_compiler/scalar_op_compiler.py

@@ -26,6 +26,7 @@
 from bigframes.core import agg_expressions, ordering
 import bigframes.core.compile.ibis_types
 import bigframes.core.expression as ex
+from bigframes.operations import numeric_ops
 
 if TYPE_CHECKING:
     import bigframes.operations as ops
@@ -267,3 +268,13 @@ def _convert_range_ordering_to_table_value(
 
 # Singleton compiler
 scalar_op_compiler = ExpressionCompiler()
+
+
+@scalar_op_compiler.register_unary_op(numeric_ops.isnan_op)
+def isnanornull(arg):
+    return arg.isnan()
+
+
+@scalar_op_compiler.register_unary_op(numeric_ops.isfinite_op)
+def isfinite(arg):
+    return arg.isinf().negate() & arg.isnan().negate()

bigframes/core/compile/polars/operations/numeric_ops.py

@@ -152,3 +152,21 @@ def sqrt_op_impl(
     import polars as pl
 
     return pl.when(input < 0).then(float("nan")).otherwise(input.sqrt())
+
+
+@polars_compiler.register_op(numeric_ops.IsNanOp)
+def is_nan_op_impl(
+    compiler: polars_compiler.PolarsExpressionCompiler,
+    op: numeric_ops.IsNanOp,  # type: ignore
+    input: pl.Expr,
+) -> pl.Expr:
+    return input.is_nan()
+
+
+@polars_compiler.register_op(numeric_ops.IsFiniteOp)
+def is_finite_op_impl(
+    compiler: polars_compiler.PolarsExpressionCompiler,
+    op: numeric_ops.IsFiniteOp,  # type: ignore
+    input: pl.Expr,
+) -> pl.Expr:
+    return input.is_finite()

bigframes/core/compile/sqlglot/expressions/generic_ops.py

@@ -76,8 +76,8 @@ def _(expr: TypedExpr) -> sge.Expression:
 @register_unary_op(ops.invert_op)
 def _(expr: TypedExpr) -> sge.Expression:
     if expr.dtype == dtypes.BOOL_DTYPE:
-        return sge.Not(this=expr.expr)
-    return sge.BitwiseNot(this=expr.expr)
+        return sge.Not(this=sge.paren(expr.expr))
+    return sge.BitwiseNot(this=sge.paren(expr.expr))
 
 
 @register_unary_op(ops.isnull_op)

bigframes/core/compile/sqlglot/expressions/numeric_ops.py

@@ -22,6 +22,7 @@
 import bigframes.core.compile.sqlglot.expressions.constants as constants
 from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr
 import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler
+from bigframes.operations import numeric_ops
 
 register_unary_op = scalar_compiler.scalar_op_compiler.register_unary_op
 register_binary_op = scalar_compiler.scalar_op_compiler.register_binary_op
@@ -189,7 +190,7 @@ def _(expr: TypedExpr) -> sge.Expression:
 
 @register_unary_op(ops.neg_op)
 def _(expr: TypedExpr) -> sge.Expression:
-    return sge.Neg(this=expr.expr)
+    return sge.Neg(this=sge.paren(expr.expr))
 
 
 @register_unary_op(ops.pos_op)
@@ -408,6 +409,21 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression:
     )
 
 
+@register_unary_op(numeric_ops.isnan_op)
+def isnan(arg: TypedExpr) -> sge.Expression:
+    return sge.IsNan(this=arg.expr)
+
+
+@register_unary_op(numeric_ops.isfinite_op)
+def isfinite(arg: TypedExpr) -> sge.Expression:
+    return sge.Not(
+        this=sge.Or(
+            this=sge.IsInf(this=arg.expr),
+            right=sge.IsNan(this=arg.expr),
+        ),
+    )
+
+
 def _coerce_bool_to_int(typed_expr: TypedExpr) -> sge.Expression:
     """Coerce boolean expression to integer."""
     if typed_expr.dtype == dtypes.BOOL_DTYPE:

bigframes/core/groupby/dataframe_group_by.py

@@ -177,6 +177,9 @@ def __iter__(self) -> Iterable[Tuple[blocks.Label, df.DataFrame]]:
             filtered_df = df.DataFrame(filtered_block)
             yield group_keys, filtered_df
 
+    def __len__(self) -> int:
+        return len(self.agg([]))
+
     def size(self) -> typing.Union[df.DataFrame, series.Series]:
         agg_block, _ = self._block.aggregate_size(
             by_column_ids=self._by_col_ids,
@@ -275,6 +278,76 @@ def var(
             self._raise_on_non_numeric("var")
         return self._aggregate_all(agg_ops.var_op, numeric_only=True)
 
+    def corr(
+        self,
+        *,
+        numeric_only: bool = False,
+    ) -> df.DataFrame:
+        if not numeric_only:
+            self._raise_on_non_numeric("corr")
+        if len(self._selected_cols) > 30:
+            raise ValueError(
+                f"Cannot calculate corr on >30 columns, dataframe has {len(self._selected_cols)} selected columns."
+            )
+
+        labels = self._block._get_labels_for_columns(self._selected_cols)
+        block = self._block
+        aggregations = [
+            agg_expressions.BinaryAggregation(
+                agg_ops.CorrOp(), ex.deref(left_col), ex.deref(right_col)
+            )
+            for left_col in self._selected_cols
+            for right_col in self._selected_cols
+        ]
+        # unique columns stops
+        uniq_orig_columns = utils.combine_indices(labels, pd.Index(range(len(labels))))
+        result_labels = utils.cross_indices(uniq_orig_columns, uniq_orig_columns)
+
+        block, _ = block.aggregate(
+            by_column_ids=self._by_col_ids,
+            aggregations=aggregations,
+            column_labels=result_labels,
+        )
+
+        block = block.stack(levels=labels.nlevels + 1)
+        # Drop the last level of each index, which was created to guarantee uniqueness
+        return df.DataFrame(block).droplevel(-1, axis=0).droplevel(-1, axis=1)
+
+    def cov(
+        self,
+        *,
+        numeric_only: bool = False,
+    ) -> df.DataFrame:
+        if not numeric_only:
+            self._raise_on_non_numeric("cov")
+        if len(self._selected_cols) > 30:
+            raise ValueError(
+                f"Cannot calculate cov on >30 columns, dataframe has {len(self._selected_cols)} selected columns."
+            )
+
+        labels = self._block._get_labels_for_columns(self._selected_cols)
+        block = self._block
+        aggregations = [
+            agg_expressions.BinaryAggregation(
+                agg_ops.CovOp(), ex.deref(left_col), ex.deref(right_col)
+            )
+            for left_col in self._selected_cols
+            for right_col in self._selected_cols
+        ]
+        # unique columns stops
+        uniq_orig_columns = utils.combine_indices(labels, pd.Index(range(len(labels))))
+        result_labels = utils.cross_indices(uniq_orig_columns, uniq_orig_columns)
+
+        block, _ = block.aggregate(
+            by_column_ids=self._by_col_ids,
+            aggregations=aggregations,
+            column_labels=result_labels,
+        )
+
+        block = block.stack(levels=labels.nlevels + 1)
+        # Drop the last level of each index, which was created to guarantee uniqueness
+        return df.DataFrame(block).droplevel(-1, axis=0).droplevel(-1, axis=1)
+
     def skew(
         self,
         *,

bigframes/core/groupby/series_group_by.py

@@ -108,6 +108,9 @@ def __iter__(self) -> Iterable[Tuple[blocks.Label, series.Series]]:
             filtered_series.name = self._value_name
             yield group_keys, filtered_series
 
+    def __len__(self) -> int:
+        return len(self.agg([]))
+
     def all(self) -> series.Series:
         return self._aggregate(agg_ops.all_op)
 
@@ -275,9 +278,9 @@ def agg(self, func=None) -> typing.Union[df.DataFrame, series.Series]:
         if column_names:
             agg_block = agg_block.with_column_labels(column_names)
 
-        if len(aggregations) > 1:
-            return df.DataFrame(agg_block)
-        return series.Series(agg_block)
+        if len(aggregations) == 1:
+            return series.Series(agg_block)
+        return df.DataFrame(agg_block)
 
     aggregate = agg
 

bigframes/dtypes.py

@@ -340,6 +340,12 @@ def is_struct_like(type_: ExpressionType) -> bool:
     )
 
 
+def is_json_arrow_type(type_: pa.DataType) -> bool:
+    return isinstance(type_, db_dtypes.JSONArrowType) or (
+        hasattr(pa, "JsonType") and isinstance(type_, pa.JsonType)
+    )
+
+
 def is_json_like(type_: ExpressionType) -> bool:
     return type_ == JSON_DTYPE or type_ == STRING_DTYPE  # Including JSON string
 
@@ -510,6 +516,10 @@ def arrow_dtype_to_bigframes_dtype(
     if arrow_dtype == pa.null():
         return DEFAULT_DTYPE
 
+    # Allow both db_dtypes.JSONArrowType() and pa.json_(pa.string())
+    if is_json_arrow_type(arrow_dtype):
+        return JSON_DTYPE
+
     # No other types matched.
     raise TypeError(
         f"Unexpected Arrow data type {arrow_dtype}. {constants.FEEDBACK_LINK}"

bigframes/operations/numeric_ops.py

@@ -348,3 +348,19 @@ def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionT
     name="unsafe_pow_op", type_signature=op_typing.BINARY_REAL_NUMERIC
 )
 unsafe_pow_op = UnsafePowOp()
+
+IsNanOp = base_ops.create_unary_op(
+    name="isnan",
+    type_signature=op_typing.FixedOutputType(
+        dtypes.is_numeric, dtypes.BOOL_DTYPE, "numeric"
+    ),
+)
+isnan_op = IsNanOp()
+
+IsFiniteOp = base_ops.create_unary_op(
+    name="isfinite",
+    type_signature=op_typing.FixedOutputType(
+        dtypes.is_numeric, dtypes.BOOL_DTYPE, "numeric"
+    ),
+)
+isfinite_op = IsFiniteOp()

bigframes/operations/numpy_op_maps.py

@@ -40,6 +40,8 @@
     np.ceil: numeric_ops.ceil_op,
     np.log1p: numeric_ops.log1p_op,
     np.expm1: numeric_ops.expm1_op,
+    np.isnan: numeric_ops.isnan_op,
+    np.isfinite: numeric_ops.isfinite_op,
 }