Merge branch 'main' into series-input

Author: jialuoo

bigframes/core/compile/sqlglot/aggregations/nullary_compiler.py

@@ -20,7 +20,7 @@
 
 from bigframes.core import window_spec
 import bigframes.core.compile.sqlglot.aggregations.op_registration as reg
-from bigframes.core.compile.sqlglot.aggregations.utils import apply_window_if_present
+from bigframes.core.compile.sqlglot.aggregations.windows import apply_window_if_present
 from bigframes.operations import aggregations as agg_ops
 
 NULLARY_OP_REGISTRATION = reg.OpRegistration()

bigframes/core/compile/sqlglot/aggregations/unary_compiler.py

@@ -20,7 +20,7 @@
 
 from bigframes.core import window_spec
 import bigframes.core.compile.sqlglot.aggregations.op_registration as reg
-from bigframes.core.compile.sqlglot.aggregations.utils import apply_window_if_present
+from bigframes.core.compile.sqlglot.aggregations.windows import apply_window_if_present
 import bigframes.core.compile.sqlglot.expressions.typed_expr as typed_expr
 import bigframes.core.compile.sqlglot.sqlglot_ir as ir
 from bigframes.operations import aggregations as agg_ops

bigframes/core/compile/sqlglot/aggregations/utils.py

@@ -0,0 +1,153 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import typing
+
+import sqlglot.expressions as sge
+
+from bigframes.core import utils, window_spec
+import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler
+import bigframes.core.ordering as ordering_spec
+
+
+def apply_window_if_present(
+    value: sge.Expression,
+    window: typing.Optional[window_spec.WindowSpec] = None,
+) -> sge.Expression:
+    if window is None:
+        return value
+
+    if window.is_row_bounded and not window.ordering:
+        raise ValueError("No ordering provided for ordered analytic function")
+    elif (
+        not window.is_row_bounded
+        and not window.is_range_bounded
+        and not window.ordering
+    ):
+        # Unbound grouping window.
+        order_by = None
+    elif window.is_range_bounded:
+        # Note that, when the window is range-bounded, we only need one ordering key.
+        # There are two reasons:
+        # 1. Manipulating null positions requires more than one ordering key, which
+        #  is forbidden by SQL window syntax for range rolling.
+        # 2. Pandas does not allow range rolling on timeseries with nulls.
+        order_by = get_window_order_by((window.ordering[0],), override_null_order=False)
+    else:
+        order_by = get_window_order_by(window.ordering, override_null_order=True)
+
+    order = sge.Order(expressions=order_by) if order_by else None
+
+    group_by = (
+        [scalar_compiler.compile_scalar_expression(key) for key in window.grouping_keys]
+        if window.grouping_keys
+        else None
+    )
+
+    # This is the key change. Don't create a spec for the default window frame
+    # if there's no ordering. This avoids generating an `ORDER BY NULL` clause.
+    if not window.bounds and not order:
+        return sge.Window(this=value, partition_by=group_by)
+
+    kind = (
+        "ROWS" if isinstance(window.bounds, window_spec.RowsWindowBounds) else "RANGE"
+    )
+
+    start: typing.Union[int, float, None] = None
+    end: typing.Union[int, float, None] = None
+    if isinstance(window.bounds, window_spec.RangeWindowBounds):
+        if window.bounds.start is not None:
+            start = utils.timedelta_to_micros(window.bounds.start)
+        if window.bounds.end is not None:
+            end = utils.timedelta_to_micros(window.bounds.end)
+    elif window.bounds:
+        start = window.bounds.start
+        end = window.bounds.end
+
+    start_value, start_side = _get_window_bounds(start, is_preceding=True)
+    end_value, end_side = _get_window_bounds(end, is_preceding=False)
+
+    spec = sge.WindowSpec(
+        kind=kind,
+        start=start_value,
+        start_side=start_side,
+        end=end_value,
+        end_side=end_side,
+        over="OVER",
+    )
+
+    return sge.Window(this=value, partition_by=group_by, order=order, spec=spec)
+
+
+def get_window_order_by(
+    ordering: typing.Tuple[ordering_spec.OrderingExpression, ...],
+    override_null_order: bool = False,
+) -> typing.Optional[tuple[sge.Ordered, ...]]:
+    """Returns the SQL order by clause for a window specification."""
+    if not ordering:
+        return None
+
+    order_by = []
+    for ordering_spec_item in ordering:
+        expr = scalar_compiler.compile_scalar_expression(
+            ordering_spec_item.scalar_expression
+        )
+        desc = not ordering_spec_item.direction.is_ascending
+        nulls_first = not ordering_spec_item.na_last
+
+        if override_null_order:
+            # Bigquery SQL considers NULLS to be "smallest" values, but we need
+            # to override in these cases.
+            is_null_expr = sge.Is(this=expr, expression=sge.Null())
+            if nulls_first and desc:
+                order_by.append(
+                    sge.Ordered(
+                        this=is_null_expr,
+                        desc=desc,
+                        nulls_first=nulls_first,
+                    )
+                )
+            elif not nulls_first and not desc:
+                order_by.append(
+                    sge.Ordered(
+                        this=is_null_expr,
+                        desc=desc,
+                        nulls_first=nulls_first,
+                    )
+                )
+
+        order_by.append(
+            sge.Ordered(
+                this=expr,
+                desc=desc,
+                nulls_first=nulls_first,
+            )
+        )
+    return tuple(order_by)
+
+
+def _get_window_bounds(
+    value, is_preceding: bool
+) -> tuple[typing.Union[str, sge.Expression], typing.Optional[str]]:
+    """Compiles a single boundary value into its SQL components."""
+    if value is None:
+        side = "PRECEDING" if is_preceding else "FOLLOWING"
+        return "UNBOUNDED", side
+
+    if value == 0:
+        return "CURRENT ROW", None
+
+    side = "PRECEDING" if value < 0 else "FOLLOWING"
+    return sge.convert(abs(value)), side

bigframes/core/compile/sqlglot/aggregations/windows.py

@@ -23,6 +23,7 @@
 from bigframes.core import expression, guid, identifiers, nodes, pyarrow_utils, rewrite
 from bigframes.core.compile import configs
 import bigframes.core.compile.sqlglot.aggregate_compiler as aggregate_compiler
+from bigframes.core.compile.sqlglot.aggregations import windows
 from bigframes.core.compile.sqlglot.expressions import typed_expr
 import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler
 import bigframes.core.compile.sqlglot.sqlglot_ir as ir
@@ -272,18 +273,16 @@ def compile_random_sample(
     def compile_aggregate(
         self, node: nodes.AggregateNode, child: ir.SQLGlotIR
     ) -> ir.SQLGlotIR:
-        ordering_cols = tuple(
-            sge.Ordered(
-                this=scalar_compiler.compile_scalar_expression(
-                    ordering.scalar_expression
-                ),
-                desc=ordering.direction.is_ascending is False,
-                nulls_first=ordering.na_last is False,
-            )
-            for ordering in node.order_by
+        ordering_cols = windows.get_window_order_by(
+            node.order_by, override_null_order=True
         )
         aggregations: tuple[tuple[str, sge.Expression], ...] = tuple(
-            (id.sql, aggregate_compiler.compile_aggregate(agg, order_by=ordering_cols))
+            (
+                id.sql,
+                aggregate_compiler.compile_aggregate(
+                    agg, order_by=ordering_cols if ordering_cols else ()
+                ),
+            )
             for agg, id in node.aggregations
         )
         by_cols: tuple[sge.Expression, ...] = tuple(

bigframes/core/compile/sqlglot/compiler.py

@@ -16,6 +16,8 @@
 
 import typing
 
+import pandas as pd
+import pyarrow as pa
 import sqlglot
 import sqlglot.expressions as sge
 
@@ -105,6 +107,12 @@ def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
     )
 
 
+@UNARY_OP_REGISTRATION.register(ops.AsTypeOp)
+def _(op: ops.AsTypeOp, expr: TypedExpr) -> sge.Expression:
+    # TODO: Support more types for casting, such as JSON, etc.
+    return sge.Cast(this=expr.expr, to=op.to_type)
+
+
 @UNARY_OP_REGISTRATION.register(ops.ArrayToStringOp)
 def _(op: ops.ArrayToStringOp, expr: TypedExpr) -> sge.Expression:
     return sge.ArrayToString(this=expr.expr, expression=f"'{op.delimiter}'")
@@ -234,6 +242,12 @@ def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
     ) - sge.convert(1)
 
 
+@UNARY_OP_REGISTRATION.register(ops.FloorDtOp)
+def _(op: ops.FloorDtOp, expr: TypedExpr) -> sge.Expression:
+    # TODO: Remove this method when it is covered by ops.FloorOp
+    return sge.TimestampTrunc(this=expr.expr, unit=sge.Identifier(this=op.freq))
+
+
 @UNARY_OP_REGISTRATION.register(ops.floor_op)
 def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
     return sge.Floor(this=expr.expr)
@@ -249,6 +263,26 @@ def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
     return sge.func("ST_ASTEXT", expr.expr)
 
 
+@UNARY_OP_REGISTRATION.register(ops.geo_st_boundary_op)
+def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
+    return sge.func("ST_BOUNDARY", expr.expr)
+
+
+@UNARY_OP_REGISTRATION.register(ops.geo_st_geogfromtext_op)
+def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
+    return sge.func("SAFE.ST_GEOGFROMTEXT", expr.expr)
+
+
+@UNARY_OP_REGISTRATION.register(ops.geo_st_isclosed_op)
+def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
+    return sge.func("ST_ISCLOSED", expr.expr)
+
+
+@UNARY_OP_REGISTRATION.register(ops.GeoStLengthOp)
+def _(op: ops.GeoStLengthOp, expr: TypedExpr) -> sge.Expression:
+    return sge.func("ST_LENGTH", expr.expr)
+
+
 @UNARY_OP_REGISTRATION.register(ops.geo_x_op)
 def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
     return sge.func("SAFE.ST_X", expr.expr)
@@ -274,6 +308,11 @@ def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
     return sge.BitwiseNot(this=expr.expr)
 
 
+@UNARY_OP_REGISTRATION.register(ops.IsInOp)
+def _(op: ops.IsInOp, expr: TypedExpr) -> sge.Expression:
+    return sge.In(this=expr.expr, expressions=[sge.convert(v) for v in op.values])
+
+
 @UNARY_OP_REGISTRATION.register(ops.isalnum_op)
 def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
     return sge.RegexpLike(this=expr.expr, expression=sge.convert(r"^(\p{N}|\p{L})+$"))
@@ -517,6 +556,26 @@ def _(op: ops.StrSliceOp, expr: TypedExpr) -> sge.Expression:
     )
 
 
+@UNARY_OP_REGISTRATION.register(ops.StrftimeOp)
+def _(op: ops.StrftimeOp, expr: TypedExpr) -> sge.Expression:
+    return sge.func("FORMAT_TIMESTAMP", sge.convert(op.date_format), expr.expr)
+
+
+@UNARY_OP_REGISTRATION.register(ops.StructFieldOp)
+def _(op: ops.StructFieldOp, expr: TypedExpr) -> sge.Expression:
+    if isinstance(op.name_or_index, str):
+        name = op.name_or_index
+    else:
+        pa_type = typing.cast(pd.ArrowDtype, expr.dtype)
+        pa_struct_type = typing.cast(pa.StructType, pa_type.pyarrow_dtype)
+        name = pa_struct_type.field(op.name_or_index).name
+
+    return sge.Column(
+        this=sge.to_identifier(name, quoted=True),
+        catalog=expr.expr,
+    )
+
+
 @UNARY_OP_REGISTRATION.register(ops.tan_op)
 def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
     return sge.func("TAN", expr.expr)
@@ -537,6 +596,36 @@ def _(op: ops.base_ops.UnaryOp, expr: TypedExpr) -> sge.Expression:
     return sge.Floor(this=expr.expr)
 
 
+@UNARY_OP_REGISTRATION.register(ops.ToDatetimeOp)
+def _(op: ops.ToDatetimeOp, expr: TypedExpr) -> sge.Expression:
+    return sge.Cast(this=sge.func("TIMESTAMP_SECONDS", expr.expr), to="DATETIME")
+
+
+@UNARY_OP_REGISTRATION.register(ops.ToTimestampOp)
+def _(op: ops.ToTimestampOp, expr: TypedExpr) -> sge.Expression:
+    return sge.func("TIMESTAMP_SECONDS", expr.expr)
+
+
+@UNARY_OP_REGISTRATION.register(ops.ToTimedeltaOp)
+def _(op: ops.ToTimedeltaOp, expr: TypedExpr) -> sge.Expression:
+    return sge.Interval(this=expr.expr, unit=sge.Identifier(this="SECOND"))
+
+
+@UNARY_OP_REGISTRATION.register(ops.UnixMicros)
+def _(op: ops.UnixMicros, expr: TypedExpr) -> sge.Expression:
+    return sge.func("UNIX_MICROS", expr.expr)
+
+
+@UNARY_OP_REGISTRATION.register(ops.UnixMillis)
+def _(op: ops.UnixMillis, expr: TypedExpr) -> sge.Expression:
+    return sge.func("UNIX_MILLIS", expr.expr)
+
+
+@UNARY_OP_REGISTRATION.register(ops.UnixSeconds)
+def _(op: ops.UnixSeconds, expr: TypedExpr) -> sge.Expression:
+    return sge.func("UNIX_SECONDS", expr.expr)
+
+
 # JSON Ops
 @UNARY_OP_REGISTRATION.register(ops.JSONExtract)
 def _(op: ops.JSONExtract, expr: TypedExpr) -> sge.Expression:

bigframes/core/compile/sqlglot/expressions/unary_compiler.py

@@ -61,7 +61,8 @@ def __init__(self, type_, supported_types):
         self.type = type_
         self.supported_types = supported_types
         super().__init__(
-            f"'{type_}' is not one of the supported types {supported_types}"
+            f"'{type_}' must be one of the supported types ({supported_types}) "
+            "or a list of one of those types."
         )