Merge remote-tracking branch 'origin/main' into b409390651-progress-bar

Author: tswast

bigframes/core/compile/sqlglot/aggregate_compiler.py

@@ -63,7 +63,7 @@ def compile_analytic(
     window: window_spec.WindowSpec,
 ) -> sge.Expression:
     if isinstance(aggregate, agg_expressions.NullaryAggregation):
-        return nullary_compiler.compile(aggregate.op)
+        return nullary_compiler.compile(aggregate.op, window)
     if isinstance(aggregate, agg_expressions.UnaryAggregation):
         column = typed_expr.TypedExpr(
             scalar_compiler.scalar_op_compiler.compile_expression(aggregate.arg),

bigframes/core/compile/sqlglot/aggregations/binary_compiler.py

@@ -20,6 +20,7 @@
 
 from bigframes.core import window_spec
 import bigframes.core.compile.sqlglot.aggregations.op_registration as reg
+from bigframes.core.compile.sqlglot.aggregations.windows import apply_window_if_present
 import bigframes.core.compile.sqlglot.expressions.typed_expr as typed_expr
 from bigframes.operations import aggregations as agg_ops
 
@@ -33,3 +34,25 @@ def compile(
     window: typing.Optional[window_spec.WindowSpec] = None,
 ) -> sge.Expression:
     return BINARY_OP_REGISTRATION[op](op, left, right, window=window)
+
+
+@BINARY_OP_REGISTRATION.register(agg_ops.CorrOp)
+def _(
+    op: agg_ops.CorrOp,
+    left: typed_expr.TypedExpr,
+    right: typed_expr.TypedExpr,
+    window: typing.Optional[window_spec.WindowSpec] = None,
+) -> sge.Expression:
+    result = sge.func("CORR", left.expr, right.expr)
+    return apply_window_if_present(result, window)
+
+
+@BINARY_OP_REGISTRATION.register(agg_ops.CovOp)
+def _(
+    op: agg_ops.CovOp,
+    left: typed_expr.TypedExpr,
+    right: typed_expr.TypedExpr,
+    window: typing.Optional[window_spec.WindowSpec] = None,
+) -> sge.Expression:
+    result = sge.func("COVAR_SAMP", left.expr, right.expr)
+    return apply_window_if_present(result, window)

bigframes/core/compile/sqlglot/aggregations/nullary_compiler.py

@@ -39,3 +39,15 @@ def _(
     window: typing.Optional[window_spec.WindowSpec] = None,
 ) -> sge.Expression:
     return apply_window_if_present(sge.func("COUNT", sge.convert(1)), window)
+
+
+@NULLARY_OP_REGISTRATION.register(agg_ops.RowNumberOp)
+def _(
+    op: agg_ops.RowNumberOp,
+    window: typing.Optional[window_spec.WindowSpec] = None,
+) -> sge.Expression:
+    result: sge.Expression = sge.func("ROW_NUMBER")
+    if window is None:
+        # ROW_NUMBER always needs an OVER clause.
+        return sge.Window(this=result)
+    return apply_window_if_present(result, window)

bigframes/core/compile/sqlglot/aggregations/op_registration.py

@@ -41,22 +41,16 @@ def arg_checker(*args, **kwargs):
                     )
                 return item(*args, **kwargs)
 
-            if hasattr(op, "name"):
-                key = typing.cast(str, op.name)
-                if key in self._registered_ops:
-                    raise ValueError(f"{key} is already registered")
-            else:
-                raise ValueError(f"The operator must have a 'name' attribute. Got {op}")
+            key = str(op)
+            if key in self._registered_ops:
+                raise ValueError(f"{key} is already registered")
             self._registered_ops[key] = item
             return arg_checker
 
         return decorator
 
     def __getitem__(self, op: str | agg_ops.WindowOp) -> CompilationFunc:
-        if isinstance(op, agg_ops.WindowOp):
-            if not hasattr(op, "name"):
-                raise ValueError(f"The operator must have a 'name' attribute. Got {op}")
-            else:
-                key = typing.cast(str, op.name)
-                return self._registered_ops[key]
-        return self._registered_ops[op]
+        key = op if isinstance(op, type) else type(op)
+        if str(key) not in self._registered_ops:
+            raise ValueError(f"{key} is already not registered")
+        return self._registered_ops[str(key)]

bigframes/core/compile/sqlglot/aggregations/unary_compiler.py

@@ -38,6 +38,37 @@ def compile(
     return UNARY_OP_REGISTRATION[op](op, column, window=window)
 
 
+@UNARY_OP_REGISTRATION.register(agg_ops.ApproxQuartilesOp)
+def _(
+    op: agg_ops.ApproxQuartilesOp,
+    column: typed_expr.TypedExpr,
+    window: typing.Optional[window_spec.WindowSpec] = None,
+) -> sge.Expression:
+    if window is not None:
+        raise NotImplementedError("Approx Quartiles with windowing is not supported.")
+    # APPROX_QUANTILES returns an array of the quartiles, so we need to index it.
+    # The op.quartile is 1-based for the quartile, but array is 0-indexed.
+    # The quartiles are Q0, Q1, Q2, Q3, Q4. op.quartile is 1, 2, or 3.
+    # The array has 5 elements (for N=4 intervals).
+    # So we want the element at index `op.quartile`.
+    approx_quantiles_expr = sge.func("APPROX_QUANTILES", column.expr, sge.convert(4))
+    return sge.Bracket(
+        this=approx_quantiles_expr,
+        expressions=[sge.func("OFFSET", sge.convert(op.quartile))],
+    )
+
+
+@UNARY_OP_REGISTRATION.register(agg_ops.ApproxTopCountOp)
+def _(
+    op: agg_ops.ApproxTopCountOp,
+    column: typed_expr.TypedExpr,
+    window: typing.Optional[window_spec.WindowSpec] = None,
+) -> sge.Expression:
+    if window is not None:
+        raise NotImplementedError("Approx top count with windowing is not supported.")
+    return sge.func("APPROX_TOP_COUNT", column.expr, sge.convert(op.number))
+
+
 @UNARY_OP_REGISTRATION.register(agg_ops.CountOp)
 def _(
     op: agg_ops.CountOp,
@@ -53,10 +84,7 @@ def _(
     column: typed_expr.TypedExpr,
     window: typing.Optional[window_spec.WindowSpec] = None,
 ) -> sge.Expression:
-    # Ranking functions do not support window framing clauses.
-    return apply_window_if_present(
-        sge.func("DENSE_RANK"), window, include_framing_clauses=False
-    )
+    return apply_window_if_present(sge.func("DENSE_RANK"), window)
 
 
 @UNARY_OP_REGISTRATION.register(agg_ops.MaxOp)
@@ -109,13 +137,23 @@ def _(
     return apply_window_if_present(sge.func("MIN", column.expr), window)
 
 
-@UNARY_OP_REGISTRATION.register(agg_ops.SizeUnaryOp)
+@UNARY_OP_REGISTRATION.register(agg_ops.QuantileOp)
 def _(
-    op: agg_ops.SizeUnaryOp,
-    _,
+    op: agg_ops.QuantileOp,
+    column: typed_expr.TypedExpr,
     window: typing.Optional[window_spec.WindowSpec] = None,
 ) -> sge.Expression:
-    return apply_window_if_present(sge.func("COUNT", sge.convert(1)), window)
+    # TODO: Support interpolation argument
+    # TODO: Support percentile_disc
+    result: sge.Expression = sge.func("PERCENTILE_CONT", column.expr, sge.convert(op.q))
+    if window is None:
+        # PERCENTILE_CONT is a navigation function, not an aggregate function, so it always needs an OVER clause.
+        result = sge.Window(this=result)
+    else:
+        result = apply_window_if_present(result, window)
+    if op.should_floor_result:
+        result = sge.Cast(this=sge.func("FLOOR", result), to="INT64")
+    return result
 
 
 @UNARY_OP_REGISTRATION.register(agg_ops.RankOp)
@@ -124,10 +162,16 @@ def _(
     column: typed_expr.TypedExpr,
     window: typing.Optional[window_spec.WindowSpec] = None,
 ) -> sge.Expression:
-    # Ranking functions do not support window framing clauses.
-    return apply_window_if_present(
-        sge.func("RANK"), window, include_framing_clauses=False
-    )
+    return apply_window_if_present(sge.func("RANK"), window)
+
+
+@UNARY_OP_REGISTRATION.register(agg_ops.SizeUnaryOp)
+def _(
+    op: agg_ops.SizeUnaryOp,
+    _,
+    window: typing.Optional[window_spec.WindowSpec] = None,
+) -> sge.Expression:
+    return apply_window_if_present(sge.func("COUNT", sge.convert(1)), window)
 
 
 @UNARY_OP_REGISTRATION.register(agg_ops.SumOp)

bigframes/core/compile/sqlglot/aggregations/windows.py

@@ -25,7 +25,6 @@
 def apply_window_if_present(
     value: sge.Expression,
     window: typing.Optional[window_spec.WindowSpec] = None,
-    include_framing_clauses: bool = True,
 ) -> sge.Expression:
     if window is None:
         return value
@@ -65,7 +64,7 @@ def apply_window_if_present(
     if not window.bounds and not order:
         return sge.Window(this=value, partition_by=group_by)
 
-    if not window.bounds and not include_framing_clauses:
+    if not window.bounds:
         return sge.Window(this=value, partition_by=group_by, order=order)
 
     kind = (

tests/unit/core/compile/sqlglot/aggregations/snapshots/test_binary_compiler/test_corr/out.sql

@@ -0,0 +1,13 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `int64_col` AS `bfcol_0`,
+    `float64_col` AS `bfcol_1`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    CORR(`bfcol_0`, `bfcol_1`) AS `bfcol_2`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_2` AS `corr_col`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/aggregations/snapshots/test_binary_compiler/test_cov/out.sql

@@ -0,0 +1,13 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `int64_col` AS `bfcol_0`,
+    `float64_col` AS `bfcol_1`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    COVAR_SAMP(`bfcol_0`, `bfcol_1`) AS `bfcol_2`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_2` AS `cov_col`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_row_number/out.sql

@@ -0,0 +1,13 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `bool_col` AS `bfcol_0`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    ROW_NUMBER() OVER () AS `bfcol_1`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_1` AS `row_number`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/aggregations/snapshots/test_nullary_compiler/test_row_number_with_window/out.sql

@@ -0,0 +1,13 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `int64_col` AS `bfcol_0`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    ROW_NUMBER() OVER (ORDER BY `bfcol_0` IS NULL ASC NULLS LAST, `bfcol_0` ASC NULLS LAST) AS `bfcol_1`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_1` AS `row_number`
+FROM `bfcte_1`