Merge branch 'main' into migrate-manhatton-dist

Author: jialuoo

bigframes/core/compile/sqlglot/compiler.py

@@ -172,6 +172,7 @@ def compile_readtable(node: nodes.ReadTableNode, child: ir.SQLGlotIR):
         col_names=[col.source_id for col in node.scan_list.items],
         alias_names=[col.id.sql for col in node.scan_list.items],
         uid_gen=child.uid_gen,
+        system_time=node.source.at_time,
     )
 
 

bigframes/core/compile/sqlglot/expressions/generic_ops.py

@@ -134,6 +134,11 @@ def _(
     )
 
 
+@register_binary_op(ops.fillna_op)
+def _(left: TypedExpr, right: TypedExpr) -> sge.Expression:
+    return sge.Coalesce(this=left.expr, expressions=[right.expr])
+
+
 @register_nary_op(ops.case_when_op)
 def _(*cases_and_outputs: TypedExpr) -> sge.Expression:
     # Need to upcast BOOL to INT if any output is numeric

bigframes/core/compile/sqlglot/expressions/numeric_ops.py

@@ -305,6 +305,18 @@ def _(left: TypedExpr, right: TypedExpr) -> sge.Expression:
         return result
 
 
+@register_binary_op(ops.euclidean_distance_op)
+def _(left: TypedExpr, right: TypedExpr) -> sge.Expression:
+    return sge.Anonymous(
+        this="ML.DISTANCE",
+        expressions=[
+            left.expr,
+            right.expr,
+            sge.Literal.string("EUCLIDEAN"),
+        ],
+    )
+
+
 @register_binary_op(ops.floordiv_op)
 def _(left: TypedExpr, right: TypedExpr) -> sge.Expression:
     left_expr = _coerce_bool_to_int(left)

bigframes/core/compile/sqlglot/sqlglot_ir.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import dataclasses
+import datetime
 import functools
 import typing
 
@@ -118,6 +119,7 @@ def from_table(
         col_names: typing.Sequence[str],
         alias_names: typing.Sequence[str],
         uid_gen: guid.SequentialUIDGenerator,
+        system_time: typing.Optional[datetime.datetime] = None,
     ) -> SQLGlotIR:
         """Builds a SQLGlotIR expression from a BigQuery table.
 
@@ -128,6 +130,7 @@ def from_table(
             col_names (typing.Sequence[str]): The names of the columns to select.
             alias_names (typing.Sequence[str]): The aliases for the selected columns.
             uid_gen (guid.SequentialUIDGenerator): A generator for unique identifiers.
+            system_time (typing.Optional[str]): An optional system time for time-travel queries.
         """
         selections = [
             sge.Alias(
@@ -138,10 +141,20 @@ def from_table(
             else sge.to_identifier(col_name, quoted=cls.quoted)
             for col_name, alias_name in zip(col_names, alias_names)
         ]
+        version = (
+            sge.Version(
+                this="TIMESTAMP",
+                expression=sge.Literal(this=system_time.isoformat(), is_string=True),
+                kind="AS OF",
+            )
+            if system_time
+            else None
+        )
         table_expr = sge.Table(
             this=sg.to_identifier(table_id, quoted=cls.quoted),
             db=sg.to_identifier(dataset_id, quoted=cls.quoted),
             catalog=sg.to_identifier(project_id, quoted=cls.quoted),
+            version=version,
         )
         select_expr = sge.Select().select(*selections).from_(table_expr)
         return cls(expr=select_expr, uid_gen=uid_gen)

tests/system/small/engines/test_generic_ops.py

@@ -343,7 +343,7 @@ def test_engines_coalesce_op(scalars_array_value: array_value.ArrayValue, engine
     assert_equivalence_execution(arr.node, REFERENCE_ENGINE, engine)
 
 
-@pytest.mark.parametrize("engine", ["polars", "bq"], indirect=True)
+@pytest.mark.parametrize("engine", ["polars", "bq", "bq-sqlglot"], indirect=True)
 def test_engines_fillna_op(scalars_array_value: array_value.ArrayValue, engine):
     arr, _ = scalars_array_value.compute_values(
         [

tests/unit/core/compile/sqlglot/conftest.py

@@ -97,7 +97,10 @@ def scalar_types_table_schema() -> typing.Sequence[bigquery.SchemaField]:
 def scalar_types_df(compiler_session) -> bpd.DataFrame:
     """Returns a BigFrames DataFrame containing all scalar types and using the `rowindex`
     column as the index."""
-    bf_df = compiler_session.read_gbq_table("bigframes-dev.sqlglot_test.scalar_types")
+    bf_df = compiler_session._loader.read_gbq_table(
+        "bigframes-dev.sqlglot_test.scalar_types",
+        enable_snapshot=False,
+    )
     bf_df = bf_df.set_index("rowindex", drop=False)
     return bf_df
 
@@ -154,8 +157,9 @@ def nested_structs_types_table_schema() -> typing.Sequence[bigquery.SchemaField]
 def nested_structs_types_df(compiler_session_w_nested_structs_types) -> bpd.DataFrame:
     """Returns a BigFrames DataFrame containing all scalar types and using the `rowindex`
     column as the index."""
-    bf_df = compiler_session_w_nested_structs_types.read_gbq_table(
-        "bigframes-dev.sqlglot_test.nested_structs_types"
+    bf_df = compiler_session_w_nested_structs_types._loader.read_gbq_table(
+        "bigframes-dev.sqlglot_test.nested_structs_types",
+        enable_snapshot=False,
     )
     bf_df = bf_df.set_index("id", drop=False)
     return bf_df
@@ -204,8 +208,9 @@ def repeated_types_table_schema() -> typing.Sequence[bigquery.SchemaField]:
 def repeated_types_df(compiler_session_w_repeated_types) -> bpd.DataFrame:
     """Returns a BigFrames DataFrame containing all scalar types and using the `rowindex`
     column as the index."""
-    bf_df = compiler_session_w_repeated_types.read_gbq_table(
-        "bigframes-dev.sqlglot_test.repeated_types"
+    bf_df = compiler_session_w_repeated_types._loader.read_gbq_table(
+        "bigframes-dev.sqlglot_test.repeated_types",
+        enable_snapshot=False,
     )
     bf_df = bf_df.set_index("rowindex", drop=False)
     return bf_df
@@ -237,8 +242,9 @@ def json_types_table_schema() -> typing.Sequence[bigquery.SchemaField]:
 def json_types_df(compiler_session_w_json_types) -> bpd.DataFrame:
     """Returns a BigFrames DataFrame containing JSON types and using the `rowindex`
     column as the index."""
-    bf_df = compiler_session_w_json_types.read_gbq_table(
-        "bigframes-dev.sqlglot_test.json_types"
+    bf_df = compiler_session_w_json_types._loader.read_gbq_table(
+        "bigframes-dev.sqlglot_test.json_types",
+        enable_snapshot=False,
     )
     # TODO(b/427305807): Why `drop=False` will produce two "rowindex" columns?
     bf_df = bf_df.set_index("rowindex", drop=True)

tests/unit/core/compile/sqlglot/expressions/snapshots/test_generic_ops/test_fillna/out.sql

@@ -0,0 +1,14 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `float64_col`,
+    `int64_col`
+  FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    COALESCE(`int64_col`, `float64_col`) AS `bfcol_2`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_2` AS `int64_col`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/expressions/snapshots/test_numeric_ops/test_euclidean_distance/out.sql

@@ -0,0 +1,16 @@
+WITH `bfcte_0` AS (
+  SELECT
+    `int_list_col`,
+    `numeric_list_col`
+  FROM `bigframes-dev`.`sqlglot_test`.`repeated_types`
+), `bfcte_1` AS (
+  SELECT
+    *,
+    ML.DISTANCE(`int_list_col`, `int_list_col`, 'EUCLIDEAN') AS `bfcol_2`,
+    ML.DISTANCE(`numeric_list_col`, `numeric_list_col`, 'EUCLIDEAN') AS `bfcol_3`
+  FROM `bfcte_0`
+)
+SELECT
+  `bfcol_2` AS `int_list_col`,
+  `bfcol_3` AS `numeric_list_col`
+FROM `bfcte_1`

tests/unit/core/compile/sqlglot/expressions/test_generic_ops.py

@@ -239,6 +239,12 @@ def test_clip(scalar_types_df: bpd.DataFrame, snapshot):
     snapshot.assert_match(sql, "out.sql")
 
 
+def test_fillna(scalar_types_df: bpd.DataFrame, snapshot):
+    bf_df = scalar_types_df[["int64_col", "float64_col"]]
+    sql = utils._apply_binary_op(bf_df, ops.fillna_op, "int64_col", "float64_col")
+    snapshot.assert_match(sql, "out.sql")
+
+
 def test_hash(scalar_types_df: bpd.DataFrame, snapshot):
     col_name = "string_col"
     bf_df = scalar_types_df[[col_name]]

tests/unit/core/compile/sqlglot/expressions/test_numeric_ops.py

@@ -315,6 +315,21 @@ def test_div_timedelta(scalar_types_df: bpd.DataFrame, snapshot):
     snapshot.assert_match(bf_df.sql, "out.sql")
 
 
+def test_euclidean_distance(repeated_types_df: bpd.DataFrame, snapshot):
+    col_names = ["int_list_col", "numeric_list_col"]
+    bf_df = repeated_types_df[col_names]
+
+    sql = utils._apply_ops_to_sql(
+        bf_df,
+        [
+            ops.euclidean_distance_op.as_expr("int_list_col", "int_list_col"),
+            ops.euclidean_distance_op.as_expr("numeric_list_col", "numeric_list_col"),
+        ],
+        ["int_list_col", "numeric_list_col"],
+    )
+    snapshot.assert_match(sql, "out.sql")
+
+
 def test_floordiv_numeric(scalar_types_df: bpd.DataFrame, snapshot):
     bf_df = scalar_types_df[["int64_col", "bool_col", "float64_col"]]