fix: Improve strictness of nan vs None usage

Author: TrevorBergeron

bigframes/core/blocks.py

@@ -1295,7 +1295,7 @@ def aggregate_all_and_stack(
             as_array = ops.ToArrayOp().as_expr(*(col for col in self.value_columns))
             reduced = ops.ArrayReduceOp(operation).as_expr(as_array)
             block, id = self.project_expr(reduced, None)
-            return block.select_column(id)
+            return block.select_column(id).with_column_labels(pd.Index([None]))
 
     def aggregate_size(
         self,

bigframes/dataframe.py

@@ -5012,7 +5012,7 @@ def duplicated(self, subset=None, keep: str = "first") -> bigframes.series.Serie
         return bigframes.series.Series(
             block.select_column(
                 indicator,
-            )
+            ).with_column_labels(pandas.Index([None])),
         )
 
     def rank(

bigframes/series.py

@@ -2653,9 +2653,10 @@ def _apply_unary_op(
     ) -> Series:
         """Applies a unary operator to the series."""
         block, result_id = self._block.apply_unary_op(
-            self._value_column, op, result_label=self._name
+            self._value_column,
+            op,
         )
-        return Series(block.select_column(result_id))
+        return Series(block.select_column(result_id), name=self.name)  # type: ignore
 
     def _apply_binary_op(
         self,
@@ -2683,17 +2684,19 @@ def _apply_binary_op(
             expr = op.as_expr(
                 other_col if reverse else self_col, self_col if reverse else other_col
             )
-            block, result_id = block.project_expr(expr, name)
-            return Series(block.select_column(result_id))
+            block, result_id = block.project_expr(expr)
+            block = block.select_column(result_id).with_column_labels([name])
+            return Series(block)  # type: ignore
 
         else:  # Scalar binop
             name = self._name
             expr = op.as_expr(
                 ex.const(other) if reverse else self._value_column,
                 self._value_column if reverse else ex.const(other),
             )
-            block, result_id = self._block.project_expr(expr, name)
-            return Series(block.select_column(result_id))
+            block, result_id = self._block.project_expr(expr)
+            block = block.select_column(result_id).with_column_labels([name])
+            return Series(block)  # type: ignore
 
     def _apply_nary_op(
         self,

bigframes/testing/utils.py

@@ -22,11 +22,13 @@
 from google.cloud.functions_v2.types import functions
 import numpy as np
 import pandas as pd
+import pandas.api.types as pd_types
 import pyarrow as pa  # type: ignore
 import pytest
 
 from bigframes import operations as ops
 from bigframes.core import expression as ex
+import bigframes.dtypes
 import bigframes.functions._utils as bff_utils
 import bigframes.pandas as bpd
 
@@ -98,7 +100,12 @@ def assert_pandas_df_equal(df0, df1, ignore_order: bool = False, **kwargs):
 
 
 def assert_series_equal(
-    left: pd.Series, right: pd.Series, ignore_order: bool = False, **kwargs
+    left: pd.Series,
+    right: pd.Series,
+    *,
+    ignore_order: bool = False,
+    nulls_are_nan: bool = True,
+    **kwargs,
 ):
     if ignore_order:
         if left.index.name is None:
@@ -108,6 +115,16 @@ def assert_series_equal(
             left = left.sort_index()
             right = right.sort_index()
 
+    if nulls_are_nan:
+        if left.dtype == bigframes.dtypes.FLOAT_DTYPE:
+            left = left.astype("float64")
+        if right.dtype == bigframes.dtypes.FLOAT_DTYPE:
+            right = right.astype("float64")
+        if pd_types.is_object_dtype(left):
+            left = left.fillna(float("nan"))
+        if pd_types.is_object_dtype(right):
+            right = right.fillna(float("nan"))
+
     pd.testing.assert_series_equal(left, right, **kwargs)
 
 

tests/system/small/test_dataframe.py

@@ -3547,7 +3547,8 @@ def test_dataframe_diff(scalars_df_index, scalars_pandas_df_index, periods):
 def test_dataframe_pct_change(scalars_df_index, scalars_pandas_df_index, periods):
     col_names = ["int64_too", "float64_col", "int64_col"]
     bf_result = scalars_df_index[col_names].pct_change(periods=periods).to_pandas()
-    pd_result = scalars_pandas_df_index[col_names].pct_change(periods=periods)
+    # pandas 3.0 does not automatically ffill anymore
+    pd_result = scalars_pandas_df_index[col_names].ffill().pct_change(periods=periods)
     pd.testing.assert_frame_equal(
         pd_result,
         bf_result,
@@ -3657,8 +3658,12 @@ def test_df_transpose():
     )
     rows_multi = pd.MultiIndex.from_arrays([index, index], names=["r1", "r2"])
 
-    pd_df = pandas.DataFrame(values, index=rows_multi, columns=columns_multi)
-    bf_df = dataframe.DataFrame(values, index=rows_multi, columns=columns_multi)
+    pd_df = pandas.DataFrame(
+        values, index=rows_multi, columns=columns_multi, dtype="Float64"
+    )
+    bf_df = dataframe.DataFrame(
+        values, index=rows_multi, columns=columns_multi, dtype="Float64"
+    )
 
     pd_result = pd_df.T
     bf_result = bf_df.T.to_pandas()
@@ -4376,10 +4381,8 @@ def test_dataframe_aggregates_axis_1(scalars_df_index, scalars_pandas_df_index,
     bf_result = op(scalars_df_index[col_names]).to_pandas()
     pd_result = op(scalars_pandas_df_index[col_names])
 
-    # Pandas may produce narrower numeric types, but bigframes always produces Float64
-    pd_result = pd_result.astype("Float64")
     # Pandas has object index type
-    pd.testing.assert_series_equal(pd_result, bf_result, check_index_type=False)
+    assert_series_equal(pd_result, bf_result, check_index_type=False, check_dtype=False)
 
 
 def test_dataframe_aggregates_median(scalars_df_index, scalars_pandas_df_index):

tests/system/small/test_series.py

@@ -801,6 +801,8 @@ def test_series_replace_dict(scalars_dfs, replacement_dict):
 )
 def test_series_interpolate(method):
     pytest.importorskip("scipy")
+    if method == "pad" and pd.__version__.startswith("3."):
+        pytest.skip("pandas 3.0 dropped method='pad'")
 
     values = [None, 1, 2, None, None, 16, None]
     index = [-3.2, 11.4, 3.56, 4, 4.32, 5.55, 76.8]
@@ -813,11 +815,12 @@ def test_series_interpolate(method):
     bf_result = bf_series.interpolate(method=method).to_pandas()
 
     # pd uses non-null types, while bf uses nullable types
-    pd.testing.assert_series_equal(
+    assert_series_equal(
         pd_result,
         bf_result,
         check_index_type=False,
         check_dtype=False,
+        nulls_are_nan=True,
     )
 
 
@@ -2730,7 +2733,7 @@ def test_diff(scalars_df_index, scalars_pandas_df_index, periods):
 def test_series_pct_change(scalars_df_index, scalars_pandas_df_index, periods):
     bf_result = scalars_df_index["int64_col"].pct_change(periods=periods).to_pandas()
     # cumsum does not behave well on nullable ints in pandas, produces object type and never ignores NA
-    pd_result = scalars_pandas_df_index["int64_col"].pct_change(periods=periods)
+    pd_result = scalars_pandas_df_index["int64_col"].ffill().pct_change(periods=periods)
 
     pd.testing.assert_series_equal(
         bf_result,

tests/unit/test_dataframe_polars.py

@@ -2687,7 +2687,8 @@ def test_dataframe_diff(scalars_df_index, scalars_pandas_df_index, periods):
 def test_dataframe_pct_change(scalars_df_index, scalars_pandas_df_index, periods):
     col_names = ["int64_too", "float64_col", "int64_col"]
     bf_result = scalars_df_index[col_names].pct_change(periods=periods).to_pandas()
-    pd_result = scalars_pandas_df_index[col_names].pct_change(periods=periods)
+    # pandas 3.0 does not automatically ffill anymore
+    pd_result = scalars_pandas_df_index[col_names].ffill().pct_change(periods=periods)
     pd.testing.assert_frame_equal(
         pd_result,
         bf_result,
@@ -2797,8 +2798,12 @@ def test_df_transpose():
     )
     rows_multi = pd.MultiIndex.from_arrays([index, index], names=["r1", "r2"])
 
-    pd_df = pandas.DataFrame(values, index=rows_multi, columns=columns_multi)
-    bf_df = dataframe.DataFrame(values, index=rows_multi, columns=columns_multi)
+    pd_df = pandas.DataFrame(
+        values, index=rows_multi, columns=columns_multi, dtype="Float64"
+    )
+    bf_df = dataframe.DataFrame(
+        values, index=rows_multi, columns=columns_multi, dtype="Float64"
+    )
 
     pd_result = pd_df.T
     bf_result = bf_df.T.to_pandas()
@@ -3386,9 +3391,8 @@ def test_dataframe_aggregates_axis_1(scalars_df_index, scalars_pandas_df_index,
     pd_result = op(scalars_pandas_df_index[col_names])
 
     # Pandas may produce narrower numeric types, but bigframes always produces Float64
-    pd_result = pd_result.astype("Float64")
     # Pandas has object index type
-    pd.testing.assert_series_equal(pd_result, bf_result, check_index_type=False)
+    assert_series_equal(pd_result, bf_result, check_index_type=False, check_dtype=False)
 
 
 @pytest.mark.parametrize(

tests/unit/test_series_polars.py

@@ -798,6 +798,8 @@ def test_series_replace_dict(scalars_dfs, replacement_dict):
 )
 def test_series_interpolate(method):
     pytest.importorskip("scipy")
+    if method == "pad" and pd.__version__.startswith("3."):
+        pytest.skip("pandas 3.0 dropped method='pad'")
 
     values = [None, 1, 2, None, None, 16, None]
     index = [-3.2, 11.4, 3.56, 4, 4.32, 5.55, 76.8]
@@ -810,11 +812,12 @@ def test_series_interpolate(method):
     bf_result = bf_series.interpolate(method=method).to_pandas()
 
     # pd uses non-null types, while bf uses nullable types
-    pd.testing.assert_series_equal(
+    assert_series_equal(
         pd_result,
         bf_result,
         check_index_type=False,
         check_dtype=False,
+        nulls_are_nan=True,
     )
 
 
@@ -2739,12 +2742,9 @@ def test_diff(scalars_df_index, scalars_pandas_df_index, periods):
 def test_series_pct_change(scalars_df_index, scalars_pandas_df_index, periods):
     bf_result = scalars_df_index["int64_col"].pct_change(periods=periods).to_pandas()
     # cumsum does not behave well on nullable ints in pandas, produces object type and never ignores NA
-    pd_result = scalars_pandas_df_index["int64_col"].pct_change(periods=periods)
+    pd_result = scalars_pandas_df_index["int64_col"].ffill().pct_change(periods=periods)
 
-    pd.testing.assert_series_equal(
-        bf_result,
-        pd_result,
-    )
+    assert_series_equal(bf_result, pd_result, nulls_are_nan=True)
 
 
 @pytest.mark.skip(
@@ -4696,7 +4696,7 @@ def wrapped(x):
 
     pd_result = pd_col.apply(wrapped)
 
-    assert_series_equal(bf_result, pd_result, check_dtype=False)
+    assert_series_equal(bf_result, pd_result, check_dtype=False, nulls_are_nan=True)
 
 
 @pytest.mark.parametrize(