Merge branch 'main' into refactor-isnull-op

Author: tswast

bigframes/ml/metrics/__init__.py

@@ -18,6 +18,7 @@
     auc,
     confusion_matrix,
     f1_score,
+    mean_absolute_error,
     mean_squared_error,
     precision_score,
     r2_score,
@@ -36,6 +37,7 @@
     "confusion_matrix",
     "precision_score",
     "f1_score",
+    "mean_absolute_error",
     "mean_squared_error",
     "pairwise",
 ]

bigframes/ml/metrics/_metrics.py

@@ -344,3 +344,17 @@ def mean_squared_error(
 mean_squared_error.__doc__ = inspect.getdoc(
     vendored_metrics_regression.mean_squared_error
 )
+
+
+def mean_absolute_error(
+    y_true: Union[bpd.DataFrame, bpd.Series],
+    y_pred: Union[bpd.DataFrame, bpd.Series],
+) -> float:
+    y_true_series, y_pred_series = utils.batch_convert_to_series(y_true, y_pred)
+
+    return (y_pred_series - y_true_series).abs().sum() / len(y_true_series)
+
+
+mean_absolute_error.__doc__ = inspect.getdoc(
+    vendored_metrics_regression.mean_absolute_error
+)

bigframes/operations/datetimes.py

@@ -49,10 +49,18 @@ def day(self) -> series.Series:
     def dayofweek(self) -> series.Series:
         return self._apply_unary_op(ops.dayofweek_op)
 
+    @property
+    def day_of_week(self) -> series.Series:
+        return self.dayofweek
+
     @property
     def dayofyear(self) -> series.Series:
         return self._apply_unary_op(ops.dayofyear_op)
 
+    @property
+    def day_of_year(self) -> series.Series:
+        return self.dayofyear
+
     @property
     def date(self) -> series.Series:
         return self._apply_unary_op(ops.date_op)

bigframes/session/loader.py

@@ -983,6 +983,8 @@ def read_gbq_query(
             )
             job_config.dry_run = True
             query_job = self._bqclient.query(query, job_config=job_config)
+            if self._metrics is not None:
+                self._metrics.count_job_stats(query_job=query_job)
             return dry_runs.get_query_stats_with_inferred_dtypes(
                 query_job, list(columns), index_cols
             )

bigframes/session/metrics.py

@@ -51,6 +51,9 @@ def count_job_stats(
             write_stats_to_disk(len(query), total_bytes_processed)
             return
 
+        if query_job.configuration.dry_run:
+            write_stats_to_disk(len(query_job.query), 0, 0, 0)
+
         stats = get_performance_stats(query_job)
         if stats is not None:
             query_char_count, bytes_processed, slot_millis, execution_secs = stats

scripts/run_and_publish_benchmark.py

@@ -100,7 +100,12 @@ def collect_benchmark_result(
             == len(local_seconds_files)
         ):
             raise ValueError(
-                "Mismatch in the number of report files for bytes, millis, seconds and query char count."
+                "Mismatch in the number of report files for bytes, millis, seconds and query char count: \n"
+                f"millis_files: {len(millis_files)}\n"
+                f"bq_seconds_files: {len(bq_seconds_files)}\n"
+                f"bytes_files: {len(bytes_files)}\n"
+                f"query_char_count_files: {len(query_char_count_files)}\n"
+                f"local_seconds_files: {len(local_seconds_files)}\n"
             )
 
         has_full_metrics = len(bq_seconds_files) == len(local_seconds_files)

tests/system/small/ml/test_metrics.py

@@ -818,3 +818,10 @@ def test_mean_squared_error(session: bigframes.Session):
     df = session.read_pandas(pd_df)
     mse = metrics.mean_squared_error(df["y_true"], df["y_pred"])
     assert mse == 0.375
+
+
+def test_mean_absolute_error(session: bigframes.Session):
+    pd_df = pd.DataFrame({"y_true": [3, -0.5, 2, 7], "y_pred": [2.5, 0.0, 2, 8]})
+    df = session.read_pandas(pd_df)
+    mse = metrics.mean_absolute_error(df["y_true"], df["y_pred"])
+    assert mse == 0.5

tests/system/small/operations/test_datetimes.py

@@ -86,12 +86,28 @@ def test_dt_dayofweek(scalars_dfs, col_name):
     pytest.importorskip("pandas", minversion="2.0.0")
     scalars_df, scalars_pandas_df = scalars_dfs
     bf_series: bigframes.series.Series = scalars_df[col_name]
+
     bf_result = bf_series.dt.dayofweek.to_pandas()
     pd_result = scalars_pandas_df[col_name].dt.dayofweek
 
     assert_series_equal(pd_result, bf_result, check_dtype=False)
 
 
+@pytest.mark.parametrize(
+    ("col_name",),
+    DATE_COLUMNS,
+)
+def test_dt_day_of_week(scalars_dfs, col_name):
+    pytest.importorskip("pandas", minversion="2.0.0")
+    scalars_df, scalars_pandas_df = scalars_dfs
+    bf_series: bigframes.series.Series = scalars_df[col_name]
+
+    bf_result = bf_series.dt.day_of_week.to_pandas()
+    pd_result = scalars_pandas_df[col_name].dt.day_of_week
+
+    assert_series_equal(pd_result, bf_result, check_dtype=False)
+
+
 @pytest.mark.parametrize(
     ("col_name",),
     DATE_COLUMNS,
@@ -100,12 +116,28 @@ def test_dt_dayofyear(scalars_dfs, col_name):
     pytest.importorskip("pandas", minversion="2.0.0")
     scalars_df, scalars_pandas_df = scalars_dfs
     bf_series: bigframes.series.Series = scalars_df[col_name]
+
     bf_result = bf_series.dt.dayofyear.to_pandas()
     pd_result = scalars_pandas_df[col_name].dt.dayofyear
 
     assert_series_equal(pd_result, bf_result, check_dtype=False)
 
 
+@pytest.mark.parametrize(
+    ("col_name",),
+    DATE_COLUMNS,
+)
+def test_dt_day_of_year(scalars_dfs, col_name):
+    pytest.importorskip("pandas", minversion="2.0.0")
+    scalars_df, scalars_pandas_df = scalars_dfs
+    bf_series: bigframes.series.Series = scalars_df[col_name]
+
+    bf_result = bf_series.dt.day_of_year.to_pandas()
+    pd_result = scalars_pandas_df[col_name].dt.day_of_year
+
+    assert_series_equal(pd_result, bf_result, check_dtype=False)
+
+
 @pytest.mark.parametrize(
     ("col_name",),
     DATETIME_COL_NAMES,

third_party/bigframes_vendored/pandas/core/indexes/accessor.py

@@ -66,6 +66,40 @@ def dayofweek(self):
 
         raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE)
 
+    @property
+    def day_of_week(self):
+        """The day of the week with Monday=0, Sunday=6.
+
+        Return the day of the week. It is assumed the week starts on
+        Monday, which is denoted by 0 and ends on Sunday, which is denoted
+        by 6.
+
+        **Examples:**
+
+            >>> import pandas as pd
+            >>> import bigframes.pandas as bpd
+            >>> bpd.options.display.progress_bar = None
+            >>> s = bpd.Series(
+            ...     pd.date_range('2016-12-31', '2017-01-08', freq='D').to_series()
+            ... )
+            >>> s.dt.day_of_week
+            2016-12-31 00:00:00    5
+            2017-01-01 00:00:00    6
+            2017-01-02 00:00:00    0
+            2017-01-03 00:00:00    1
+            2017-01-04 00:00:00    2
+            2017-01-05 00:00:00    3
+            2017-01-06 00:00:00    4
+            2017-01-07 00:00:00    5
+            2017-01-08 00:00:00    6
+            dtype: Int64
+
+        Returns:
+            Series: Containing integers indicating the day number.
+        """
+
+        raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE)
+
     @property
     def dayofyear(self):
         """The ordinal day of the year.
@@ -94,6 +128,34 @@ def dayofyear(self):
 
         raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE)
 
+    @property
+    def day_of_year(self):
+        """The ordinal day of the year.
+
+        **Examples:**
+
+            >>> import pandas as pd
+            >>> import bigframes.pandas as bpd
+            >>> bpd.options.display.progress_bar = None
+            >>> s = bpd.Series(
+            ...     pd.date_range('2016-12-28', '2017-01-03', freq='D').to_series()
+            ... )
+            >>> s.dt.day_of_year
+            2016-12-28 00:00:00    363
+            2016-12-29 00:00:00    364
+            2016-12-30 00:00:00    365
+            2016-12-31 00:00:00    366
+            2017-01-01 00:00:00      1
+            2017-01-02 00:00:00      2
+            2017-01-03 00:00:00      3
+            dtype: Int64
+
+        Returns:
+            Series: Containing integers indicating the day number.
+        """
+
+        raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE)
+
     @property
     def date(self):
         """Returns a Series with the date part of Timestamps without time and

third_party/bigframes_vendored/sklearn/metrics/_regression.py

@@ -91,3 +91,30 @@ def mean_squared_error(y_true, y_pred) -> float:
         float: Mean squared error.
     """
     raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE)
+
+
+def mean_absolute_error(y_true, y_pred) -> float:
+    """Mean absolute error regression loss.
+
+    **Examples:**
+
+        >>> import bigframes.pandas as bpd
+        >>> import bigframes.ml.metrics
+        >>> bpd.options.display.progress_bar = None
+
+        >>> y_true = bpd.DataFrame([3, -0.5, 2, 7])
+        >>> y_pred = bpd.DataFrame([2.5, 0.0, 2, 8])
+        >>> mae = bigframes.ml.metrics.mean_absolute_error(y_true, y_pred)
+        >>> mae
+        np.float64(0.5)
+
+    Args:
+        y_true (Series or DataFrame of shape (n_samples,)):
+            Ground truth (correct) target values.
+        y_pred (Series or DataFrame of shape (n_samples,)):
+            Estimated target values.
+
+    Returns:
+        float: Mean absolute error.
+    """
+    raise NotImplementedError(constants.ABSTRACT_METHOD_ERROR_MESSAGE)