From f5bea3346f259cff3ed96876b2193f0ddd5b7b28 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Sat, 27 Dec 2025 21:27:24 +0000
Subject: [PATCH] Add z-score fallback metric tracking for anomaly detection
 tests

- Add is_zscore_fallback field to anomaly scores query (when training_stddev = 0)
- Propagate is_zscore_fallback through get_read_anomaly_scores_query
- Track zscore_fallback_passed_count and zscore_fallback_failed_count in test results
- Add new fields to elementary_test_results schema

This enables tracking the number of tests that use z-score fallback
(when all training values are identical), with distinction between
passed and failed tests.

Co-Authored-By: Yosef Arbiv <yosef.arbiv@gmail.com>
---
 .../anomaly_detection/get_anomaly_scores_query.sql  |  4 ++++
 .../store_anomaly_test_results.sql                  | 13 ++++++++++++-
 macros/edr/system/system_utils/empty_table.sql      |  4 +++-
 macros/edr/tests/test_utils/get_anomaly_query.sql   |  1 +
 4 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql b/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql
index 3ff296f5f..b7dbcbca9 100644
--- a/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql
+++ b/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql
@@ -231,6 +231,10 @@
                     when training_stddev = 0 then 0  -- Stationary data case - valid, all values are identical
                     else (metric_value - training_avg) / (training_stddev)
                 end as anomaly_score,
+                case
+                    when training_stddev is not null and training_set_size > 1 and training_stddev = 0 then TRUE
+                    else FALSE
+                end as is_zscore_fallback,
                 {{ test_configuration.anomaly_sensitivity }} as anomaly_score_threshold,
                 source_value as anomalous_value,
                 {{ elementary.edr_cast_as_timestamp('bucket_start') }} as bucket_start,
diff --git a/macros/edr/data_monitoring/anomaly_detection/store_anomaly_test_results.sql b/macros/edr/data_monitoring/anomaly_detection/store_anomaly_test_results.sql
index 2bcd60cce..e1631591b 100644
--- a/macros/edr/data_monitoring/anomaly_detection/store_anomaly_test_results.sql
+++ b/macros/edr/data_monitoring/anomaly_detection/store_anomaly_test_results.sql
@@ -48,12 +48,21 @@
       {% endif %}
   {% endset %}
   {% set failures = namespace(data=0) %}
+  {% set zscore_fallback_passed = namespace(data=0) %}
+  {% set zscore_fallback_failed = namespace(data=0) %}
   {% set filtered_anomaly_scores_rows = [] %}
   {% for row in anomaly_scores_rows %}
     {% if row.anomaly_score is not none %}
       {% do filtered_anomaly_scores_rows.append(row) %}
       {% if row.is_anomalous %}
         {% set failures.data = failures.data + 1 %}
+        {% if elementary.insensitive_get_dict_value(row, 'is_zscore_fallback') %}
+          {% set zscore_fallback_failed.data = zscore_fallback_failed.data + 1 %}
+        {% endif %}
+      {% else %}
+        {% if elementary.insensitive_get_dict_value(row, 'is_zscore_fallback') %}
+          {% set zscore_fallback_passed.data = zscore_fallback_passed.data + 1 %}
+        {% endif %}
       {% endif %}
     {% endif %}
   {% endfor %}
@@ -69,7 +78,9 @@
       'test_results_query': test_results_query,
       'test_params': test_params,
       'result_rows': filtered_anomaly_scores_rows,
-      'failures': failures.data
+      'failures': failures.data,
+      'zscore_fallback_passed_count': zscore_fallback_passed.data,
+      'zscore_fallback_failed_count': zscore_fallback_failed.data
   } %}
   {% set elementary_test_row = elementary.get_dbt_test_result_row(flattened_test) %}
   {% do elementary_test_row.update(test_result_dict) %}
diff --git a/macros/edr/system/system_utils/empty_table.sql b/macros/edr/system/system_utils/empty_table.sql
index 07ccc3e4d..3df6ae7a4 100644
--- a/macros/edr/system/system_utils/empty_table.sql
+++ b/macros/edr/system/system_utils/empty_table.sql
@@ -27,7 +27,9 @@
     ('test_short_name', 'string'),
     ('test_alias', 'string'),
     ('result_rows', 'long_string'),
-    ('failed_row_count', 'bigint')
+    ('failed_row_count', 'bigint'),
+    ('zscore_fallback_passed_count', 'bigint'),
+    ('zscore_fallback_failed_count', 'bigint')
     ]) }}
 {% endmacro %}
 
diff --git a/macros/edr/tests/test_utils/get_anomaly_query.sql b/macros/edr/tests/test_utils/get_anomaly_query.sql
index 6da0f34fc..9564b0f54 100644
--- a/macros/edr/tests/test_utils/get_anomaly_query.sql
+++ b/macros/edr/tests/test_utils/get_anomaly_query.sql
@@ -43,6 +43,7 @@
             column_name,
             metric_name,
             anomaly_score,
+            is_zscore_fallback,
             anomaly_score_threshold,
             anomalous_value,
             bucket_start,