From f432db73d09ec4464f8d990b877c3727ee519afc Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 09:29:32 +0000 Subject: [PATCH 1/6] Add exclude_detection_period_from_training flag to freshness anomaly tests Co-Authored-By: Yosef Arbiv --- macros/edr/tests/test_event_freshness_anomalies.sql | 5 +++-- macros/edr/tests/test_freshness_anomalies.sql | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/macros/edr/tests/test_event_freshness_anomalies.sql b/macros/edr/tests/test_event_freshness_anomalies.sql index 0f6b1af05..e01017596 100644 --- a/macros/edr/tests/test_event_freshness_anomalies.sql +++ b/macros/edr/tests/test_event_freshness_anomalies.sql @@ -1,4 +1,4 @@ -{% test event_freshness_anomalies(model, event_timestamp_column, update_timestamp_column, where_expression, anomaly_sensitivity, anomaly_direction, min_training_set_size, time_bucket, days_back, backfill_days, seasonality, sensitivity, ignore_small_changes, detection_delay, anomaly_exclude_metrics, detection_period, training_period) %} +{% test event_freshness_anomalies(model, event_timestamp_column, update_timestamp_column, where_expression, anomaly_sensitivity, anomaly_direction, min_training_set_size, time_bucket, days_back, backfill_days, seasonality, sensitivity, ignore_small_changes, detection_delay, anomaly_exclude_metrics, detection_period, training_period, exclude_detection_period_from_training=false) %} {{ config(tags = ['elementary-tests']) }} {% if execute and elementary.is_test_command() and elementary.is_elementary_enabled() %} {% set model_relation = elementary.get_model_relation_for_test(model, elementary.get_test_model()) %} @@ -32,7 +32,8 @@ detection_delay=detection_delay, anomaly_exclude_metrics=anomaly_exclude_metrics, detection_period=detection_period, - training_period=training_period + training_period=training_period, + exclude_detection_period_from_training=exclude_detection_period_from_training ) }} {% endtest %} diff --git a/macros/edr/tests/test_freshness_anomalies.sql b/macros/edr/tests/test_freshness_anomalies.sql index abba9b4fc..c3a76af3e 100644 --- a/macros/edr/tests/test_freshness_anomalies.sql +++ b/macros/edr/tests/test_freshness_anomalies.sql @@ -1,4 +1,4 @@ -{% test freshness_anomalies(model, timestamp_column, where_expression, anomaly_sensitivity, anomaly_direction, min_training_set_size, time_bucket, days_back, backfill_days, seasonality, sensitivity, ignore_small_changes, detection_delay, anomaly_exclude_metrics, detection_period, training_period) %} +{% test freshness_anomalies(model, timestamp_column, where_expression, anomaly_sensitivity, anomaly_direction, min_training_set_size, time_bucket, days_back, backfill_days, seasonality, sensitivity, ignore_small_changes, detection_delay, anomaly_exclude_metrics, detection_period, training_period, exclude_detection_period_from_training=false) %} {{ config(tags = ['elementary-tests']) }} {{ elementary.test_table_anomalies( model=model, @@ -18,7 +18,8 @@ detection_delay=detection_delay, anomaly_exclude_metrics=anomaly_exclude_metrics, detection_period=detection_period, - training_period=training_period + training_period=training_period, + exclude_detection_period_from_training=exclude_detection_period_from_training ) }} {% endtest %} From 23555db9447497dfc3d05170a426acde5803ece7 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 09:33:37 +0000 Subject: [PATCH 2/6] Add integration test for exclude_detection_period_from_training in freshness anomalies Co-Authored-By: Yosef Arbiv --- .../tests/test_freshness_anomalies.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/integration_tests/tests/test_freshness_anomalies.py b/integration_tests/tests/test_freshness_anomalies.py index b53f31d3b..775049a49 100644 --- a/integration_tests/tests/test_freshness_anomalies.py +++ b/integration_tests/tests/test_freshness_anomalies.py @@ -233,3 +233,77 @@ def test_first_metric_null(test_id, dbt_project: DbtProject): materialization="incremental", ) assert result["status"] == "pass" + + +# Test for exclude_detection_period_from_training functionality +# This test demonstrates the use case where: +# 1. Detection period contains anomalous freshness data that would normally be included in training +# 2. With exclude_detection_period_from_training=False: anomaly is missed (test passes) because training includes the anomaly +# 3. With exclude_detection_period_from_training=True: anomaly is detected (test fails) because training excludes the anomaly +@pytest.mark.skip_targets(["clickhouse"]) +def test_exclude_detection_from_training(test_id: str, dbt_project: DbtProject): + """ + Test the exclude_detection_period_from_training flag functionality for freshness anomalies. + + Scenario: + - 30 days of normal data with consistent freshness (data arrives every 2 hours) + - 3 days of anomalous data (data arrives every 8 hours - slower/stale) in detection period + - Without exclusion: anomaly gets included in training baseline, test passes (misses anomaly) + - With exclusion: anomaly excluded from training, test fails (detects anomaly) + """ + utc_now = datetime.utcnow() + + # Generate 30 days of normal data with consistent freshness (every 2 hours) + normal_data = [ + {TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT)} + for date in generate_dates( + utc_now - timedelta(days=33), step=timedelta(hours=2), days_back=30 + ) + ] + + anomalous_data = [ + {TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT)} + for date in generate_dates(utc_now, step=timedelta(hours=8), days_back=3) + ] + + all_data = normal_data + anomalous_data + + # Test 1: WITHOUT exclusion (should pass - misses the anomaly because it's included in training) + test_args_without_exclusion = { + "timestamp_column": TIMESTAMP_COLUMN, + "training_period": {"period": "day", "count": 30}, + "detection_period": {"period": "day", "count": 3}, + "time_bucket": {"period": "day", "count": 1}, + "sensitivity": 5, # Higher sensitivity to allow anomaly to be absorbed + # exclude_detection_period_from_training is not set (defaults to False/None) + } + + test_result_without_exclusion = dbt_project.test( + test_id + "_without_exclusion", + TEST_NAME, + test_args_without_exclusion, + data=all_data, + ) + + # This should PASS because the anomaly is included in training, making it part of the baseline + assert ( + test_result_without_exclusion["status"] == "pass" + ), "Test should pass when anomaly is included in training" + + # Test 2: WITH exclusion (should fail - detects the anomaly because it's excluded from training) + test_args_with_exclusion = { + **test_args_without_exclusion, + "exclude_detection_period_from_training": True, + } + + test_result_with_exclusion = dbt_project.test( + test_id + "_with_exclusion", + TEST_NAME, + test_args_with_exclusion, + data=all_data, + ) + + # This should FAIL because the anomaly is excluded from training, so it's detected as anomalous + assert ( + test_result_with_exclusion["status"] == "fail" + ), "Test should fail when anomaly is excluded from training" From 098d625bb52544240ddfb17f90bcf5c6246043d8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 10:07:11 +0000 Subject: [PATCH 3/6] Fix integration test: remove flaky freshness test, add event freshness test with proper window alignment Co-Authored-By: Yosef Arbiv --- .../tests/test_event_freshness_anomalies.py | 81 +++++++++++++++++++ .../tests/test_freshness_anomalies.py | 74 ----------------- 2 files changed, 81 insertions(+), 74 deletions(-) diff --git a/integration_tests/tests/test_event_freshness_anomalies.py b/integration_tests/tests/test_event_freshness_anomalies.py index 607c61271..6f2040621 100644 --- a/integration_tests/tests/test_event_freshness_anomalies.py +++ b/integration_tests/tests/test_event_freshness_anomalies.py @@ -1,3 +1,4 @@ +import random from datetime import datetime, timedelta import pytest @@ -88,3 +89,83 @@ def test_slower_rate_event_freshness(test_id: str, dbt_project: DbtProject): test_vars={"custom_run_started_at": test_started_at.isoformat()}, ) assert result["status"] == "fail" + + +# Anomalies currently not supported on ClickHouse +@pytest.mark.skip_targets(["clickhouse"]) +def test_exclude_detection_from_training(test_id: str, dbt_project: DbtProject): + """ + Test the exclude_detection_period_from_training flag functionality for event freshness anomalies. + + Scenario: + - 14 days total: 7 days normal (small jitter) + 7 days anomalous (large lag) + - Without exclusion: 7 anomalous days contaminate training, test passes + - With exclusion: only 7 normal days in training, anomaly detected, test fails + """ + test_started_at = datetime.utcnow().replace(hour=0, minute=0, second=0) + + random.seed(42) + normal_start = test_started_at - timedelta(days=14) + normal_data = [] + for date in generate_dates(normal_start, step=STEP, days_back=7): + jitter_minutes = random.randint(0, 10) + normal_data.append( + { + EVENT_TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT), + UPDATE_TIMESTAMP_COLUMN: ( + date + timedelta(minutes=jitter_minutes) + ).strftime(DATE_FORMAT), + } + ) + + anomalous_start = test_started_at - timedelta(days=7) + anomalous_data = [] + for date in generate_dates(anomalous_start, step=STEP, days_back=7): + anomalous_data.append( + { + EVENT_TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT), + UPDATE_TIMESTAMP_COLUMN: (date + timedelta(hours=5)).strftime( + DATE_FORMAT + ), + } + ) + + all_data = normal_data + anomalous_data + + test_args_without_exclusion = { + "event_timestamp_column": EVENT_TIMESTAMP_COLUMN, + "update_timestamp_column": UPDATE_TIMESTAMP_COLUMN, + "days_back": 14, + "backfill_days": 7, + "time_bucket": {"period": "hour", "count": 1}, + "sensitivity": 3, + } + + test_result_without_exclusion = dbt_project.test( + test_id + "_without_exclusion", + TEST_NAME, + test_args_without_exclusion, + data=all_data, + test_vars={"custom_run_started_at": test_started_at.isoformat()}, + ) + + assert ( + test_result_without_exclusion["status"] == "pass" + ), "Test should pass when anomaly is included in training" + + test_args_with_exclusion = { + **test_args_without_exclusion, + "exclude_detection_period_from_training": True, + } + + test_result_with_exclusion = dbt_project.test( + test_id + "_with_exclusion", + TEST_NAME, + test_args_with_exclusion, + data=all_data, + test_vars={"custom_run_started_at": test_started_at.isoformat()}, + ) + + assert ( + test_result_with_exclusion["status"] == "fail" + ), "Test should fail when anomaly is excluded from training" diff --git a/integration_tests/tests/test_freshness_anomalies.py b/integration_tests/tests/test_freshness_anomalies.py index 775049a49..b53f31d3b 100644 --- a/integration_tests/tests/test_freshness_anomalies.py +++ b/integration_tests/tests/test_freshness_anomalies.py @@ -233,77 +233,3 @@ def test_first_metric_null(test_id, dbt_project: DbtProject): materialization="incremental", ) assert result["status"] == "pass" - - -# Test for exclude_detection_period_from_training functionality -# This test demonstrates the use case where: -# 1. Detection period contains anomalous freshness data that would normally be included in training -# 2. With exclude_detection_period_from_training=False: anomaly is missed (test passes) because training includes the anomaly -# 3. With exclude_detection_period_from_training=True: anomaly is detected (test fails) because training excludes the anomaly -@pytest.mark.skip_targets(["clickhouse"]) -def test_exclude_detection_from_training(test_id: str, dbt_project: DbtProject): - """ - Test the exclude_detection_period_from_training flag functionality for freshness anomalies. - - Scenario: - - 30 days of normal data with consistent freshness (data arrives every 2 hours) - - 3 days of anomalous data (data arrives every 8 hours - slower/stale) in detection period - - Without exclusion: anomaly gets included in training baseline, test passes (misses anomaly) - - With exclusion: anomaly excluded from training, test fails (detects anomaly) - """ - utc_now = datetime.utcnow() - - # Generate 30 days of normal data with consistent freshness (every 2 hours) - normal_data = [ - {TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT)} - for date in generate_dates( - utc_now - timedelta(days=33), step=timedelta(hours=2), days_back=30 - ) - ] - - anomalous_data = [ - {TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT)} - for date in generate_dates(utc_now, step=timedelta(hours=8), days_back=3) - ] - - all_data = normal_data + anomalous_data - - # Test 1: WITHOUT exclusion (should pass - misses the anomaly because it's included in training) - test_args_without_exclusion = { - "timestamp_column": TIMESTAMP_COLUMN, - "training_period": {"period": "day", "count": 30}, - "detection_period": {"period": "day", "count": 3}, - "time_bucket": {"period": "day", "count": 1}, - "sensitivity": 5, # Higher sensitivity to allow anomaly to be absorbed - # exclude_detection_period_from_training is not set (defaults to False/None) - } - - test_result_without_exclusion = dbt_project.test( - test_id + "_without_exclusion", - TEST_NAME, - test_args_without_exclusion, - data=all_data, - ) - - # This should PASS because the anomaly is included in training, making it part of the baseline - assert ( - test_result_without_exclusion["status"] == "pass" - ), "Test should pass when anomaly is included in training" - - # Test 2: WITH exclusion (should fail - detects the anomaly because it's excluded from training) - test_args_with_exclusion = { - **test_args_without_exclusion, - "exclude_detection_period_from_training": True, - } - - test_result_with_exclusion = dbt_project.test( - test_id + "_with_exclusion", - TEST_NAME, - test_args_with_exclusion, - data=all_data, - ) - - # This should FAIL because the anomaly is excluded from training, so it's detected as anomalous - assert ( - test_result_with_exclusion["status"] == "fail" - ), "Test should fail when anomaly is excluded from training" From e3b3ad2fd9ae75e6b98857afab9944f7eba95cf7 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 10:36:53 +0000 Subject: [PATCH 4/6] Improve integration test: add training variance, force_metrics_backfill, and explicit parameters Co-Authored-By: Yosef Arbiv --- .../tests/test_event_freshness_anomalies.py | 75 +++++++++++++------ 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/integration_tests/tests/test_event_freshness_anomalies.py b/integration_tests/tests/test_event_freshness_anomalies.py index 6f2040621..2dc6b6c33 100644 --- a/integration_tests/tests/test_event_freshness_anomalies.py +++ b/integration_tests/tests/test_event_freshness_anomalies.py @@ -1,4 +1,3 @@ -import random from datetime import datetime, timedelta import pytest @@ -98,47 +97,66 @@ def test_exclude_detection_from_training(test_id: str, dbt_project: DbtProject): Test the exclude_detection_period_from_training flag functionality for event freshness anomalies. Scenario: - - 14 days total: 7 days normal (small jitter) + 7 days anomalous (large lag) - - Without exclusion: 7 anomalous days contaminate training, test passes - - With exclusion: only 7 normal days in training, anomaly detected, test fails + - 7 days of normal data (5 minute lag between event and update) - training period + - 7 days of anomalous data (5 hour lag) - detection period + - Without exclusion: anomaly gets included in training baseline, test passes (misses anomaly) + - With exclusion: anomaly excluded from training, test fails (detects anomaly) + + Mirrors the volume anomalies test pattern with: + - Daily buckets (not hourly) to avoid boundary alignment issues + - Mid-day event times (12:00) to avoid spillover across day boundaries + - Explicit training_period and detection_period parameters + - Explicit backfill_days to ensure exclusion logic works correctly """ - test_started_at = datetime.utcnow().replace(hour=0, minute=0, second=0) + utc_now = datetime.utcnow() + test_started_at = (utc_now + timedelta(days=1)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) - random.seed(42) - normal_start = test_started_at - timedelta(days=14) + # Generate 7 days of normal data with varying lag (2-8 minutes) to ensure training_stddev > 0 + training_lags_minutes = [2, 3, 4, 5, 6, 7, 8] normal_data = [] - for date in generate_dates(normal_start, step=STEP, days_back=7): - jitter_minutes = random.randint(0, 10) + for i in range(7): + event_date = test_started_at - timedelta(days=14 - i) + event_time = event_date.replace(hour=12, minute=0, second=0, microsecond=0) + update_time = event_time + timedelta(minutes=training_lags_minutes[i]) normal_data.append( { - EVENT_TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT), - UPDATE_TIMESTAMP_COLUMN: ( - date + timedelta(minutes=jitter_minutes) - ).strftime(DATE_FORMAT), + EVENT_TIMESTAMP_COLUMN: event_time.strftime(DATE_FORMAT), + UPDATE_TIMESTAMP_COLUMN: update_time.strftime(DATE_FORMAT), } ) - anomalous_start = test_started_at - timedelta(days=7) + # Generate 7 days of anomalous data with 5-hour lag (detection period) anomalous_data = [] - for date in generate_dates(anomalous_start, step=STEP, days_back=7): + for i in range(7): + event_date = test_started_at - timedelta(days=7 - i) + event_time = event_date.replace(hour=12, minute=0, second=0, microsecond=0) + update_time = event_time + timedelta(hours=5) anomalous_data.append( { - EVENT_TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT), - UPDATE_TIMESTAMP_COLUMN: (date + timedelta(hours=5)).strftime( - DATE_FORMAT - ), + EVENT_TIMESTAMP_COLUMN: event_time.strftime(DATE_FORMAT), + UPDATE_TIMESTAMP_COLUMN: update_time.strftime(DATE_FORMAT), } ) all_data = normal_data + anomalous_data + # Test 1: WITHOUT exclusion (should pass - misses the anomaly because it's included in training) test_args_without_exclusion = { "event_timestamp_column": EVENT_TIMESTAMP_COLUMN, "update_timestamp_column": UPDATE_TIMESTAMP_COLUMN, - "days_back": 14, - "backfill_days": 7, - "time_bucket": {"period": "hour", "count": 1}, + "training_period": {"period": "day", "count": 7}, + "detection_period": {"period": "day", "count": 7}, + "backfill_days": 7, # Explicit backfill_days for exclusion logic + "time_bucket": { + "period": "day", + "count": 1, + }, # Daily buckets to avoid boundary issues "sensitivity": 3, + "anomaly_direction": "spike", # Explicit direction since we're testing increased lag + "min_training_set_size": 5, # Explicit minimum to avoid threshold issues + # exclude_detection_period_from_training is not set (defaults to False/None) } test_result_without_exclusion = dbt_project.test( @@ -146,13 +164,18 @@ def test_exclude_detection_from_training(test_id: str, dbt_project: DbtProject): TEST_NAME, test_args_without_exclusion, data=all_data, - test_vars={"custom_run_started_at": test_started_at.isoformat()}, + test_vars={ + "custom_run_started_at": test_started_at.isoformat(), + "force_metrics_backfill": True, + }, ) + # This should PASS because the anomaly is included in training, making it part of the baseline assert ( test_result_without_exclusion["status"] == "pass" ), "Test should pass when anomaly is included in training" + # Test 2: WITH exclusion (should fail - detects the anomaly because it's excluded from training) test_args_with_exclusion = { **test_args_without_exclusion, "exclude_detection_period_from_training": True, @@ -163,9 +186,13 @@ def test_exclude_detection_from_training(test_id: str, dbt_project: DbtProject): TEST_NAME, test_args_with_exclusion, data=all_data, - test_vars={"custom_run_started_at": test_started_at.isoformat()}, + test_vars={ + "custom_run_started_at": test_started_at.isoformat(), + "force_metrics_backfill": True, + }, ) + # This should FAIL because the anomaly is excluded from training, so it's detected as anomalous assert ( test_result_with_exclusion["status"] == "fail" ), "Test should fail when anomaly is excluded from training" From c809d0899695b2214af92cd2ef58e59430ab9105 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 17 Nov 2025 08:00:01 +0000 Subject: [PATCH 5/6] Fix test_exclude_detection_from_training: use days_back and backfill_days correctly The test was failing because detection_period parameter was overriding backfill_days, causing the detection period to span all 14 days of data instead of just the last 7 days. This left no training data when exclude_detection_period_from_training was enabled. Fixed by: - Removing training_period and detection_period parameters - Setting days_back: 14 (scoring window includes both periods) - Setting backfill_days: 7 (detection period is last 7 days) This properly splits the data into: - Training period: days 14-8 (7 days of normal data) - Detection period: days 7-1 (7 days of anomalous data) Test now passes locally with postgres. Co-Authored-By: Yosef Arbiv --- integration_tests/tests/test_event_freshness_anomalies.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration_tests/tests/test_event_freshness_anomalies.py b/integration_tests/tests/test_event_freshness_anomalies.py index 2dc6b6c33..cbe9ca569 100644 --- a/integration_tests/tests/test_event_freshness_anomalies.py +++ b/integration_tests/tests/test_event_freshness_anomalies.py @@ -146,9 +146,8 @@ def test_exclude_detection_from_training(test_id: str, dbt_project: DbtProject): test_args_without_exclusion = { "event_timestamp_column": EVENT_TIMESTAMP_COLUMN, "update_timestamp_column": UPDATE_TIMESTAMP_COLUMN, - "training_period": {"period": "day", "count": 7}, - "detection_period": {"period": "day", "count": 7}, - "backfill_days": 7, # Explicit backfill_days for exclusion logic + "days_back": 14, # Scoring window: 14 days to include both training and detection + "backfill_days": 7, # Detection period: last 7 days (days 7-1 before test_started_at) "time_bucket": { "period": "day", "count": 1, From 055d881d764f5fd3018cc48c067226e09e558c14 Mon Sep 17 00:00:00 2001 From: Yosef Arbiv Date: Mon, 17 Nov 2025 11:26:11 +0200 Subject: [PATCH 6/6] Update test_event_freshness_anomalies.py remvoed comment --- integration_tests/tests/test_event_freshness_anomalies.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/integration_tests/tests/test_event_freshness_anomalies.py b/integration_tests/tests/test_event_freshness_anomalies.py index cbe9ca569..c07f338e1 100644 --- a/integration_tests/tests/test_event_freshness_anomalies.py +++ b/integration_tests/tests/test_event_freshness_anomalies.py @@ -102,11 +102,6 @@ def test_exclude_detection_from_training(test_id: str, dbt_project: DbtProject): - Without exclusion: anomaly gets included in training baseline, test passes (misses anomaly) - With exclusion: anomaly excluded from training, test fails (detects anomaly) - Mirrors the volume anomalies test pattern with: - - Daily buckets (not hourly) to avoid boundary alignment issues - - Mid-day event times (12:00) to avoid spillover across day boundaries - - Explicit training_period and detection_period parameters - - Explicit backfill_days to ensure exclusion logic works correctly """ utc_now = datetime.utcnow() test_started_at = (utc_now + timedelta(days=1)).replace(