From 536513c274656ecebc11dca5801c397910146a1f Mon Sep 17 00:00:00 2001 From: Itamar Hartstein Date: Wed, 8 Oct 2025 10:49:05 +0300 Subject: [PATCH 1/4] replace_table_data: bugfix for dremio --- macros/utils/table_operations/replace_table_data.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/macros/utils/table_operations/replace_table_data.sql b/macros/utils/table_operations/replace_table_data.sql index 1ed641769..0bf8e4dcf 100644 --- a/macros/utils/table_operations/replace_table_data.sql +++ b/macros/utils/table_operations/replace_table_data.sql @@ -17,9 +17,15 @@ {# Spark - truncate and insert (non-atomic) #} {% macro spark__replace_table_data(relation, rows) %} - {% call statement('truncate_relation') -%} + {% call statement('truncate_relation') -%} delete from {{ relation }} where 1=1 - {%- endcall %} + {%- endcall %} + {% do elementary.insert_rows(relation, rows, should_commit=false, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %} +{% endmacro %} + +{# Dremio - truncate and insert (non-atomic) #} +{% macro dremio__replace_table_data(relation, rows) %} + {% do dbt.truncate_relation(relation) %} {% do elementary.insert_rows(relation, rows, should_commit=false, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %} {% endmacro %} From b7d4d7fab5a78c450878607dd485aa6b482be6bd Mon Sep 17 00:00:00 2001 From: Itamar Hartstein Date: Wed, 8 Oct 2025 10:53:00 +0300 Subject: [PATCH 2/4] get_config_var: disable artifacts caching for dremio --- macros/edr/system/system_utils/get_config_var.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/macros/edr/system/system_utils/get_config_var.sql b/macros/edr/system/system_utils/get_config_var.sql index 97da70f01..c3040cd61 100644 --- a/macros/edr/system/system_utils/get_config_var.sql +++ b/macros/edr/system/system_utils/get_config_var.sql @@ -113,6 +113,9 @@ {%- macro dremio__get_default_config() -%} {% set default_config = elementary.default__get_default_config() %} - {% do default_config.update({'dbt_artifacts_chunk_size': 100}) %} + {% do default_config.update({ + 'dbt_artifacts_chunk_size': 100, + 'cache_artifacts': false + }) %} {{- return(default_config) -}} {%- endmacro -%} From c9d2a47db8447bcf4708ea2174720b38410ba591 Mon Sep 17 00:00:00 2001 From: Itamar Hartstein Date: Wed, 8 Oct 2025 11:20:52 +0300 Subject: [PATCH 3/4] test_artifacts: explicitly set cache_artifacts to True for tests that need it --- integration_tests/tests/test_dbt_artifacts/test_artifacts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration_tests/tests/test_dbt_artifacts/test_artifacts.py b/integration_tests/tests/test_dbt_artifacts/test_artifacts.py index 44b048544..defb3a665 100644 --- a/integration_tests/tests/test_dbt_artifacts/test_artifacts.py +++ b/integration_tests/tests/test_dbt_artifacts/test_artifacts.py @@ -17,6 +17,8 @@ def test_artifacts_caching(dbt_project: DbtProject): # Disabled by default in the tests for performance reasons. dbt_project.dbt_runner.vars["disable_dbt_artifacts_autoupload"] = False + dbt_project.dbt_runner.vars["cache_artifacts"] = True + dbt_project.dbt_runner.run(select=TEST_MODEL, vars={"one_tags": ["hello", "world"]}) first_row = read_model_artifact_row(dbt_project) dbt_project.dbt_runner.run(select=TEST_MODEL, vars={"one_tags": ["world", "hello"]}) @@ -29,6 +31,8 @@ def test_artifacts_updating(dbt_project: DbtProject): # Disabled by default in the tests for performance reasons. dbt_project.dbt_runner.vars["disable_dbt_artifacts_autoupload"] = False + dbt_project.dbt_runner.vars["cache_artifacts"] = True + dbt_project.dbt_runner.run(select=TEST_MODEL) first_row = read_model_artifact_row(dbt_project) dbt_project.dbt_runner.run(select=TEST_MODEL, vars={"one_owner": "ele"}) From 75fa9681ab677651f9aedf9bb5f403d22057943a Mon Sep 17 00:00:00 2001 From: Itamar Hartstein Date: Wed, 8 Oct 2025 11:25:55 +0300 Subject: [PATCH 4/4] add comment --- macros/edr/system/system_utils/get_config_var.sql | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/macros/edr/system/system_utils/get_config_var.sql b/macros/edr/system/system_utils/get_config_var.sql index c3040cd61..20b53da80 100644 --- a/macros/edr/system/system_utils/get_config_var.sql +++ b/macros/edr/system/system_utils/get_config_var.sql @@ -113,9 +113,13 @@ {%- macro dremio__get_default_config() -%} {% set default_config = elementary.default__get_default_config() %} - {% do default_config.update({ - 'dbt_artifacts_chunk_size': 100, - 'cache_artifacts': false - }) %} + {% do default_config.update({'dbt_artifacts_chunk_size': 100}) %} + + {# Caching does work in Dremio, but there is a race between the creation of the temp table + and its usage, and it's causing failures (querying the same table 2 seconds later works). + This is likely a bug in Dremio. + So to be safe we disable caching in Dremio by default. #} + {% do default_config.update({'cache_artifacts': false}) %} + {{- return(default_config) -}} {%- endmacro -%}