diff --git a/integration_tests/tests/test_dbt_artifacts/test_artifacts.py b/integration_tests/tests/test_dbt_artifacts/test_artifacts.py index 44b048544..defb3a665 100644 --- a/integration_tests/tests/test_dbt_artifacts/test_artifacts.py +++ b/integration_tests/tests/test_dbt_artifacts/test_artifacts.py @@ -17,6 +17,8 @@ def test_artifacts_caching(dbt_project: DbtProject): # Disabled by default in the tests for performance reasons. dbt_project.dbt_runner.vars["disable_dbt_artifacts_autoupload"] = False + dbt_project.dbt_runner.vars["cache_artifacts"] = True + dbt_project.dbt_runner.run(select=TEST_MODEL, vars={"one_tags": ["hello", "world"]}) first_row = read_model_artifact_row(dbt_project) dbt_project.dbt_runner.run(select=TEST_MODEL, vars={"one_tags": ["world", "hello"]}) @@ -29,6 +31,8 @@ def test_artifacts_updating(dbt_project: DbtProject): # Disabled by default in the tests for performance reasons. dbt_project.dbt_runner.vars["disable_dbt_artifacts_autoupload"] = False + dbt_project.dbt_runner.vars["cache_artifacts"] = True + dbt_project.dbt_runner.run(select=TEST_MODEL) first_row = read_model_artifact_row(dbt_project) dbt_project.dbt_runner.run(select=TEST_MODEL, vars={"one_owner": "ele"}) diff --git a/macros/edr/system/system_utils/get_config_var.sql b/macros/edr/system/system_utils/get_config_var.sql index 97da70f01..20b53da80 100644 --- a/macros/edr/system/system_utils/get_config_var.sql +++ b/macros/edr/system/system_utils/get_config_var.sql @@ -114,5 +114,12 @@ {%- macro dremio__get_default_config() -%} {% set default_config = elementary.default__get_default_config() %} {% do default_config.update({'dbt_artifacts_chunk_size': 100}) %} + + {# Caching does work in Dremio, but there is a race between the creation of the temp table + and its usage, and it's causing failures (querying the same table 2 seconds later works). + This is likely a bug in Dremio. + So to be safe we disable caching in Dremio by default. #} + {% do default_config.update({'cache_artifacts': false}) %} + {{- return(default_config) -}} {%- endmacro -%} diff --git a/macros/utils/table_operations/replace_table_data.sql b/macros/utils/table_operations/replace_table_data.sql index 1ed641769..0bf8e4dcf 100644 --- a/macros/utils/table_operations/replace_table_data.sql +++ b/macros/utils/table_operations/replace_table_data.sql @@ -17,9 +17,15 @@ {# Spark - truncate and insert (non-atomic) #} {% macro spark__replace_table_data(relation, rows) %} - {% call statement('truncate_relation') -%} + {% call statement('truncate_relation') -%} delete from {{ relation }} where 1=1 - {%- endcall %} + {%- endcall %} + {% do elementary.insert_rows(relation, rows, should_commit=false, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %} +{% endmacro %} + +{# Dremio - truncate and insert (non-atomic) #} +{% macro dremio__replace_table_data(relation, rows) %} + {% do dbt.truncate_relation(relation) %} {% do elementary.insert_rows(relation, rows, should_commit=false, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %} {% endmacro %}