Ele 5121 dremio fix for replace table data

integration_tests/tests/test_dbt_artifacts/test_artifacts.py

@@ -17,6 +17,8 @@ def test_artifacts_caching(dbt_project: DbtProject):
     # Disabled by default in the tests for performance reasons.
     dbt_project.dbt_runner.vars["disable_dbt_artifacts_autoupload"] = False
 
+    dbt_project.dbt_runner.vars["cache_artifacts"] = True
+
     dbt_project.dbt_runner.run(select=TEST_MODEL, vars={"one_tags": ["hello", "world"]})
     first_row = read_model_artifact_row(dbt_project)
     dbt_project.dbt_runner.run(select=TEST_MODEL, vars={"one_tags": ["world", "hello"]})
@@ -29,6 +31,8 @@ def test_artifacts_updating(dbt_project: DbtProject):
     # Disabled by default in the tests for performance reasons.
     dbt_project.dbt_runner.vars["disable_dbt_artifacts_autoupload"] = False
 
+    dbt_project.dbt_runner.vars["cache_artifacts"] = True
+
     dbt_project.dbt_runner.run(select=TEST_MODEL)
     first_row = read_model_artifact_row(dbt_project)
     dbt_project.dbt_runner.run(select=TEST_MODEL, vars={"one_owner": "ele"})

macros/edr/system/system_utils/get_config_var.sql

@@ -114,5 +114,12 @@
 {%- macro dremio__get_default_config() -%}
     {% set default_config = elementary.default__get_default_config() %}
     {% do default_config.update({'dbt_artifacts_chunk_size': 100}) %}
+
+    {# Caching does work in Dremio, but there is a race between the creation of the temp table
+       and its usage, and it's causing failures (querying the same table 2 seconds later works).
+       This is likely a bug in Dremio.
+       So to be safe we disable caching in Dremio by default. #}
+    {% do default_config.update({'cache_artifacts': false}) %}
+
     {{- return(default_config) -}}
 {%- endmacro -%}

macros/utils/table_operations/replace_table_data.sql

@@ -17,9 +17,15 @@
 
 {# Spark - truncate and insert (non-atomic) #}
 {% macro spark__replace_table_data(relation, rows) %}
-     {% call statement('truncate_relation') -%}
+    {% call statement('truncate_relation') -%}
         delete from {{ relation }} where 1=1
-      {%- endcall %}
+    {%- endcall %}
+    {% do elementary.insert_rows(relation, rows, should_commit=false, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %}
+{% endmacro %}
+
+{# Dremio - truncate and insert (non-atomic) #}
+{% macro dremio__replace_table_data(relation, rows) %}
+    {% do dbt.truncate_relation(relation) %}
     {% do elementary.insert_rows(relation, rows, should_commit=false, chunk_size=elementary.get_config_var('dbt_artifacts_chunk_size')) %}
 {% endmacro %}