diff --git a/.github/fixtures/profiles.yml b/.github/fixtures/profiles.yml new file mode 100644 index 000000000..7394d01a5 --- /dev/null +++ b/.github/fixtures/profiles.yml @@ -0,0 +1,29 @@ +elementary: + target: vertica + outputs: + vertica: + type: vertica + host: "{{ env_var('VERTICA_HOST') }}" + port: "{{ env_var('VERTICA_PORT') | as_number }}" + username: "{{ env_var('VERTICA_USER') }}" + password: "{{ env_var('VERTICA_PASS') }}" + database: "{{ env_var('VERTICA_DATABASE') }}" + schema: "{{ env_var('SCHEMA_NAME') }}" + connection_load_balance: false + retries: 2 + threads: 4 + +elementary_tests: + target: vertica + outputs: + vertica: + type: vertica + host: "{{ env_var('VERTICA_HOST') }}" + port: "{{ env_var('VERTICA_PORT') | as_number }}" + username: "{{ env_var('VERTICA_USER') }}" + password: "{{ env_var('VERTICA_PASS') }}" + database: "{{ env_var('VERTICA_DATABASE') }}" + schema: "{{ env_var('SCHEMA_NAME') }}" + connection_load_balance: false + retries: 2 + threads: 4 diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml index c5e7ad4d1..ce7388e13 100644 --- a/.github/workflows/test-all-warehouses.yml +++ b/.github/workflows/test-all-warehouses.yml @@ -49,6 +49,7 @@ jobs: trino, clickhouse, dremio, + vertica, ] include: - dbt-version: "${{ inputs.dbt-version || 'latest_pre' }}" @@ -61,6 +62,8 @@ jobs: warehouse-type: redshift - dbt-version: "${{ inputs.dbt-version || 'fusion' }}" warehouse-type: databricks_catalog + - dbt-version: "${{ inputs.dbt-version || '1.8.5' }}" + warehouse-type: vertica uses: ./.github/workflows/test-warehouse.yml with: warehouse-type: ${{ matrix.warehouse-type }} diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 65bed62cd..455df6b45 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -17,6 +17,7 @@ on: - trino - clickhouse - dremio + - vertica elementary-ref: type: string required: false @@ -30,6 +31,11 @@ on: required: false default: "latest_official" description: dbt's version to test with + dbt-vertica-version: + type: string + required: false + default: "1.8.5" + description: dbt-vertica's version to test with workflow_call: inputs: @@ -46,6 +52,10 @@ on: type: string default: "latest_official" required: false + dbt-vertica-version: + type: string + default: "1.8.5" + required: false env: BRANCH_NAME: ${{ github.head_ref || github.ref_name }} @@ -123,8 +133,13 @@ jobs: if: startsWith(inputs.warehouse-type, 'databricks') && inputs.dbt-version < '1.7.0' run: pip install databricks-sql-connector==2.9.3 + - name: Install dbt-vertica + if: inputs.warehouse-type == 'vertica' + run: + pip install "dbt-vertica==${{ inputs.dbt-vertica-version }}" + - name: Install dbt - if: ${{ inputs.dbt-version != 'fusion' }} + if: ${{ inputs.dbt-version != 'fusion' && inputs.warehouse-type != 'vertica' }} run: pip install${{ (inputs.dbt-version == 'latest_pre' && ' --pre') || '' }} "dbt-core${{ (!startsWith(inputs.dbt-version, 'latest') && format('=={0}', inputs.dbt-version)) || '' }}" @@ -145,7 +160,9 @@ jobs: mkdir -p ~/.dbt DBT_VERSION=$(echo "${{ inputs.dbt-version }}" | sed 's/\.//g') UNDERSCORED_REF_NAME=$(echo "${{ inputs.warehouse-type }}_dbt_${DBT_VERSION}_${BRANCH_NAME}" | awk '{print tolower($0)}' | head -c 40 | sed "s/[-\/]/_/g") - echo "$PROFILES_YML" | base64 -d | sed "s//dbt_pkg_$UNDERSCORED_REF_NAME/g" > ~/.dbt/profiles.yml + SCHEMA_NAME="dbt_pkg_$UNDERSCORED_REF_NAME" + echo "SCHEMA_NAME=$SCHEMA_NAME" >> $GITHUB_ENV + echo "$PROFILES_YML" | base64 -d | sed "s//$SCHEMA_NAME/g" > ~/.dbt/profiles.yml - name: Install dependencies working-directory: ${{ env.TESTS_DIR }} @@ -154,6 +171,28 @@ jobs: ln -sfn ${{ github.workspace }}/dbt-data-reliability dbt_project/dbt_packages/elementary pip install -r requirements.txt + - name: Set Vertica environment variables + if: inputs.warehouse-type == 'vertica' + run: | + echo "VERTICA_PORT=5433" >> $GITHUB_ENV + echo "VERTICA_HOST=localhost" >> $GITHUB_ENV + echo "VERTICA_USER=dbadmin" >> $GITHUB_ENV + echo "VERTICA_PASS=vertica" >> $GITHUB_ENV + echo "VERTICA_DATABASE=elementary_tests" >> $GITHUB_ENV + echo "VERTICA_SCHEMA=$SCHEMA_NAME" >> $GITHUB_ENV + + - name: Start Vertica + if: inputs.warehouse-type == 'vertica' + working-directory: ${{ env.TESTS_DIR }} + run: docker compose -f docker-compose-vertica.yml up -d + + - name: Wait for Vertica to be ready + if: inputs.warehouse-type == 'vertica' + run: | + echo "Waiting for Vertica to be healthy..." + timeout 60 bash -c 'until [ "$(docker inspect --format="{{.State.Health.Status}}" vertica)" == "healthy" ]; do echo "Waiting..."; sleep 5; done' + echo "Vertica is ready!" + - name: Check DWH connection working-directory: ${{ env.TESTS_DIR }} run: | diff --git a/.gitignore b/.gitignore index 5172d726c..93a57e825 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ dbt_internal_packages/ logs/ scripts/ +.github/fixtures/.user.yml .idea .DS_Store diff --git a/integration_tests/docker-compose-vertica.yml b/integration_tests/docker-compose-vertica.yml new file mode 100644 index 000000000..bae881fb7 --- /dev/null +++ b/integration_tests/docker-compose-vertica.yml @@ -0,0 +1,36 @@ +services: + vertica: + environment: + VERTICA_USER: dbadmin + VERTICA_PASS: vertica + VERTICA_HOST: localhost + VERTICA_PORT: 5433 + VERTICA_DATABASE: elementary_tests + VERTICA_SCHEMA: ${SCHEMA_NAME} + APP_DB_USER: ${VERTICA_USER} + APP_DB_PASSWORD: ${VERTICA_PASS} + TZ: "America/Los_Angeles" + VERTICA_DB_NAME: ${VERTICA_DATABASE} + VMART_ETL_SCRIPT: "" + container_name: vertica + image: ghcr.io/ratiopbc/vertica-ce + ports: + - "${VERTICA_PORT}:${VERTICA_PORT}" + - "5444:5444" + deploy: + mode: global + ulimits: + nofile: + soft: 65536 + hard: 65536 + volumes: + - type: volume + source: vertica-data + target: /data + healthcheck: + test: ["CMD-SHELL", "/opt/vertica/bin/vsql -U ${VERTICA_USER} -w ${VERTICA_PASS} -c 'SELECT 1;'"] + interval: 5s + timeout: 5s + retries: 10 +volumes: + vertica-data: diff --git a/macros/edr/system/system_utils/empty_table.sql b/macros/edr/system/system_utils/empty_table.sql index 07ccc3e4d..8987f52d4 100644 --- a/macros/edr/system/system_utils/empty_table.sql +++ b/macros/edr/system/system_utils/empty_table.sql @@ -121,7 +121,7 @@ {%- set empty_table_query -%} select * from ( select - {% for column in column_name_and_type_list %} + {%- for column in column_name_and_type_list -%} {{ elementary.empty_column(column[0], column[1]) }} {%- if not loop.last -%},{%- endif %} {%- endfor %} ) as empty_table @@ -161,7 +161,7 @@ cast({{ dummy_values['int'] }} as Nullable({{ elementary.edr_type_int() }})) as {{ column_name }} {%- else %} cast('{{ dummy_values['string'] }}' as {{ elementary.edr_type_string() }}) as {{ column_name }} - {%- endif %} + {%- endif -%} {% endmacro %} diff --git a/macros/utils/cross_db_utils/timeadd.sql b/macros/utils/cross_db_utils/timeadd.sql index 0f2419e86..69d46c938 100644 --- a/macros/utils/cross_db_utils/timeadd.sql +++ b/macros/utils/cross_db_utils/timeadd.sql @@ -22,6 +22,10 @@ {{ elementary.edr_cast_as_timestamp(timestamp_expression) }} + {{ elementary.edr_cast_as_int(number) }} * INTERVAL '1 {{ date_part }}' {% endmacro %} +{% macro vertica__edr_timeadd(date_part, number, timestamp_expression) %} + timestampadd({{ date_part | upper }}, {{ elementary.edr_cast_as_int(number) }}, {{ elementary.edr_cast_as_timestamp(timestamp_expression) }}) +{% endmacro %} + {% macro redshift__edr_timeadd(date_part, number, timestamp_expression) %} dateadd({{ date_part }}, {{ elementary.edr_cast_as_int(number) }}, {{ elementary.edr_cast_as_timestamp(timestamp_expression) }}) {% endmacro %} diff --git a/macros/utils/table_operations/insert_rows.sql b/macros/utils/table_operations/insert_rows.sql index aa278027a..48c840805 100644 --- a/macros/utils/table_operations/insert_rows.sql +++ b/macros/utils/table_operations/insert_rows.sql @@ -149,6 +149,10 @@ {{- return(string_value | replace("'", "''")) -}} {%- endmacro -%} +{%- macro vertica__escape_special_chars(string_value) -%} + {{- return(string_value | replace("'", "''")) -}} +{%- endmacro -%} + {%- macro athena__escape_special_chars(string_value) -%} {{- return(string_value | replace("'", "''")) -}} {%- endmacro -%}