From 2a5f0f032a970bac574ccef20070258f17096278 Mon Sep 17 00:00:00 2001 From: Tom Juntunen Date: Sat, 20 Apr 2024 01:51:30 -0700 Subject: [PATCH 1/5] add compatiblity for 1.5.0 - sourced from MatthewPocock with a minor adjustment made to the anonymous telemetry `unique_key` to use socket.gethostname() --- .github/workflows/ci.yml | 2 +- Dockerfile | 2 +- README.md | 1 + dbt/adapters/sqlite/__version__.py | 2 +- dbt/adapters/sqlite/connections.py | 3 +- dbt/include/sqlite/macros/adapters.sql | 30 ++++++++----------- .../sqlite/macros/utils/timestamps.sql | 12 ++++++++ setup.py | 2 +- 8 files changed, 32 insertions(+), 22 deletions(-) create mode 100644 dbt/include/sqlite/macros/utils/timestamps.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 531346d..a76eeaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ jobs: build: strategy: matrix: - python_version: ["3.8", "3.9", "3.10"] + python_version: ["3.8", "3.9", "3.10", "3.11"] runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index c531b93..cdb3813 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get -y install git python3 python3-pip python3-venv sq WORKDIR /opt/dbt-sqlite RUN python3 -m pip install --upgrade pip \ - && python3 -m pip install pytest pytest-dotenv dbt-core~=1.4.0 dbt-tests-adapter~=1.4.0 + && python3 -m pip install pytest pytest-dotenv dbt-core~=1.5.0 dbt-tests-adapter~=1.5.0 RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/crypto.so RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/math.so diff --git a/README.md b/README.md index 15628bb..6c2fd2e 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Use the right version. Starting with the release of dbt-core 1.0.0, versions of dbt-sqlite are aligned to the same major+minor [version](https://semver.org/) of dbt-core. +- versions 1.5.x of this adapter work with dbt-core 1.5.x - versions 1.4.x of this adapter work with dbt-core 1.4.x - versions 1.3.x of this adapter work with dbt-core 1.3.x - versions 1.2.x of this adapter work with dbt-core 1.2.x diff --git a/dbt/adapters/sqlite/__version__.py b/dbt/adapters/sqlite/__version__.py index f5a0a88..4139253 100644 --- a/dbt/adapters/sqlite/__version__.py +++ b/dbt/adapters/sqlite/__version__.py @@ -1 +1 @@ -version = '1.4.0' +version = '1.5.0' diff --git a/dbt/adapters/sqlite/connections.py b/dbt/adapters/sqlite/connections.py index bd37deb..b96bcdb 100644 --- a/dbt/adapters/sqlite/connections.py +++ b/dbt/adapters/sqlite/connections.py @@ -4,6 +4,7 @@ import glob import os.path import sqlite3 +from socket import gethostname from typing import Optional, Tuple, Any, Dict, List @@ -37,7 +38,7 @@ def unique_field(self): Hashed and included in anonymous telemetry to track adapter adoption. Pick a field that can uniquely identify one team/organization building with this adapter """ - return self.host + return gethostname() def _connection_keys(self): """ Keys to show when debugging """ diff --git a/dbt/include/sqlite/macros/adapters.sql b/dbt/include/sqlite/macros/adapters.sql index f8b99ff..18cb88e 100644 --- a/dbt/include/sqlite/macros/adapters.sql +++ b/dbt/include/sqlite/macros/adapters.sql @@ -79,15 +79,24 @@ {% endmacro %} {% macro sqlite__create_table_as(temporary, relation, sql) -%} - create {% if temporary -%} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{exceptions.warn("Model contracts cannot be enforced by sqlite!")}} + {% endif %} + create {% if temporary -%} temporary - {%- endif %} table {{ relation }} - as + {%- endif %} table {{ relation }} + as {{ sql }} {% endmacro %} {% macro sqlite__create_view_as(relation, sql, auto_begin=False) -%} - create view {{ relation }} as + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{exceptions.warn("Model contracts cannot be enforced by sqlite!")}} + {% endif %} + create view {{ relation }} + as {{ sql }}; {%- endmacro %} @@ -96,15 +105,6 @@ {# see SQLiteAdapter.rename_relation() #} {% endmacro %} -{% macro sqlite__snapshot_get_time() -%} - datetime() -{%- endmacro %} - -{% macro sqlite__snapshot_string_as_time(timestamp) -%} - {# just return the string; SQLite doesn't have a timestamp data type per se #} - {{ return("'" + timestamp|string + "'") }} -{%- endmacro %} - {# the only allowable schema for temporary tables in SQLite is 'temp', so set that here when making the relation and everything else should Just Work @@ -116,7 +116,3 @@ that here when making the relation and everything else should Just Work {% do return(tmp_relation) %} {% endmacro %} - -{% macro sqlite__current_timestamp() -%} - datetime() -{%- endmacro %} diff --git a/dbt/include/sqlite/macros/utils/timestamps.sql b/dbt/include/sqlite/macros/utils/timestamps.sql new file mode 100644 index 0000000..e75e7a3 --- /dev/null +++ b/dbt/include/sqlite/macros/utils/timestamps.sql @@ -0,0 +1,12 @@ +{% macro sqlite__current_timestamp() -%} + datetime() +{%- endmacro %} + +{% macro sqlite__snapshot_string_as_time(timestamp) -%} + {# just return the string; SQLite doesn''t have a timestamp data type per se #} + {{ return("'" + timestamp|string + "'") }} +{%- endmacro %} + +{% macro sqlite__snapshot_get_time() -%} + datetime() +{%- endmacro %} \ No newline at end of file diff --git a/setup.py b/setup.py index 0ac10e9..29cd0ff 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def _get_plugin_version(): ] }, install_requires=[ - "dbt-core>=1.4.0" + "dbt-core~=1.5.0" ], classifiers=[ 'Development Status :: 4 - Beta', From bd1c175076479d95f2371ac6c9c79fea5185dbde Mon Sep 17 00:00:00 2001 From: Tom Juntunen Date: Sat, 20 Apr 2024 17:48:11 -0700 Subject: [PATCH 2/5] update sqlite_dateadd to handle datetimes and more dateparts --- dbt/include/sqlite/macros/utils/dateadd.sql | 36 ++++++++++++++++---- tests/functional/adapter/utils/test_utils.py | 35 ++++++++++++++++--- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/dbt/include/sqlite/macros/utils/dateadd.sql b/dbt/include/sqlite/macros/utils/dateadd.sql index 2a605b4..332c07f 100644 --- a/dbt/include/sqlite/macros/utils/dateadd.sql +++ b/dbt/include/sqlite/macros/utils/dateadd.sql @@ -1,8 +1,32 @@ -{% macro sqlite__dateadd(datepart, interval, from_date_or_timestamp) %} - - date( - {{ from_date_or_timestamp }}, - "{{ datepart }} {{ datepart }}" - ) +{% macro sqlite__dateadd(from_date_or_timestamp, interval, datepart) %} + -- Perform date addition based on the detected input format using CASE statements with LIKE for string contains + CASE + -- Check if format is DATETIME + WHEN {{ from_date_or_timestamp }} LIKE '%:%' OR ({{ from_date_or_timestamp }} LIKE '%T%' AND {{ from_date_or_timestamp }} LIKE '%Z%') THEN + CASE + WHEN LOWER({{ datepart }}) = 'second' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' seconds') + WHEN LOWER({{ datepart }}) = 'minute' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' minutes') + WHEN LOWER({{ datepart }}) = 'hour' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' hours') + WHEN LOWER({{ datepart }}) = 'day' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' days') + WHEN LOWER({{ datepart }}) = 'week' THEN datetime({{ from_date_or_timestamp }}, '+' || ({{ interval }} * 7) || ' days') + WHEN LOWER({{ datepart }}) = 'month' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' months') + WHEN LOWER({{ datepart }}) = 'quarter' THEN datetime({{ from_date_or_timestamp }}, '+' || ({{ interval }} * 3) || ' months') + WHEN LOWER({{ datepart }}) = 'year' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' years') + ELSE NULL + END + -- Check if format is DATE + WHEN {{ from_date_or_timestamp }} LIKE '%-%' AND {{ from_date_or_timestamp }} NOT LIKE '%T%' AND {{ from_date_or_timestamp }} NOT LIKE '% %' THEN + CASE + WHEN LOWER({{ datepart }}) IN ('second', 'minute', 'hour') THEN date({{ from_date_or_timestamp }}) + WHEN LOWER({{ datepart }}) = 'day' THEN date({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' days') + WHEN LOWER({{ datepart }}) = 'week' THEN date({{ from_date_or_timestamp }}, '+' || ({{ interval }} * 7) || ' days') + WHEN LOWER({{ datepart }}) = 'month' THEN date({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' months') + WHEN LOWER({{ datepart }}) = 'quarter' THEN date({{ from_date_or_timestamp }}, '+' || ({{ interval }} * 3) || ' months') + WHEN LOWER({{ datepart }}) = 'year' THEN date({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' years') + ELSE NULL + END + ELSE + NULL + END {% endmacro %} diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index fd40e79..ff3a25c 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -4,6 +4,10 @@ seeds__data_datediff_csv, models__test_datediff_yml, ) +from dbt.tests.adapter.utils.fixture_dateadd import ( + seeds__data_dateadd_csv, + models__test_dateadd_yml, +) from dbt.tests.adapter.utils.test_any_value import BaseAnyValue from dbt.tests.adapter.utils.test_array_append import BaseArrayAppend from dbt.tests.adapter.utils.test_array_concat import BaseArrayConcat @@ -66,11 +70,35 @@ class TestConcat(BaseConcat): class TestCurrentTimestampNaive(BaseCurrentTimestampNaive): pass +class BaseDateAdd(BaseUtils): + + models__test_dateadd_sql = """ + with data as ( + select * from {{ ref('data_dateadd') }} + ) + + select + {{ dateadd('from_time', 'interval_length', 'datepart') }} AS actual, + result as expected + from data + """ + + @pytest.fixture(scope="class") + def seeds(self): + return {"data_dateadd.csv": seeds__data_dateadd_csv} + + @pytest.fixture(scope="class") + def models(self): + return { + "test_dateadd.yml": models__test_dateadd_yml, + "test_dateadd.sql": self.interpolate_macro_namespace( + self.models__test_dateadd_sql, "dateadd" + ), + } class TestDateAdd(BaseDateAdd): pass - class BaseDateDiff(BaseUtils): models__test_datediff_sql = """ @@ -179,10 +207,9 @@ class TestRight(BaseRight): class TestSafeCast(BaseSafeCast): pass - +@pytest.mark.skip("TODO: implement split_part, either using sqlite>=3.8.3 for WITH RECURSIVE support, or possibly sooner using jinja and agate tables") class TestSplitPart(BaseSplitPart): - pass - + pass class TestStringLiteral(BaseStringLiteral): pass From 76dfc4d353b51bf81b9947eb72635000b2a4bd55 Mon Sep 17 00:00:00 2001 From: Tom Juntunen Date: Sat, 20 Apr 2024 18:00:57 -0700 Subject: [PATCH 3/5] bump version ref to 1.6.0 Remaining todo per upgrade guide: - stub materialized views (MV) as a materialization option to show users MV are not supported in sqlite - stub dbt_clone - add new available tests from 1.6.0 and re-test to find any remaining issues No action required: - Support for Python 3.7 is dropped - debug_query() does not need to be updated, as its not overwritten in our adapter code - We use SQLConnectionManager.execute() directly; no changes required --- Dockerfile | 2 +- dbt/adapters/sqlite/__version__.py | 2 +- dbt/include/sqlite/macros/utils/dateadd.sql | 7 ++++--- setup.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index cdb3813..841fed5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get -y install git python3 python3-pip python3-venv sq WORKDIR /opt/dbt-sqlite RUN python3 -m pip install --upgrade pip \ - && python3 -m pip install pytest pytest-dotenv dbt-core~=1.5.0 dbt-tests-adapter~=1.5.0 + && python3 -m pip install pytest pytest-dotenv dbt-core~=1.6.0 dbt-tests-adapter~=1.6.0 RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/crypto.so RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/math.so diff --git a/dbt/adapters/sqlite/__version__.py b/dbt/adapters/sqlite/__version__.py index 4139253..f7c7de2 100644 --- a/dbt/adapters/sqlite/__version__.py +++ b/dbt/adapters/sqlite/__version__.py @@ -1 +1 @@ -version = '1.5.0' +version = '1.6.0' diff --git a/dbt/include/sqlite/macros/utils/dateadd.sql b/dbt/include/sqlite/macros/utils/dateadd.sql index 332c07f..cbf3db2 100644 --- a/dbt/include/sqlite/macros/utils/dateadd.sql +++ b/dbt/include/sqlite/macros/utils/dateadd.sql @@ -1,8 +1,9 @@ {% macro sqlite__dateadd(from_date_or_timestamp, interval, datepart) %} + -- If provided a DATETIME, returns a DATETIME + -- If provided a DATE, returns a DATE - -- Perform date addition based on the detected input format using CASE statements with LIKE for string contains CASE - -- Check if format is DATETIME + -- Matches DATETIME type based on ISO-8601 WHEN {{ from_date_or_timestamp }} LIKE '%:%' OR ({{ from_date_or_timestamp }} LIKE '%T%' AND {{ from_date_or_timestamp }} LIKE '%Z%') THEN CASE WHEN LOWER({{ datepart }}) = 'second' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' seconds') @@ -15,7 +16,7 @@ WHEN LOWER({{ datepart }}) = 'year' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' years') ELSE NULL END - -- Check if format is DATE + -- Matches DATE type based on ISO-8601 WHEN {{ from_date_or_timestamp }} LIKE '%-%' AND {{ from_date_or_timestamp }} NOT LIKE '%T%' AND {{ from_date_or_timestamp }} NOT LIKE '% %' THEN CASE WHEN LOWER({{ datepart }}) IN ('second', 'minute', 'hour') THEN date({{ from_date_or_timestamp }}) diff --git a/setup.py b/setup.py index 29cd0ff..9edb95d 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def _get_plugin_version(): ] }, install_requires=[ - "dbt-core~=1.5.0" + "dbt-core~=1.6.0" ], classifiers=[ 'Development Status :: 4 - Beta', From dec62d3b79da668780d578510344f7e6d30974e1 Mon Sep 17 00:00:00 2001 From: Tom Juntunen Date: Sat, 20 Apr 2024 18:56:26 -0700 Subject: [PATCH 4/5] switch parameter order for test_dateadd to match typical order --- dbt/include/sqlite/macros/utils/dateadd.sql | 2 +- tests/functional/adapter/utils/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/include/sqlite/macros/utils/dateadd.sql b/dbt/include/sqlite/macros/utils/dateadd.sql index cbf3db2..ce4ccc3 100644 --- a/dbt/include/sqlite/macros/utils/dateadd.sql +++ b/dbt/include/sqlite/macros/utils/dateadd.sql @@ -1,4 +1,4 @@ -{% macro sqlite__dateadd(from_date_or_timestamp, interval, datepart) %} +{% macro sqlite__dateadd(datepart, interval, from_date_or_timestamp) %} -- If provided a DATETIME, returns a DATETIME -- If provided a DATE, returns a DATE diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index ff3a25c..66ef3e2 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -78,7 +78,7 @@ class BaseDateAdd(BaseUtils): ) select - {{ dateadd('from_time', 'interval_length', 'datepart') }} AS actual, + {{ dateadd('datepart', 'interval_length', 'from_time') }} AS actual, result as expected from data """ From 7cc01b5b07440dbe08c3f6a368f3638b4304a0e5 Mon Sep 17 00:00:00 2001 From: Tom Juntunen Date: Sat, 20 Apr 2024 18:59:13 -0700 Subject: [PATCH 5/5] add four new tests in test_utils from dbt-tests-adapter v1.6.0 - mention this adapter works with v1.6.x in README --- README.md | 1 + tests/functional/adapter/utils/test_utils.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c2fd2e..1642ec0 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Use the right version. Starting with the release of dbt-core 1.0.0, versions of dbt-sqlite are aligned to the same major+minor [version](https://semver.org/) of dbt-core. +- versions 1.6.x of this adapter work with dbt-core 1.6.x - versions 1.5.x of this adapter work with dbt-core 1.5.x - versions 1.4.x of this adapter work with dbt-core 1.4.x - versions 1.3.x of this adapter work with dbt-core 1.3.x diff --git a/tests/functional/adapter/utils/test_utils.py b/tests/functional/adapter/utils/test_utils.py index 66ef3e2..7ecfeb7 100644 --- a/tests/functional/adapter/utils/test_utils.py +++ b/tests/functional/adapter/utils/test_utils.py @@ -33,7 +33,9 @@ from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast from dbt.tests.adapter.utils.test_split_part import BaseSplitPart from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral - +from dbt.tests.adapter.utils.test_equals import BaseEquals +from dbt.tests.adapter.utils.test_null_compare import BaseMixedNullCompare, BaseNullCompare +from dbt.tests.adapter.utils.test_validate_sql import BaseValidateSqlMethod class TestAnyValue(BaseAnyValue): pass @@ -213,3 +215,15 @@ class TestSplitPart(BaseSplitPart): class TestStringLiteral(BaseStringLiteral): pass + +class TestEquals(BaseEquals): + pass + +class TestMixedNullCompare(BaseMixedNullCompare): + pass + +class TestNullCompare(BaseNullCompare): + pass + +class TestValidateSqlMethod(BaseValidateSqlMethod): + pass \ No newline at end of file