Release/4.12.6

deploy/charts/testgen-app/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.0
+version: 1.0.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

deploy/charts/testgen-app/templates/_environment.yaml

@@ -31,8 +31,6 @@
   value: {{ .Values.testgen.trustTargetDatabaseCertificate | ternary "yes" "no" | quote }}
 - name: TG_EXPORT_TO_OBSERVABILITY_VERIFY_SSL
   value: {{ .Values.testgen.observabilityVerifySsl | ternary "yes" "no" | quote }}
-- name: TG_DOCKER_RELEASE_CHECK_ENABLED
-  value: {{ .Values.testgen.releaseCheck | ternary "yes" "no" | quote }}
 {{- end -}}
 
 {{- define "testgen.hookEnvironment" -}}

deploy/charts/testgen-app/values.yaml

@@ -15,7 +15,6 @@ testgen:
   uiPassword:
   trustTargetDatabaseCertificate: false
   observabilityVerifySsl: true
-  releaseCheck: true
   labels:
 
 cliHooks:

deploy/docker-bake.hcl

@@ -4,12 +4,16 @@ variable "TESTGEN_VERSION" {}
 variable "TESTGEN_DOCKER_HUB_REPO" {
   default = "datakitchen/dataops-testgen"
 }
+variable "TESTGEN_SUPPORT_EMAIL" {
+  default = "open-source-support@datakitchen.io"
+}
 
 target "testgen-release" {
   args = {
     TESTGEN_VERSION = "${TESTGEN_VERSION}"
     TESTGEN_BASE_LABEL = "${TESTGEN_BASE_LABEL}"
     TESTGEN_DOCKER_HUB_REPO = "${TESTGEN_DOCKER_HUB_REPO}"
+    TESTGEN_SUPPORT_EMAIL = "${TESTGEN_SUPPORT_EMAIL}"
   }
   context = "."
   dockerfile = "deploy/testgen.dockerfile"
@@ -31,6 +35,7 @@ target "testgen-qa" {
     TESTGEN_VERSION = "${TESTGEN_VERSION}"
     TESTGEN_BASE_LABEL = "${TESTGEN_BASE_LABEL}"
     TESTGEN_DOCKER_HUB_REPO = "${TESTGEN_DOCKER_HUB_REPO}"
+    TESTGEN_SUPPORT_EMAIL = "${TESTGEN_SUPPORT_EMAIL}"
   }
   context = "."
   dockerfile = "deploy/testgen.dockerfile"

deploy/testgen.dockerfile

@@ -1,10 +1,11 @@
-ARG TESTGEN_BASE_LABEL=v6
+ARG TESTGEN_BASE_LABEL=v7
 
 FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image
 
 # Args have to be set in current build stage: https://github.com/moby/moby/issues/37345
 ARG TESTGEN_VERSION
 ARG TESTGEN_DOCKER_HUB_REPO
+ARG TESTGEN_SUPPORT_EMAIL
 
 ENV PYTHONPATH=/dk/lib/python3.12/site-packages
 ENV PATH=$PATH:/dk/bin
@@ -24,6 +25,7 @@ RUN chown -R testgen:testgen /var/lib/testgen /dk/lib/python3.12/site-packages/s
 
 ENV TESTGEN_VERSION=${TESTGEN_VERSION}
 ENV TESTGEN_DOCKER_HUB_REPO=${TESTGEN_DOCKER_HUB_REPO}
+ENV TESTGEN_SUPPORT_EMAIL=${TESTGEN_SUPPORT_EMAIL}
 ENV TG_RELEASE_CHECK=docker
 
 USER testgen

docker-compose.yml

@@ -9,7 +9,6 @@ x-common-variables: &common-variables
   TG_METADATA_DB_HOST: postgres
   TG_TARGET_DB_TRUST_SERVER_CERTIFICATE: yes
   TG_EXPORT_TO_OBSERVABILITY_VERIFY_SSL: no
-  TG_DOCKER_RELEASE_CHECK_ENABLED: yes
 
 
 services:

docs/configuration.md

@@ -282,9 +282,3 @@ default: `dataset`
 
 When exporting to your instance of Observabilty, the key sent to the events API to identify the components.
 default: `default`
-
-#### `TG_DOCKER_RELEASE_CHECK_ENABLED`
-
-Enables calling Docker Hub API to fetch the latest released image tag. The fetched tag is displayed in the UI menu.
-
-default: `yes`

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "4.1.3"
+version = "4.12.6"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },
@@ -40,7 +40,7 @@ dependencies = [
     "requests_extensions==1.1.3",
     "numpy==1.26.4",
     "pandas==2.1.4",
-    "streamlit==1.44.1",
+    "streamlit==1.46.1",
     "streamlit-extras==0.3.0",
     "streamlit-aggrid==0.3.4.post3",
     "plotly_express==0.4.1",
@@ -61,7 +61,6 @@ dependencies = [
 
     # Pinned to match the manually compiled libs or for security
     "pyarrow==18.1.0",
-    "snowflake-connector-python==3.13.1",
     "matplotlib==3.9.2",
     "scipy==1.14.1",
     "jinja2==3.1.6",

testgen/__main__.py

@@ -48,6 +48,7 @@
 LOG = logging.getLogger("testgen")
 
 APP_MODULES = ["ui", "scheduler"]
+VERSION_DATA = version_service.get_version()
 
 
 @dataclass
@@ -69,7 +70,13 @@ def invoke(self, ctx: Context):
 
 @click.group(
     cls=CliGroup,
-    help=f"This version: {settings.VERSION} \n\nLatest version: {version_service.get_latest_version()} \n\nSchema revision: {get_schema_revision()}"
+    help=f"""
+    {VERSION_DATA.edition} {VERSION_DATA.current or ""}
+
+    {f"New version available! {VERSION_DATA.latest}" if VERSION_DATA.latest != VERSION_DATA.current else ""}
+
+    Schema revision: {get_schema_revision()}
+    """
 )
 @click.option(
     "-v",

testgen/commands/queries/execute_tests_query.py

@@ -29,12 +29,30 @@ def __init__(self, strProjectCode, strFlavor, strTestSuiteId, strTestSuite, minu
 
     def _AssembleDisplayParameters(self):
 
-        lst_parms = ["column_name", "skip_errors", "baseline_ct", "baseline_unique_ct", "baseline_value",
-                     "baseline_value_ct", "baseline_sum", "baseline_avg", "baseline_sd", "subset_condition",
-                     "groupby_names", "having_condition", "window_date_column", "window_days",
-                     "match_column_names", "match_subset_condition", "match_schema_name", "match_table_name",
-                     "match_groupby_names", "match_having_condition",
-                     ]
+        lst_parms = [
+            "column_name",
+            "skip_errors",
+            "baseline_ct",
+            "baseline_unique_ct",
+            "baseline_value",
+            "baseline_value_ct",
+            "baseline_sum",
+            "baseline_avg",
+            "baseline_sd",
+            "lower_tolerance",
+            "upper_tolerance",
+            "subset_condition",
+            "groupby_names",
+            "having_condition",
+            "window_date_column",
+            "window_days",
+            "match_column_names",
+            "match_subset_condition",
+            "match_schema_name",
+            "match_table_name",
+            "match_groupby_names",
+            "match_having_condition",
+        ]
         str_parms = "; ".join(f"{key}={self.dctTestParms[key]}"
                              for key in lst_parms
                              if key.lower() in self.dctTestParms and self.dctTestParms[key] not in [None, ""])
@@ -107,11 +125,6 @@ def GetTestsNonCAT(self, booClean):
 
         return strQ
 
-    def AddTestRecordtoTestRunTable(self):
-        strQ = self._ReplaceParms(read_template_sql_file("ex_write_test_record_to_testrun_table.sql", "execution"))
-
-        return strQ
-
     def PushTestRunStatusUpdateSQL(self):
         # Runs on DK DB
         strQ = self._ReplaceParms(read_template_sql_file("ex_update_test_record_in_testrun_table.sql", "execution"))

testgen/commands/queries/generate_tests_query.py

@@ -60,11 +60,11 @@ def GetTestTypesSQL(self, booClean):
 
         return strQuery
 
-    def GetTestDerivationQueriesAsList(self, booClean):
+    def GetTestDerivationQueriesAsList(self, template_directory, booClean):
         # This assumes the queries run in no particular order,
         # and will order them alphabetically by file name
         lstQueries = sorted(
-            get_template_files(mask=r"^.*sql$", sub_directory="gen_funny_cat_tests"), key=lambda key: str(key)
+            get_template_files(mask=r"^.*sql$", sub_directory=template_directory), key=lambda key: str(key)
         )
         lstTemplate = []
 

testgen/commands/queries/profiling_query.py

@@ -125,6 +125,7 @@ def ReplaceParms(self, strInputString):
         strInputString = strInputString.replace("{CONTINGENCY_COLUMNS}", self.contingency_columns)
         strInputString = strInputString.replace("{CONTINGENCY_MAX_VALUES}", self.contingency_max_values)
         strInputString = strInputString.replace("{PROCESS_ID}", str(self.process_id))
+        strInputString = strInputString.replace("{SQL_FLAVOR}", self.flavor)
         strInputString = replace_templated_functions(strInputString, self.flavor)
 
         return strInputString

testgen/commands/queries/refresh_data_chars_query.py

@@ -10,7 +10,7 @@ class CRefreshDataCharsSQL:
     sql_flavor: str
     table_group_schema: str
     table_group_id: str
-    
+
     max_query_chars: int
     profiling_table_set: str
     profiling_include_mask: str
@@ -37,20 +37,31 @@ def _replace_params(self, sql_query: str) -> str:
         sql_query = sql_query.replace("{RUN_DATE}", self.run_date)
         sql_query = sql_query.replace("{SOURCE_TABLE}", self.source_table)
         return sql_query
-    
+
     def _get_mask_query(self, mask: str, is_include: bool) -> str:
         sub_query = ""
         if mask:
             sub_query += " AND (" if is_include else " AND NOT ("
             is_first = True
+            escape = ""
+            if self.sql_flavor.startswith("mssql"):
+                escaped_underscore = "[_]"
+            elif self.sql_flavor == "snowflake":
+                escaped_underscore = "\\\\_"
+                escape = "ESCAPE '\\\\'"
+            elif self.sql_flavor == "redshift":
+                escaped_underscore = "\\\\_"
+            else:
+                escaped_underscore = "\\_"
             for item in mask.split(","):
                 if not is_first:
                     sub_query += " OR "
-                sub_query += "(c.table_name LIKE '" + item.strip() + "')"
+                item = item.strip().replace("_", escaped_underscore)
+                sub_query += f"(c.table_name LIKE '{item}' {escape})"
                 is_first = False
             sub_query += ")"
         return sub_query
-    
+
     def GetDDFQuery(self) -> str:
         # Runs on Project DB
         sql_query = self._replace_params(
@@ -67,18 +78,18 @@ def GetDDFQuery(self) -> str:
         sql_query = sql_query.replace("{TABLE_CRITERIA}", table_criteria)
 
         return sql_query
-    
+
     def GetRecordCountQueries(self, schema_tables: list[str]) -> list[str]:
         count_queries = [
             f"SELECT '{item}', COUNT(*) FROM {item}"
             for item in schema_tables
         ]
         return chunk_queries(count_queries, " UNION ALL ", self.max_query_chars)
-    
+
     def GetDataCharsUpdateQuery(self) -> str:
         # Runs on DK Postgres Server
         return self._replace_params(read_template_sql_file("data_chars_update.sql", sub_directory="data_chars"))
-    
+
     def GetStagingDeleteQuery(self) -> str:
         # Runs on DK Postgres Server
         return self._replace_params(read_template_sql_file("data_chars_staging_delete.sql", sub_directory="data_chars"))

testgen/commands/run_execute_tests.py

@@ -17,7 +17,7 @@
     WriteListToDB,
     date_service,
 )
-from testgen.common.database.database_service import empty_cache
+from testgen.common.database.database_service import ExecuteDBQuery, empty_cache
 
 from .run_execute_cat_tests import run_cat_test_queries
 from .run_refresh_data_chars import run_refresh_data_chars_queries
@@ -26,6 +26,17 @@
 LOG = logging.getLogger("testgen")
 
 
+def add_test_run_record(test_run_id, test_suite_id, test_time, process_id):
+    query = f"""
+        INSERT INTO test_runs(id, test_suite_id, test_starttime, process_id)
+        (SELECT '{test_run_id}':: UUID as id,
+                '{test_suite_id}' as test_suite_id,
+                '{test_time}' as test_starttime,
+                '{process_id}' as process_id);
+    """
+    ExecuteDBQuery("DKTG", query)
+
+
 def run_test_queries(dctParms, strTestRunID, strTestTime, strProjectCode, strTestSuite, minutes_offset=0, spinner=None):
     booErrors = False
     error_msg = ""
@@ -38,11 +49,6 @@ def run_test_queries(dctParms, strTestRunID, strTestTime, strProjectCode, strTes
     clsExecute.process_id = process_service.get_current_process_id()
     booClean = False
 
-    # Add a record in Test Run table for the new Test Run
-    strTestRunQuery = clsExecute.AddTestRecordtoTestRunTable()
-    lstTestRunQuery = [strTestRunQuery]
-    RunActionQueryList("DKTG", lstTestRunQuery)
-
     try:
         # Retrieve non-CAT Queries
         LOG.info("CurrentStep: Retrieve Non-CAT Queries")
@@ -131,6 +137,11 @@ def run_execution_steps(
     LOG.info("CurrentStep: Retrieving TestExec Parameters")
     test_exec_params = RetrieveTestExecParms(project_code, test_suite)
 
+    # Add a record in Test Run table for the new Test Run
+    add_test_run_record(
+        test_run_id, test_exec_params["test_suite_id"], test_time, process_service.get_current_process_id()
+    )
+
     LOG.info("CurrentStep: Assigning Connection Parms")
     AssignConnectParms(
         test_exec_params["project_code"],

testgen/commands/run_generate_tests.py

@@ -65,7 +65,8 @@ def run_test_gen_queries(strTableGroupsID, strTestSuite, strGenerationSet=None):
 
     LOG.info("CurrentStep: Compiling Test Gen Queries")
 
-    lstFunnyTemplateQueries = clsTests.GetTestDerivationQueriesAsList(booClean)
+    lstFunnyTemplateQueries = clsTests.GetTestDerivationQueriesAsList("gen_funny_cat_tests", booClean)
+    lstQueryTemplateQueries = clsTests.GetTestDerivationQueriesAsList("gen_query_tests", booClean)
     lstGenericTemplateQueries = []
 
     # Delete old Tests
@@ -102,15 +103,15 @@ def run_test_gen_queries(strTableGroupsID, strTestSuite, strGenerationSet=None):
     LOG.info("TestGen CAT Queries were compiled")
 
     # Make sure delete, then generic templates run before the funny templates
-    lstQueries = [strDeleteQuery, *lstGenericTemplateQueries, *lstFunnyTemplateQueries]
+    lstQueries = [strDeleteQuery, *lstGenericTemplateQueries, *lstFunnyTemplateQueries, *lstQueryTemplateQueries]
 
     if lstQueries:
         LOG.info("Running Test Generation Template Queries")
         RunActionQueryList("DKTG", lstQueries)
         message = "Test generation completed successfully."
     else:
         message = "No TestGen Queries were compiled."
-    
+
     MixpanelService().send_event(
         "generate-tests",
         source=settings.ANALYTICS_JOB_SOURCE,

testgen/commands/run_launch_db_config.py

@@ -38,6 +38,8 @@ def _get_params_mapping() -> dict:
         "PROJECT_USER": settings.PROJECT_DATABASE_USER,
         "PROJECT_PORT": settings.PROJECT_DATABASE_PORT,
         "PROJECT_HOST": settings.PROJECT_DATABASE_HOST,
+        "PROJECT_PW_ENCRYPTED": EncryptText(settings.PROJECT_DATABASE_PASSWORD),
+        "PROJECT_HTTP_PATH": "",
         "PROJECT_SCHEMA": settings.PROJECT_DATABASE_SCHEMA,
         "PROFILING_TABLE_SET": settings.DEFAULT_PROFILING_TABLE_SET,
         "PROFILING_INCLUDE_MASK": settings.DEFAULT_PROFILING_INCLUDE_MASK,
@@ -54,7 +56,6 @@ def _get_params_mapping() -> dict:
         "TEST_SUITE_DESCRIPTION": settings.DEFAULT_TEST_SUITE_DESCRIPTION,
         "MAX_THREADS": settings.PROJECT_CONNECTION_MAX_THREADS,
         "MAX_QUERY_CHARS": settings.PROJECT_CONNECTION_MAX_QUERY_CHAR,
-        "PROJECT_PW_ENCRYPTED": EncryptText(settings.PROJECT_DATABASE_PASSWORD),
         "OBSERVABILITY_API_URL": settings.OBSERVABILITY_API_URL,
         "OBSERVABILITY_API_KEY": settings.OBSERVABILITY_API_KEY,
         "OBSERVABILITY_COMPONENT_KEY": settings.OBSERVABILITY_DEFAULT_COMPONENT_KEY,