From 0d7490ea57ac337feab81bdfd34a3b3d039008dd Mon Sep 17 00:00:00 2001 From: Adam Fiedler Date: Tue, 29 Jul 2025 17:00:28 +0200 Subject: [PATCH] test: add aggregated dataset to testing ldm risk: nonprod --- .../logical_model/dataset/dataset.py | 1 + .../entity_model/content_objects/dataset.py | 8 +++ .../catalog/workspace/model_container.py | 5 ++ .../expected/declarative_workspaces.json | 57 ++++++++++++++++ .../campaign_channels_per_category.yaml | 37 +++++++++++ .../campaign_channels_per_category.yaml | 37 +++++++++++ .../tests/catalog/test_catalog_workspace.py | 5 +- .../catalog/test_catalog_workspace_content.py | 13 ++-- .../fixtures/demo_declarative_hierarchy.json | 65 ++++++++++++++++++- tests-support/upload_demo_layout.py | 2 +- 10 files changed, 218 insertions(+), 12 deletions(-) create mode 100644 gooddata-sdk/tests/catalog/load/gooddata_layouts/default/workspaces/demo/ldm/datasets/campaign_channels_per_category.yaml create mode 100644 gooddata-sdk/tests/catalog/load/workspace_content/gooddata_layouts/default/workspaces/demo_testing/ldm/datasets/campaign_channels_per_category.yaml diff --git a/gooddata-sdk/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py b/gooddata-sdk/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py index 86751dceb..6d20610cb 100644 --- a/gooddata-sdk/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py +++ b/gooddata-sdk/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py @@ -42,6 +42,7 @@ class CatalogDeclarativeDataset(Base): attributes: Optional[list[CatalogDeclarativeAttribute]] = None facts: Optional[list[CatalogDeclarativeFact]] = None aggregated_facts: Optional[list[CatalogDeclarativeAggregatedFact]] = None + precedence: Optional[int] = None data_source_table_id: Optional[CatalogDataSourceTableIdentifier] = None sql: Optional[CatalogDeclarativeDatasetSql] = None tags: Optional[list[str]] = None diff --git a/gooddata-sdk/gooddata_sdk/catalog/workspace/entity_model/content_objects/dataset.py b/gooddata-sdk/gooddata_sdk/catalog/workspace/entity_model/content_objects/dataset.py index 472985528..514fb7f1f 100644 --- a/gooddata-sdk/gooddata_sdk/catalog/workspace/entity_model/content_objects/dataset.py +++ b/gooddata-sdk/gooddata_sdk/catalog/workspace/entity_model/content_objects/dataset.py @@ -145,6 +145,14 @@ def generate_facts_from_api(self) -> list[CatalogFact]: default=attr.Factory(lambda self: self.generate_facts_from_api(), takes_self=True), ) + # TODO: Doublecheck if we shouldn't do something like for facts + aggregated_facts: Optional[list[CatalogAggregatedFact]] = attr.field( + default=attr.Factory(lambda self: self.json_api_attributes.get("aggregatedFacts"), takes_self=True), + ) + precedence: Optional[int] = attr.field( + default=attr.Factory(lambda self: self.json_api_attributes.get("precedence"), takes_self=True) + ) + grain: Optional[list] = attr.field( default=attr.Factory(lambda self: self.json_api_attributes.get("grain"), takes_self=True) ) diff --git a/gooddata-sdk/gooddata_sdk/catalog/workspace/model_container.py b/gooddata-sdk/gooddata_sdk/catalog/workspace/model_container.py index b5e29053b..8c3c39671 100644 --- a/gooddata-sdk/gooddata_sdk/catalog/workspace/model_container.py +++ b/gooddata-sdk/gooddata_sdk/catalog/workspace/model_container.py @@ -6,6 +6,7 @@ from gooddata_sdk.catalog.types import ValidObjects from gooddata_sdk.catalog.workspace.entity_model.content_objects.dataset import ( + CatalogAggregatedFact, CatalogAttribute, CatalogDataset, CatalogFact, @@ -46,6 +47,10 @@ def datasets(self) -> list[CatalogDataset]: def facts(self) -> list[CatalogFact]: return [f for d in self._datasets for f in d.facts] + @property + def aggregated_facts(self) -> list[CatalogAggregatedFact]: + return [f for d in self._datasets for f in d.aggregated_facts] + @property def attributes(self) -> list[CatalogAttribute]: return [a for d in self._datasets for a in d.attributes] diff --git a/gooddata-sdk/tests/catalog/expected/declarative_workspaces.json b/gooddata-sdk/tests/catalog/expected/declarative_workspaces.json index 281b779a4..5dd917d02 100644 --- a/gooddata-sdk/tests/catalog/expected/declarative_workspaces.json +++ b/gooddata-sdk/tests/catalog/expected/declarative_workspaces.json @@ -2376,6 +2376,59 @@ ], "title": "Campaign channels" }, + { + "attributes": [], + "description": "Campaign channels per categories", + "facts": [], + "aggregatedFacts": [ + { + "description": "Budget Agg", + "id": "budget_agg", + "sourceColumn": "budget", + "sourceColumnDataType": "INT", + "tags": [ + "Campaign channels per category" + ], + "sourceFactReference": { + "reference": { + "id": "budget", + "type": "fact" + }, + "operation": "SUM" + } + } + ], + "grain": [], + "id": "campaign_channels_per_category", + "precedence": 1, + "references": [ + { + "identifier": { + "id": "campaign_channels", + "type": "dataset" + }, + "multivalue": false, + "sources": [ + { + "column": "campaign_channel_id", + "dataType": "STRING", + "target": { + "id": "campaign_channel_id", + "type": "attribute" + } + } + ] + } + ], + "tags": [ + "Campaign channels per category" + ], + "sql": { + "statement": "SELECT category, SUM(budget) FROM campaign_channels GROUP BY category", + "dataSourceId": "demo-test-ds" + }, + "title": "Campaign channels per category" + }, { "attributes": [ { @@ -2412,6 +2465,7 @@ }, "description": "Campaigns", "facts": [], + "aggregatedFacts": [], "grain": [ { "id": "campaign_id", @@ -2494,6 +2548,7 @@ }, "description": "Customers", "facts": [], + "aggregatedFacts": [], "grain": [ { "id": "customer_id", @@ -2575,6 +2630,7 @@ "title": "Quantity" } ], + "aggregatedFacts": [], "grain": [ { "id": "order_line_id", @@ -2724,6 +2780,7 @@ }, "description": "Products", "facts": [], + "aggregatedFacts": [], "grain": [ { "id": "product_id", diff --git a/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/workspaces/demo/ldm/datasets/campaign_channels_per_category.yaml b/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/workspaces/demo/ldm/datasets/campaign_channels_per_category.yaml new file mode 100644 index 000000000..6373b5e98 --- /dev/null +++ b/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/workspaces/demo/ldm/datasets/campaign_channels_per_category.yaml @@ -0,0 +1,37 @@ +# (C) 2025 GoodData Corporation +id: campaign_channels_per_category +references: + - identifier: + id: campaign_channels + type: dataset + multivalue: false + sources: + - column: campaign_channel_id + dataType: NUMERIC + target: + id: campaign_channel_id + type: attribute +title: Campaign channels per category +description: Campaign channels per categories +attributes: [] +facts: [] +grain: [] +aggregatedFacts: + - id: budget_agg + sourceColumn: budget + sourceFactReference: + operation: SUM + reference: + id: budget + type: fact + sourceColumnDataType: INT + description: Budget Agg + tags: + - Campaign channels per category +precedence: 1 +sql: + dataSourceId: demo-test-ds + statement: SELECT category, SUM(budget) FROM campaign_channels GROUP + BY category +tags: + - Campaign channels per category diff --git a/gooddata-sdk/tests/catalog/load/workspace_content/gooddata_layouts/default/workspaces/demo_testing/ldm/datasets/campaign_channels_per_category.yaml b/gooddata-sdk/tests/catalog/load/workspace_content/gooddata_layouts/default/workspaces/demo_testing/ldm/datasets/campaign_channels_per_category.yaml new file mode 100644 index 000000000..6373b5e98 --- /dev/null +++ b/gooddata-sdk/tests/catalog/load/workspace_content/gooddata_layouts/default/workspaces/demo_testing/ldm/datasets/campaign_channels_per_category.yaml @@ -0,0 +1,37 @@ +# (C) 2025 GoodData Corporation +id: campaign_channels_per_category +references: + - identifier: + id: campaign_channels + type: dataset + multivalue: false + sources: + - column: campaign_channel_id + dataType: NUMERIC + target: + id: campaign_channel_id + type: attribute +title: Campaign channels per category +description: Campaign channels per categories +attributes: [] +facts: [] +grain: [] +aggregatedFacts: + - id: budget_agg + sourceColumn: budget + sourceFactReference: + operation: SUM + reference: + id: budget + type: fact + sourceColumnDataType: INT + description: Budget Agg + tags: + - Campaign channels per category +precedence: 1 +sql: + dataSourceId: demo-test-ds + statement: SELECT category, SUM(budget) FROM campaign_channels GROUP + BY category +tags: + - Campaign channels per category diff --git a/gooddata-sdk/tests/catalog/test_catalog_workspace.py b/gooddata-sdk/tests/catalog/test_catalog_workspace.py index 1182d14a6..545631452 100644 --- a/gooddata-sdk/tests/catalog/test_catalog_workspace.py +++ b/gooddata-sdk/tests/catalog/test_catalog_workspace.py @@ -701,8 +701,7 @@ def test_clone_workspace(test_config): default_cloned_decl_ws = sdk.catalog_workspace.get_declarative_workspace(default_cloned_ws_id) assert default_cloned_decl_ws.ldm.datasets[0].data_source_table_id.data_source_id == test_config["data_source2"] assert default_cloned_decl_ws.ldm.datasets[0].facts[0].source_column == "BUDGET" - # TODO: Add a nontrivial test for agg facts here - assert default_cloned_decl_ws.ldm.datasets[0].aggregated_facts == [] + assert default_cloned_decl_ws.ldm.datasets[1].aggregated_facts[0].source_column == "BUDGET" sdk.catalog_workspace.clone_workspace( source_ws_id, target_workspace_id=custom_cloned_ws_id, target_workspace_name=custom_cloned_ws_name @@ -756,7 +755,7 @@ def test_translate_workspace(test_config): for fact in dataset.facts: if fact.id == "budget": assert fact.title == "Rozpočet" - # TODO: Add agg facts here for descriptions? + # TODO: Do for aggregated facts descriptions # Run second time without translation function. Previous execution created translation file, which is used. sdk.catalog_workspace.generate_localized_workspaces( diff --git a/gooddata-sdk/tests/catalog/test_catalog_workspace_content.py b/gooddata-sdk/tests/catalog/test_catalog_workspace_content.py index 58eb95cd7..94e4ec710 100644 --- a/gooddata-sdk/tests/catalog/test_catalog_workspace_content.py +++ b/gooddata-sdk/tests/catalog/test_catalog_workspace_content.py @@ -63,8 +63,7 @@ def test_catalog_list_facts(test_config): def test_catalog_list_aggregated_facts(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) agg_facts_list = sdk.catalog_workspace_content.get_aggregated_facts_catalog(test_config["workspace"]) - # TODO: Add a non-trivial test - assert len(agg_facts_list) == 0 + assert len(agg_facts_list) == 1 @gd_vcr.use_cassette(str(_fixtures_dir / "demo_catalog_list_attributes.yaml")) @@ -129,7 +128,7 @@ def test_load_and_modify_ds_and_put_declarative_ldm(test_config): sdk.catalog_workspace.create_or_update(workspace) ldm_e = sdk.catalog_workspace_content.get_declarative_ldm(workspace_id) - ds_e = list(set([d.data_source_table_id.data_source_id for d in ldm_e.ldm.datasets])) + ds_e = list(set([d.data_source_table_id.data_source_id for d in ldm_e.ldm.datasets if d.data_source_table_id])) assert ds_e == [test_config["data_source"]] try: @@ -147,7 +146,7 @@ def test_load_and_modify_ds_and_put_declarative_ldm(test_config): ldm_e.ldm.modify_mapped_data_source(data_source_mapping=reverse_data_source_mapping) sdk.catalog_workspace_content.put_declarative_ldm(identifier, ldm_e, validator, standalone_copy=True) ldm_o = sdk.catalog_workspace_content.get_declarative_ldm(identifier) - ds_o = list(set([d.data_source_table_id.data_source_id for d in ldm_o.ldm.datasets])) + ds_o = list(set([d.data_source_table_id.data_source_id for d in ldm_o.ldm.datasets if d.data_source_table_id])) assert ds_o == [test_config["data_source"]] finally: _refresh_workspaces(sdk) @@ -166,7 +165,7 @@ def test_load_ldm_and_modify_tables_columns_case(test_config): assert ldm_e.ldm.datasets[0].data_source_table_id.id == table_id.upper() assert ldm_e.ldm.datasets[0].attributes[0].source_column == attribute_column.upper() assert ldm_e.ldm.datasets[0].facts[0].source_column == fact_column.upper() - # TODO: Add agg facts here + assert ldm_e.ldm.datasets[1].aggregated_facts[0].source_column == fact_column.upper() assert ldm_e.ldm.datasets[0].references[0].source_columns is None assert ldm_e.ldm.datasets[0].references[0].sources[0].column == reference_column.upper() # Test chaining approach as well @@ -178,7 +177,7 @@ def test_load_ldm_and_modify_tables_columns_case(test_config): assert ldm_o.ldm.datasets[0].data_source_table_id.id == table_id assert ldm_o.ldm.datasets[0].attributes[0].source_column == attribute_column assert ldm_o.ldm.datasets[0].facts[0].source_column == fact_column - # TODO: Add agg facts here + assert ldm_o.ldm.datasets[1].aggregated_facts[0].source_column == fact_column assert ldm_o.ldm.datasets[0].references[0].source_columns is None assert ldm_e.ldm.datasets[0].references[0].sources[0].column == reference_column @@ -313,7 +312,7 @@ def test_catalog_load(test_config): # rough initial smoke-test; just do a quick 'rub' assert len(catalog.metrics) == 24 - assert len(catalog.datasets) == 6 + assert len(catalog.datasets) == 7 assert catalog.get_metric("order_amount") is not None assert catalog.get_metric("revenue") is not None diff --git a/tests-support/fixtures/demo_declarative_hierarchy.json b/tests-support/fixtures/demo_declarative_hierarchy.json index cde44a80a..41a292370 100644 --- a/tests-support/fixtures/demo_declarative_hierarchy.json +++ b/tests-support/fixtures/demo_declarative_hierarchy.json @@ -2346,6 +2346,59 @@ ], "title": "Campaign channels" }, + { + "attributes": [], + "description": "Campaign channels per categories", + "facts": [], + "aggregatedFacts": [ + { + "description": "Budget Agg", + "id": "budget_agg", + "sourceColumn": "budget", + "sourceColumnDataType": "INT", + "tags": [ + "Campaign channels per category" + ], + "sourceFactReference": { + "reference": { + "id": "budget", + "type": "fact" + }, + "operation": "SUM" + } + } + ], + "grain": [], + "id": "campaign_channels_per_category", + "precedence": 1, + "references": [ + { + "identifier": { + "id": "campaign_channels", + "type": "dataset" + }, + "multivalue": false, + "sources": [ + { + "column": "campaign_channel_id", + "dataType": "STRING", + "target": { + "id": "campaign_channel_id", + "type": "attribute" + } + } + ] + } + ], + "tags": [ + "Campaign channels per category" + ], + "sql": { + "statement": "SELECT category, SUM(budget) FROM campaign_channels GROUP BY category", + "dataSourceId": "demo-test-ds" + }, + "title": "Campaign channels per category" + }, { "attributes": [ { @@ -2672,11 +2725,21 @@ "dateInstances": [ { "granularities": [ + "MINUTE", + "HOUR", "DAY", "MONTH", "QUARTER", "WEEK", - "YEAR" + "YEAR", + "MINUTE_OF_HOUR", + "HOUR_OF_DAY", + "DAY_OF_WEEK", + "DAY_OF_MONTH", + "DAY_OF_YEAR", + "WEEK_OF_YEAR", + "MONTH_OF_YEAR", + "QUARTER_OF_YEAR" ], "granularitiesFormatting": { "titleBase": "", diff --git a/tests-support/upload_demo_layout.py b/tests-support/upload_demo_layout.py index 2f4a9bc43..543a86a85 100644 --- a/tests-support/upload_demo_layout.py +++ b/tests-support/upload_demo_layout.py @@ -35,7 +35,7 @@ def rest_op(op, url_path, data=None, raise_ex=True): if response.status_code < 200 or response.status_code > 299: if raise_ex: - raise Exception(f"Call to {url} failed - {str(response)}") + raise Exception(f"Call to {url} failed - {str(response.text)}") else: return None