feat: POC automation test for AI using python

JIRA: QA-23855
risk: nonprod

Author: Tubt

gooddata-sdk/integration_tests/.env.template

@@ -0,0 +1,6 @@
+# (C) 2024 GoodData Corporation
+HOST=
+TOKEN=
+DATASOURCE_ID=
+WORKSPACE_ID=
+LLM_TOKEN=

gooddata-sdk/integration_tests/__init__.py

@@ -0,0 +1 @@
+# (C) 2021 GoodData Corporation

gooddata-sdk/integration_tests/conftest.py

@@ -0,0 +1,34 @@
+# (C) 2024 GoodData Corporation
+# filepath: /Users/tubui/Documents/CODE/gooddata-python-sdk-1/gooddata-sdk/integration_tests/scripts/conftest.py
+import os
+
+import pytest
+from dotenv import load_dotenv
+
+# Load the .env file from the current directory
+load_dotenv()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_env():
+    # Ensure that the environment variables are set
+    os.environ["HOST"] = os.getenv("HOST", "https://checklist.staging.stg11.panther.intgdc.com")
+    os.environ["TOKEN"] = os.getenv("TOKEN", "")
+    os.environ["DATASOURCE_ID"] = os.getenv("DATASOURCE_ID", "")
+    os.environ["WORKSPACE_ID"] = os.getenv("WORKSPACE_ID", "")
+    os.environ["DATASOURCE_TYPE"] = os.getenv("DATASOURCE_TYPE", "")
+    os.environ["DATASOURCE_PASSWORD"] = os.getenv("DATASOURCE_PASSWORD", "")
+
+    # Check if the necessary environment variables are set
+    if not os.environ["HOST"]:
+        raise OSError("\nHOST environment variable is not set.")
+    if not os.environ["TOKEN"]:
+        raise OSError("\nTOKEN environment variable is not set.")
+    if not os.environ["DATASOURCE_ID"]:
+        print("\nWarning: DATA_SOURCE_ID environment variable is not set.")
+    if not os.environ["WORKSPACE_ID"]:
+        print("\nWarning: WORKSPACE_ID environment variable is not set.")
+    if not os.environ["DATASOURCE_TYPE"]:
+        print("\nWarning: DATASOURCE_TYPE environment variable is not set.")
+    if not os.environ["DATASOURCE_PASSWORD"]:
+        print("\nWarning: DATASOURCE_PASSWORD environment variable is not set.")

gooddata-sdk/integration_tests/expected/column_total_returns_by_month.json

@@ -0,0 +1,30 @@
+{
+    "id": "total_returns_per_month",
+    "title": "Total Returns per Month",
+    "visualizationType": "COLUMN",
+    "metrics": [
+      {
+        "id": "total_returns",
+        "type": "metric",
+        "title": "Total Returns"
+      }
+    ],
+    "dimensionality": [
+      {
+        "id": "return_date.month",
+        "type": "attribute",
+        "title": "Return date - Month/Year"
+      }
+    ],
+    "filters": [],
+    "suggestions": [
+      {
+        "query": "Switch to a line chart to better visualize the trend of total returns over the months.",
+        "label": "Line Chart for Trends"
+      },
+      {
+        "query": "Filter the data to show total returns for this year only.",
+        "label": "This Year's Returns"
+      }
+    ]
+}

gooddata-sdk/integration_tests/expected/headline_count_of_order.json

@@ -0,0 +1,21 @@
+{
+    "id": "number_of_order_ids",
+    "title": "Number of Order IDs",
+    "visualizationType": "HEADLINE",
+    "metrics": [
+      {
+        "id": "order_id",
+        "type": "attribute",
+        "title": "Number of Order IDs",
+        "aggFunction": "COUNT"
+      }
+    ],
+    "dimensionality": [],
+    "filters": [],
+    "suggestions": [
+      {
+        "query": "Show the number of orders by year",
+        "label": "Show by Year"
+      }
+    ]
+}

gooddata-sdk/integration_tests/fixtures/ai_questions.json

@@ -0,0 +1,10 @@
+[
+    {
+        "question": "What is number of order id, show as HEADLINE chart?",
+        "expected_objects_file": "headline_count_of_order.json"
+    },
+    {
+        "question": "What is total returns per month? show as COLUMN chart",
+        "expected_objects_file": "column_total_returns_by_month.json"
+    }
+]

gooddata-sdk/integration_tests/scenarios/__init__.py

@@ -0,0 +1 @@
+# (C) 2021 GoodData Corporation

gooddata-sdk/integration_tests/scenarios/aiChat.py

@@ -0,0 +1,36 @@
+# (C) 2024 GoodData Corporation
+import os
+import sys
+
+import pytest
+from dotenv import load_dotenv
+from gooddata_sdk import GoodDataSdk
+
+SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(SCRIPTS_DIR)
+
+
+# Load environment variables from the .env file
+load_dotenv()
+
+# Create the test_config dictionary with the loaded environment variables
+test_config = {"host": os.getenv("HOST"), "token": os.getenv("TOKEN")}
+workspace_id = os.getenv("WORKSPACE_ID")
+
+questions = ["What is number of order line id ?"]
+sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
+
+
+@pytest.mark.parametrize("question", questions)
+def test_ask_ai(question):
+    sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
+    chat_ai_res = sdk.compute.ai_chat(workspace_id, question=question)
+
+    print(f"Chat AI response: {chat_ai_res}")
+    assert chat_ai_res is not None, "Response should not be None"
+
+    print("before test")
+
+
+if __name__ == "__main__":
+    pytest.main()

gooddata-sdk/integration_tests/scenarios/chatHistory.py

@@ -0,0 +1,100 @@
+# (C) 2024 GoodData Corporation
+
+import os
+import sys
+from pathlib import Path
+from pprint import pprint
+
+import gooddata_api_client
+import pytest
+from dotenv import load_dotenv
+from gooddata_api_client.api import smart_functions_api
+from gooddata_api_client.model.chat_history_request import ChatHistoryRequest
+from gooddata_api_client.model.chat_request import ChatRequest
+
+SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(SCRIPTS_DIR)
+from utils import compare_and_print_diff, load_json, normalize_metrics
+
+_current_dir = Path(__file__).parent.absolute()
+parent_dir = _current_dir.parent
+expected_object_dir = parent_dir / "expected"
+questions_list_dir = parent_dir / "fixtures" / "ai_questions.json"
+
+# Load environment variables from the .env file
+load_dotenv()
+
+
+@pytest.fixture(scope="module")
+def test_config():
+    return {
+        "host": os.getenv("HOST"),
+        "token": os.getenv("TOKEN"),
+        "workspace_id": os.getenv("WORKSPACE_ID"),
+        "llm_token": os.getenv("LLM_TOKEN"),
+    }
+
+
+@pytest.fixture(scope="module")
+def api_client(test_config):
+    configuration = gooddata_api_client.Configuration(host=test_config["host"])
+    api_client = gooddata_api_client.ApiClient(configuration)
+    api_client.default_headers["Authorization"] = f"Bearer {test_config['token']}"
+    return api_client
+
+
+def validate_response(actual_response, expected_response):
+    actual_metrics = normalize_metrics(
+        actual_response["created_visualizations"]["objects"][0]["metrics"], exclude_keys=["title"]
+    )
+    expected_metrics = normalize_metrics(expected_response["metrics"], exclude_keys=["title"])
+    compare_and_print_diff(actual_metrics, expected_metrics, "Metrics")
+    actual_visualization_type = actual_response["created_visualizations"]["objects"][0]["visualization_type"]
+    expected_visualization_type = expected_response["visualizationType"]
+    compare_and_print_diff(actual_visualization_type, expected_visualization_type, "Visualization type")
+    actual_dimensionality = actual_response["created_visualizations"]["objects"][0]["dimensionality"]
+    expected_dimensionality = expected_response["dimensionality"]
+    compare_and_print_diff(actual_dimensionality, expected_dimensionality, "Dimensionality")
+    actual_filters = actual_response["created_visualizations"]["objects"][0]["filters"]
+    expected_filters = expected_response["filters"]
+    compare_and_print_diff(actual_filters, expected_filters, "Filters")
+
+
+def test_ai_chat_history_reset(api_client, test_config):
+    api_instance = smart_functions_api.SmartFunctionsApi(api_client)
+    chat_history_request = ChatHistoryRequest(reset=True)
+    try:
+        api_response = api_instance.ai_chat_history(test_config["workspace_id"], chat_history_request)
+        pprint(api_response)
+    except gooddata_api_client.ApiException as e:
+        pytest.fail(f"API exception: {e}")
+    except Exception as e:
+        pytest.fail(f"Unexpected error: {e}")
+
+
+questions_list = load_json(questions_list_dir)
+
+
+@pytest.mark.parametrize(
+    "question, expected_file",
+    [(item["question"], item["expected_objects_file"]) for item in questions_list],
+    ids=[item["question"] for item in questions_list],
+)
+def test_ai_chat(api_client, test_config, question, expected_file):
+    expected_objects = load_json(os.path.join(expected_object_dir, expected_file))
+    api_instance = smart_functions_api.SmartFunctionsApi(api_client)
+    try:
+        api_response = api_instance.ai_chat(test_config["workspace_id"], ChatRequest(question=question))
+        print("\napi_response", api_response.created_visualizations.objects[0])
+        print("\nexpected_file", expected_objects)
+
+        validate_response(api_response.to_dict(), expected_objects)
+
+    except gooddata_api_client.ApiException as e:
+        pytest.fail(f"API exception: {e}")
+    except Exception as e:
+        pytest.fail(f"Unexpected error: {e}")
+
+
+if __name__ == "__main__":
+    pytest.main(["-s", __file__])

gooddata-sdk/integration_tests/scenarios/llmEndpoint.py

@@ -0,0 +1,116 @@
+# (C) 2024 GoodData Corporation
+
+import os
+import sys
+import time
+import uuid
+from pprint import pprint
+
+import gooddata_api_client
+import pytest
+from dotenv import load_dotenv
+from gooddata_api_client.api import entities_api, llm_endpoints_api, metadata_sync_api
+from gooddata_api_client.model.json_api_llm_endpoint_in import JsonApiLlmEndpointIn
+from gooddata_api_client.model.json_api_llm_endpoint_in_attributes import JsonApiLlmEndpointInAttributes
+from gooddata_api_client.model.json_api_llm_endpoint_in_document import JsonApiLlmEndpointInDocument
+from gooddata_api_client.model.json_api_workspace_in import JsonApiWorkspaceIn
+from gooddata_api_client.model.json_api_workspace_in_attributes import JsonApiWorkspaceInAttributes
+from gooddata_api_client.model.json_api_workspace_in_document import JsonApiWorkspaceInDocument
+
+SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(SCRIPTS_DIR)
+
+
+# Load environment variables from the .env file
+load_dotenv()
+
+
+@pytest.fixture(scope="module")
+def test_config():
+    return {
+        "host": os.getenv("HOST"),
+        "token": os.getenv("TOKEN"),
+        "workspace_id": os.getenv("WORKSPACE_ID"),
+        "llm_token": os.getenv("LLM_TOKEN"),
+    }
+
+
+@pytest.fixture(scope="module")
+def api_client(test_config):
+    configuration = gooddata_api_client.Configuration(host=test_config["host"])
+    api_client = gooddata_api_client.ApiClient(configuration)
+    api_client.default_headers["Authorization"] = f"Bearer {test_config['token']}"
+    return api_client
+
+
+def test_create_llm_endpoint(api_client, test_config):
+    llm_title = f"python_sdk_test_{int(time.time())}"
+    api_instance = llm_endpoints_api.LLMEndpointsApi(api_client)
+    json_api_llm_endpoint_in_document = JsonApiLlmEndpointInDocument(
+        data=JsonApiLlmEndpointIn(
+            attributes=JsonApiLlmEndpointInAttributes(
+                llm_model="gpt-4o",
+                provider="OPENAI",
+                title=llm_title,
+                token=test_config["llm_token"],
+                workspaceIds=[test_config["workspace_id"]],
+            ),
+            id=uuid.uuid4().hex,
+            type="llmEndpoint",
+        ),
+    )
+
+    print("json_api_llm_endpoint_in_document", json_api_llm_endpoint_in_document)
+    try:
+        # Post LLM endpoint entities
+        api_response = api_instance.create_entity_llm_endpoints(json_api_llm_endpoint_in_document)
+        pprint(api_response)
+        assert api_response is not None, "API response should not be None"
+    except gooddata_api_client.ApiException as e:
+        pytest.fail(f"API exception: {e}")
+    except Exception as e:
+        pytest.fail(f"Unexpected error: {e}")
+
+
+def enable_early_access_per_workspace(api_client, test_config, feature_flag_name):
+    api_instance = entities_api.EntitiesApi(api_client)
+    json_api_workspace_in_document = JsonApiWorkspaceInDocument(
+        data=JsonApiWorkspaceIn(
+            attributes=JsonApiWorkspaceInAttributes(early_access_values=[feature_flag_name]),
+            id=test_config["workspace_id"],
+            type="workspace",
+        ),
+    )
+    try:
+        print("Attempting to enable early access feature flag...")
+        print("workspace_id", test_config["workspace_id"])
+
+        api_response = api_instance.update_entity_workspaces(
+            test_config["workspace_id"], json_api_workspace_in_document
+        )
+        pprint(api_response)
+    except gooddata_api_client.ApiException as e:
+        pytest.fail(f"API exception: {e}")
+    except Exception as e:
+        pytest.fail(f"Unexpected error: {e}")
+
+
+def test_enable_early_access_per_workspace(api_client, test_config):
+    enable_early_access_per_workspace(api_client, test_config, "experimental-genai-chat")
+
+
+def test_metadata_sync(api_client, test_config):
+    api_instance = metadata_sync_api.MetadataSyncApi(api_client)
+    try:
+        print("Attempting to sync metadata...")
+        print("workspace_id", test_config["workspace_id"])
+
+        api_instance.metadata_sync(test_config["workspace_id"])
+    except gooddata_api_client.ApiException as e:
+        pytest.fail(f"API exception: {e}")
+    except Exception as e:
+        pytest.fail(f"Unexpected error: {e}")
+
+
+if __name__ == "__main__":
+    pytest.main(["-s", __file__])