Users/robrandao/data driven tests

dev/integration/data_driven_tests/basic_test.yaml

@@ -0,0 +1,25 @@
+parent: ./defaults.yaml
+description: A basic data driven test example
+defaults:
+  input:
+  sleep:
+  assertion:
+test:
+  - type: input
+    activity:
+      type: message
+      text: "Hello, World!"
+
+  - type: sleep
+    duration: 1
+
+  - type: assertion
+    quantifier: ONE
+    selector:
+      index: -1
+      quantifier: ONE
+      activity:
+        type: message
+    activity:
+      type: message
+      text: "Hello, World!"

dev/integration/data_driven_tests/defaults.yaml

@@ -0,0 +1,10 @@
+defaults:
+  all:
+    channel_id: "test_channel"
+    service_url: "http://localhost:3978"
+    locale: "en-US"
+    conversation_id: "test_conversation"
+  input:
+    user_id: "test_user"
+  output:
+    user_id: "test_bot"

dev/integration/integration/README.md

@@ -0,0 +1,2 @@
+# Microsoft 365 Agents SDK for Python Integration Testing Framework
+

dev/integration/integration/integration/foundational/_common.py

@@ -0,0 +1,16 @@
+import json
+
+from microsoft_agents.activity import Activity
+
+
+def load_activity(channel: str, name: str) -> Activity:
+
+    with open(
+        "./dev/integration/src/tests/integration/foundational/activities/{}/{}.json".format(
+            channel, name
+        ),
+        "r",
+    ) as f:
+        activity = json.load(f)
+
+    return Activity.model_validate(activity)

dev/integration/integration/integration/foundational/test_suite.py

@@ -0,0 +1,141 @@
+import json
+import pytest
+import asyncio
+
+from microsoft_agents.activity import (
+    ActivityTypes,
+)
+
+from src.core import integration, IntegrationFixtures, AiohttpEnvironment
+from src.samples import QuickstartSample
+
+from ._common import load_activity
+
+DIRECTLINE = "directline"
+
+@integration()
+class TestFoundation(IntegrationFixtures):
+
+    def load_activity(self, activity_name) -> Activity:
+        return load_activity(DIRECTLINE, activity_name)
+
+    @pytest.mark.asyncio
+    async def test__send_activity__sends_hello_world__returns_hello_world(self, agent_client):
+        activity = load_activity(DIRECTLINE, "hello_world.json")
+        result = await agent_client.send_activity(activity)
+        assert result is not None
+        last = result[-1]
+        assert last.type == ActivityTypes.message
+        assert last.text.lower() == "you said: {activity.text}".lower()
+
+    @pytest.mark.asyncio
+    async def test__send_invoke__send_basic_invoke_activity__receive_invoke_response(self, agent_client):
+        activity = load_activity(DIRECTLINE, "basic_invoke.json")
+        result = await agent_client.send_activity(activity)
+        assert result
+        data = json.loads(result)
+        message = data.get("message", {})
+        assert "Invoke received." in message
+        assert "data" in data
+        assert data["parameters"] and len(data["parameters"]) > 0
+        assert "hi" in data["value"]
+
+    @pytest.mark.asyncio
+    async def test__send_activity__sends_message_activity_to_ac_submit__return_valid_response(self, agent_client):
+        activity = load_activity(DIRECTLINE, "ac_submit.json")
+        result = await agent_client.send_activity(activity)
+        assert result is not None
+        last = result[-1]
+        assert last.type == ActivityTypes.message
+        assert "doStuff" in last.text
+        assert "Action.Submit" in last.text
+        assert "hello" in last.text
+
+    @pytest.mark.asyncio
+    async def test__send_invoke_sends_invoke_activity_to_ac_execute__returns_valid_adaptive_card_invoke_response(self, agent_client):
+        activity = load_activity(DIRECTLINE, "ac_execute.json")
+        result = await agent_client.send_invoke(activity)
+
+        result = json.loads(result)
+
+        assert result.status == 200
+        assert result.value
+
+        assert "application/vnd.microsoft.card.adaptive" in result.type
+
+        activity_data = json.loads(activity.value)
+        assert activity_data.get("action")
+        user_text = activity_data.get("usertext")
+        assert user_text in result.value
+
+    @pytest.mark.asyncio
+    async def test__send_activity_sends_text__returns_poem(self, agent_client):
+        pass
+
+    @pytest.mark.asyncio
+    async def test__send_expected_replies_activity__sends_text__returns_poem(self, agent_client):
+        activity = self.load_activity("expected_replies.json")
+        result = await agent_client.send_expected_replies(activity)
+        last = result[-1]
+        assert last.type == ActivityTypes.message
+        assert "Apollo" in last.text
+        assert "\n" in last.text
+
+    @pytest.mark.asyncio
+    async def test__send_invoke__query_link__returns_text(self, agent_client):
+        activity = self.load_activity("query_link.json")
+        result = await agent_client.send_invoke(activity)
+        pass # TODO
+
+    @pytest.mark.asyncio
+    async def test__send_invoke__select_item__receive_item(self, agent_client):
+        activity = self.load_activity("select_item.json")
+        result = await agent_client.send_invoke(activity)
+        pass # TODO
+
+    @pytest.mark.asyncio
+    async def test__send_activity__conversation_update__returns_welcome_message(self, agent_client):
+        activity = self.load_activity("conversation_update.json")
+        result = await agent_client.send_activity(activity)
+        last = result[-1]
+        assert "Hello and Welcome!" in last.text
+
+    @pytest.mark.asyncio
+    async def test__send_activity__send_heart_message_reaction__returns_message_reaction_heart(self, agent_client):
+        activity = self.load_activity("message_reaction_heart.json")
+        result = await agent_client.send_activity(activity)
+        last = result[-1]
+        assert last.type == ActivityTypes.message
+        assert "Message Reaction Added: heart" in last.text
+
+    @pytest.mark.asyncio
+    async def test__send_activity__remove_heart_message_reaction__returns_message_reaction_heart(self, agent_client):
+        activity = self.load_activity
+        result = await agent_client.send_activity(activity)
+        last = result[-1]
+        assert last.type == ActivityTypes.message
+        assert "Message Reaction Removed: heart" in last.text
+
+    @pytest.mark.asyncio
+    async def test__send_expected_replies_activity__send_seattle_today_weather__returns_weather(self, agent_client):
+        activity = self.load_activity("expected_replies_seattle_weather.json")
+        result = await agent_client.send_expected_replies(activity)
+        last = result[-1]
+        assert last.type == ActivityTypes.message
+        assert last.attachments and len(last.attachments) > 0
+
+        adaptive_card = last.attachments.first()
+        assert adaptive_card
+        assert "application/vnd.microsoft.card.adaptive" == adaptive_card.content_type
+        assert adaptive_card.content
+
+        assert \
+            "�" in adaptive_card.content or \
+            "\\u00B0" in adaptive_card.content or \
+            f"Missing temperature inside adaptive card: {adaptive_card.content}" in adaptive_card.content
+
+    @pytest.mark.asyncio
+    async def test__send_activity__simulate_message_loop__expect_question_about_time_and_returns_weather(self, agent_client):
+        activities = self.load_activity("message_loop_1.json")
+        fresult = await agent_client.send_activity(activities[0])
+        assert 

dev/integration/integration/integration/test_quickstart.py

@@ -0,0 +1,21 @@
+# import pytest
+# import asyncio
+
+# from src.core import IntegrationFixtures, AiohttpEnvironment
+# from src.samples import QuickstartSample
+
+# class TestQuickstart(Integration):
+#     _sample_cls = QuickstartSample
+#     _environment_cls = AiohttpEnvironment
+
+#     @pytest.mark.asyncio
+#     async def test_welcome_message(self, agent_client, response_client):
+#         res = await agent_client.send_expect_replies("hi")
+#         await asyncio.sleep(1)  # Wait for processing
+#         responses = await response_client.pop()
+
+#         assert len(responses) == 0
+
+#         first_non_typing = next((r for r in res if r.type != "typing"), None)
+#         assert first_non_typing is not None
+#         assert first_non_typing.text == "you said: hi"

dev/integration/integration/requirements.txt

@@ -0,0 +1 @@
+aioresponses

dev/integration/test_basics.py

@@ -0,0 +1,8 @@
+from microsoft_agents.testing import DataDrivenTester
+
+
+class TestBasics(DataDrivenTester):
+    _input_dir = "./data_driven_tests"
+
+    def test(self):
+        self._run_data_driven_test("input_file.json")

dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py

@@ -1,3 +1,6 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
 from .sdk_config import SDKConfig
 
 from .auth import generate_token, generate_token_from_config

dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from .activity_assertion import ActivityAssertion
+from .assertions import (
+    assert_activity,
+    assert_field,
+)
+from .check_activity import check_activity
+from .check_field import check_field
+from .type_defs import FieldAssertionType, SelectorQuantifier, AssertionQuantifier
+
+__all__ = [
+    "assert_activity",
+    "assert_field",
+    "check_activity",
+    "check_field",
+    "FieldAssertionType",
+]

dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py

@@ -0,0 +1,65 @@
+from typing import Optional
+
+from microsoft_agents.activity import Activity
+
+from .select_activity import select_activities
+from .check_activity import check_activity_verbose
+from .type_defs import AssertionQuantifier, AssertionErrorData
+
+
+class ActivityAssertion:
+
+    def __init__(self, config: dict) -> None:
+        """Initializes the ActivityAssertion with the given configuration.
+
+        :param config: The configuration dictionary containing quantifier, selector, and assertion.
+        """
+        quantifier_name = config.get("quantifier", AssertionQuantifier.ALL)
+        self._quantifier = AssertionQuantifier(quantifier_name)
+
+        self._selector = config.get("selector", {})
+        self._assertion = config.get("assertion", {})
+
+    @staticmethod
+    def _combine_assertion_errors(errors: list[AssertionErrorData]) -> str:
+        """Combines multiple assertion errors into a single string representation.
+
+        :param errors: The list of assertion errors to be combined.
+        :return: A string representation of the combined assertion errors.
+        """
+        return "\n".join(str(error) for error in errors)
+
+    def check(self, activities: list[Activity]) -> tuple[bool, Optional[str]]:
+        """Asserts that the given activities match the assertion criteria.
+
+        :param activities: The list of activities to be tested.
+        :return: A tuple containing a boolean indicating if the assertion passed and an optional error message.
+        """
+
+        activities = select_activities(activities, self._selector)
+
+        count = 0
+        for activity in activities:
+            res, assertion_error_data = check_activity_verbose(
+                activity, self._assertion
+            )
+            if self._quantifier == AssertionQuantifier.ALL and not res:
+                return (
+                    False,
+                    f"Activity did not match the assertion: {activity}\nError: {assertion_error_data}",
+                )
+            if self._quantifier == AssertionQuantifier.NONE and res:
+                return (
+                    False,
+                    f"Activity matched the assertion when none were expected: {activity}",
+                )
+            count += 1
+
+        passes = True
+        if self._quantifier == AssertionQuantifier.ONE and count != 1:
+            return (
+                False,
+                f"Expected exactly one activity to match the assertion, but found {count}.",
+            )
+
+        return passes, None

dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py

@@ -0,0 +1,52 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from typing import Any
+
+from microsoft_agents.activity import Activity
+
+from .type_defs import FieldAssertionType, AssertionQuantifier
+from .check_activity import check_activity_verbose
+from .check_field import check_field_verbose
+
+
+def assert_field(
+    actual_value: Any, assertion: Any, assertion_type: FieldAssertionType
+) -> None:
+    """Asserts that a specific field in the target matches the baseline.
+
+    :param key_in_baseline: The key of the field to be tested.
+    :param target: The target dictionary containing the actual values.
+    :param assertion: The baseline dictionary containing the expected values.
+    """
+    res, assertion_error_message = check_field_verbose(
+        actual_value, assertion, assertion_type
+    )
+    assert res, assertion_error_message
+
+
+def assert_activity(activity: Activity, assertion: Activity | dict) -> None:
+    """Asserts that the given activity matches the baseline activity.
+
+    :param activity: The activity to be tested.
+    :param assertion: The baseline activity or a dictionary representing the expected activity data.
+    """
+    res, assertion_error_data = check_activity_verbose(activity, assertion)
+    assert res, str(assertion_error_data)
+
+
+def assert_activities(activities: list[Activity], assertion_config: dict) -> None:
+    """Asserts that the given list of activities matches the baseline activities.
+
+    :param activities: The list of activities to be tested.
+    :param assertion: The baseline dictionary representing the expected activities data.
+    """
+
+    quantifier = assertion_config.get(
+        "quantifier",
+    )
+    selector = assertion_config.get("selector", {})
+
+    for activity in activities:
+        res, assertion_error_data = check_activity_verbose(activity, assertion)
+        assert res, str(assertion_error_data)