diff --git a/dev/integration/data_driven_tests/basic_test.json b/dev/integration/data_driven_tests/basic_test.json new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/data_driven_tests/basic_test.yaml b/dev/integration/data_driven_tests/basic_test.yaml new file mode 100644 index 00000000..a9199fbb --- /dev/null +++ b/dev/integration/data_driven_tests/basic_test.yaml @@ -0,0 +1,25 @@ +parent: ./defaults.yaml +description: A basic data driven test example +defaults: + input: + sleep: + assertion: +test: + - type: input + activity: + type: message + text: "Hello, World!" + + - type: sleep + duration: 1 + + - type: assertion + quantifier: ONE + selector: + index: -1 + quantifier: ONE + activity: + type: message + activity: + type: message + text: "Hello, World!" \ No newline at end of file diff --git a/dev/integration/data_driven_tests/defaults.yaml b/dev/integration/data_driven_tests/defaults.yaml new file mode 100644 index 00000000..6d65a3b3 --- /dev/null +++ b/dev/integration/data_driven_tests/defaults.yaml @@ -0,0 +1,10 @@ +defaults: + all: + channel_id: "test_channel" + service_url: "http://localhost:3978" + locale: "en-US" + conversation_id: "test_conversation" + input: + user_id: "test_user" + output: + user_id: "test_bot" \ No newline at end of file diff --git a/dev/integration/integration/README.md b/dev/integration/integration/README.md new file mode 100644 index 00000000..614462c9 --- /dev/null +++ b/dev/integration/integration/README.md @@ -0,0 +1,2 @@ +# Microsoft 365 Agents SDK for Python Integration Testing Framework + diff --git a/dev/integration/integration/integration/__init__.py b/dev/integration/integration/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/integration/integration/foundational/__init__.py b/dev/integration/integration/integration/foundational/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/integration/integration/foundational/_common.py b/dev/integration/integration/integration/foundational/_common.py new file mode 100644 index 00000000..f4d22304 --- /dev/null +++ b/dev/integration/integration/integration/foundational/_common.py @@ -0,0 +1,16 @@ +import json + +from microsoft_agents.activity import Activity + + +def load_activity(channel: str, name: str) -> Activity: + + with open( + "./dev/integration/src/tests/integration/foundational/activities/{}/{}.json".format( + channel, name + ), + "r", + ) as f: + activity = json.load(f) + + return Activity.model_validate(activity) diff --git a/dev/integration/integration/integration/foundational/test_suite.py b/dev/integration/integration/integration/foundational/test_suite.py new file mode 100644 index 00000000..a38edf0b --- /dev/null +++ b/dev/integration/integration/integration/foundational/test_suite.py @@ -0,0 +1,141 @@ +import json +import pytest +import asyncio + +from microsoft_agents.activity import ( + ActivityTypes, +) + +from src.core import integration, IntegrationFixtures, AiohttpEnvironment +from src.samples import QuickstartSample + +from ._common import load_activity + +DIRECTLINE = "directline" + +@integration() +class TestFoundation(IntegrationFixtures): + + def load_activity(self, activity_name) -> Activity: + return load_activity(DIRECTLINE, activity_name) + + @pytest.mark.asyncio + async def test__send_activity__sends_hello_world__returns_hello_world(self, agent_client): + activity = load_activity(DIRECTLINE, "hello_world.json") + result = await agent_client.send_activity(activity) + assert result is not None + last = result[-1] + assert last.type == ActivityTypes.message + assert last.text.lower() == "you said: {activity.text}".lower() + + @pytest.mark.asyncio + async def test__send_invoke__send_basic_invoke_activity__receive_invoke_response(self, agent_client): + activity = load_activity(DIRECTLINE, "basic_invoke.json") + result = await agent_client.send_activity(activity) + assert result + data = json.loads(result) + message = data.get("message", {}) + assert "Invoke received." in message + assert "data" in data + assert data["parameters"] and len(data["parameters"]) > 0 + assert "hi" in data["value"] + + @pytest.mark.asyncio + async def test__send_activity__sends_message_activity_to_ac_submit__return_valid_response(self, agent_client): + activity = load_activity(DIRECTLINE, "ac_submit.json") + result = await agent_client.send_activity(activity) + assert result is not None + last = result[-1] + assert last.type == ActivityTypes.message + assert "doStuff" in last.text + assert "Action.Submit" in last.text + assert "hello" in last.text + + @pytest.mark.asyncio + async def test__send_invoke_sends_invoke_activity_to_ac_execute__returns_valid_adaptive_card_invoke_response(self, agent_client): + activity = load_activity(DIRECTLINE, "ac_execute.json") + result = await agent_client.send_invoke(activity) + + result = json.loads(result) + + assert result.status == 200 + assert result.value + + assert "application/vnd.microsoft.card.adaptive" in result.type + + activity_data = json.loads(activity.value) + assert activity_data.get("action") + user_text = activity_data.get("usertext") + assert user_text in result.value + + @pytest.mark.asyncio + async def test__send_activity_sends_text__returns_poem(self, agent_client): + pass + + @pytest.mark.asyncio + async def test__send_expected_replies_activity__sends_text__returns_poem(self, agent_client): + activity = self.load_activity("expected_replies.json") + result = await agent_client.send_expected_replies(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert "Apollo" in last.text + assert "\n" in last.text + + @pytest.mark.asyncio + async def test__send_invoke__query_link__returns_text(self, agent_client): + activity = self.load_activity("query_link.json") + result = await agent_client.send_invoke(activity) + pass # TODO + + @pytest.mark.asyncio + async def test__send_invoke__select_item__receive_item(self, agent_client): + activity = self.load_activity("select_item.json") + result = await agent_client.send_invoke(activity) + pass # TODO + + @pytest.mark.asyncio + async def test__send_activity__conversation_update__returns_welcome_message(self, agent_client): + activity = self.load_activity("conversation_update.json") + result = await agent_client.send_activity(activity) + last = result[-1] + assert "Hello and Welcome!" in last.text + + @pytest.mark.asyncio + async def test__send_activity__send_heart_message_reaction__returns_message_reaction_heart(self, agent_client): + activity = self.load_activity("message_reaction_heart.json") + result = await agent_client.send_activity(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert "Message Reaction Added: heart" in last.text + + @pytest.mark.asyncio + async def test__send_activity__remove_heart_message_reaction__returns_message_reaction_heart(self, agent_client): + activity = self.load_activity + result = await agent_client.send_activity(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert "Message Reaction Removed: heart" in last.text + + @pytest.mark.asyncio + async def test__send_expected_replies_activity__send_seattle_today_weather__returns_weather(self, agent_client): + activity = self.load_activity("expected_replies_seattle_weather.json") + result = await agent_client.send_expected_replies(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert last.attachments and len(last.attachments) > 0 + + adaptive_card = last.attachments.first() + assert adaptive_card + assert "application/vnd.microsoft.card.adaptive" == adaptive_card.content_type + assert adaptive_card.content + + assert \ + "�" in adaptive_card.content or \ + "\\u00B0" in adaptive_card.content or \ + f"Missing temperature inside adaptive card: {adaptive_card.content}" in adaptive_card.content + + @pytest.mark.asyncio + async def test__send_activity__simulate_message_loop__expect_question_about_time_and_returns_weather(self, agent_client): + activities = self.load_activity("message_loop_1.json") + fresult = await agent_client.send_activity(activities[0]) + assert \ No newline at end of file diff --git a/dev/integration/integration/integration/test_quickstart.py b/dev/integration/integration/integration/test_quickstart.py new file mode 100644 index 00000000..967d4ebc --- /dev/null +++ b/dev/integration/integration/integration/test_quickstart.py @@ -0,0 +1,21 @@ +# import pytest +# import asyncio + +# from src.core import IntegrationFixtures, AiohttpEnvironment +# from src.samples import QuickstartSample + +# class TestQuickstart(Integration): +# _sample_cls = QuickstartSample +# _environment_cls = AiohttpEnvironment + +# @pytest.mark.asyncio +# async def test_welcome_message(self, agent_client, response_client): +# res = await agent_client.send_expect_replies("hi") +# await asyncio.sleep(1) # Wait for processing +# responses = await response_client.pop() + +# assert len(responses) == 0 + +# first_non_typing = next((r for r in res if r.type != "typing"), None) +# assert first_non_typing is not None +# assert first_non_typing.text == "you said: hi" diff --git a/dev/integration/integration/requirements.txt b/dev/integration/integration/requirements.txt new file mode 100644 index 00000000..d524e63a --- /dev/null +++ b/dev/integration/integration/requirements.txt @@ -0,0 +1 @@ +aioresponses \ No newline at end of file diff --git a/dev/integration/test_basics.py b/dev/integration/test_basics.py new file mode 100644 index 00000000..28c2244e --- /dev/null +++ b/dev/integration/test_basics.py @@ -0,0 +1,8 @@ +from microsoft_agents.testing import DataDrivenTester + + +class TestBasics(DataDrivenTester): + _input_dir = "./data_driven_tests" + + def test(self): + self._run_data_driven_test("input_file.json") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py index c8364fff..b85d37e9 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .sdk_config import SDKConfig from .auth import generate_token, generate_token_from_config diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py new file mode 100644 index 00000000..9e8695be --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .activity_assertion import ActivityAssertion +from .assertions import ( + assert_activity, + assert_field, +) +from .check_activity import check_activity +from .check_field import check_field +from .type_defs import FieldAssertionType, SelectorQuantifier, AssertionQuantifier + +__all__ = [ + "assert_activity", + "assert_field", + "check_activity", + "check_field", + "FieldAssertionType", +] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py new file mode 100644 index 00000000..57299304 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py @@ -0,0 +1,65 @@ +from typing import Optional + +from microsoft_agents.activity import Activity + +from .select_activity import select_activities +from .check_activity import check_activity_verbose +from .type_defs import AssertionQuantifier, AssertionErrorData + + +class ActivityAssertion: + + def __init__(self, config: dict) -> None: + """Initializes the ActivityAssertion with the given configuration. + + :param config: The configuration dictionary containing quantifier, selector, and assertion. + """ + quantifier_name = config.get("quantifier", AssertionQuantifier.ALL) + self._quantifier = AssertionQuantifier(quantifier_name) + + self._selector = config.get("selector", {}) + self._assertion = config.get("assertion", {}) + + @staticmethod + def _combine_assertion_errors(errors: list[AssertionErrorData]) -> str: + """Combines multiple assertion errors into a single string representation. + + :param errors: The list of assertion errors to be combined. + :return: A string representation of the combined assertion errors. + """ + return "\n".join(str(error) for error in errors) + + def check(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: + """Asserts that the given activities match the assertion criteria. + + :param activities: The list of activities to be tested. + :return: A tuple containing a boolean indicating if the assertion passed and an optional error message. + """ + + activities = select_activities(activities, self._selector) + + count = 0 + for activity in activities: + res, assertion_error_data = check_activity_verbose( + activity, self._assertion + ) + if self._quantifier == AssertionQuantifier.ALL and not res: + return ( + False, + f"Activity did not match the assertion: {activity}\nError: {assertion_error_data}", + ) + if self._quantifier == AssertionQuantifier.NONE and res: + return ( + False, + f"Activity matched the assertion when none were expected: {activity}", + ) + count += 1 + + passes = True + if self._quantifier == AssertionQuantifier.ONE and count != 1: + return ( + False, + f"Expected exactly one activity to match the assertion, but found {count}.", + ) + + return passes, None diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py new file mode 100644 index 00000000..f778b191 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -0,0 +1,52 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Any + +from microsoft_agents.activity import Activity + +from .type_defs import FieldAssertionType, AssertionQuantifier +from .check_activity import check_activity_verbose +from .check_field import check_field_verbose + + +def assert_field( + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType +) -> None: + """Asserts that a specific field in the target matches the baseline. + + :param key_in_baseline: The key of the field to be tested. + :param target: The target dictionary containing the actual values. + :param assertion: The baseline dictionary containing the expected values. + """ + res, assertion_error_message = check_field_verbose( + actual_value, assertion, assertion_type + ) + assert res, assertion_error_message + + +def assert_activity(activity: Activity, assertion: Activity | dict) -> None: + """Asserts that the given activity matches the baseline activity. + + :param activity: The activity to be tested. + :param assertion: The baseline activity or a dictionary representing the expected activity data. + """ + res, assertion_error_data = check_activity_verbose(activity, assertion) + assert res, str(assertion_error_data) + + +def assert_activities(activities: list[Activity], assertion_config: dict) -> None: + """Asserts that the given list of activities matches the baseline activities. + + :param activities: The list of activities to be tested. + :param assertion: The baseline dictionary representing the expected activities data. + """ + + quantifier = assertion_config.get( + "quantifier", + ) + selector = assertion_config.get("selector", {}) + + for activity in activities: + res, assertion_error_data = check_activity_verbose(activity, assertion) + assert res, str(assertion_error_data) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py new file mode 100644 index 00000000..ba286c22 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Any, TypeVar, Optional + +from microsoft_agents.activity import Activity +from microsoft_agents.testing.utils import normalize_activity_data + +from .check_field import check_field, _parse_assertion +from .type_defs import UNSET_FIELD, FieldAssertionType, AssertionErrorData + + +def _check( + actual: Any, baseline: Any, field_path: str = "" +) -> tuple[bool, Optional[AssertionErrorData]]: + + assertion, assertion_type = _parse_assertion(baseline) + + if assertion_type is None: + if isinstance(baseline, dict): + for key in baseline: + new_field_path = f"{field_path}.{key}" if field_path else key + new_actual = actual.get(key, UNSET_FIELD) + new_baseline = baseline[key] + + res, assertion_error_data = _check( + new_actual, new_baseline, new_field_path + ) + if not res: + return False, assertion_error_data + return True, None + + elif isinstance(baseline, list): + for index, item in enumerate(baseline): + new_field_path = ( + f"{field_path}[{index}]" if field_path else f"[{index}]" + ) + new_actual = actual[index] if index < len(actual) else UNSET_FIELD + new_baseline = item + + res, assertion_error_data = _check( + new_actual, new_baseline, new_field_path + ) + if not res: + return False, assertion_error_data + return True, None + else: + raise ValueError("Unsupported baseline type for complex assertion.") + else: + assert isinstance(assertion_type, FieldAssertionType) + res = check_field(actual, assertion, assertion_type) + if res: + return True, None + else: + assertion_error_data = AssertionErrorData( + field_path=field_path, + actual_value=actual, + assertion=assertion, + assertion_type=assertion_type, + ) + return False, assertion_error_data + + +def check_activity(activity: Activity, baseline: Activity | dict) -> bool: + """Asserts that the given activity matches the baseline activity. + + :param activity: The activity to be tested. + :param baseline: The baseline activity or a dictionary representing the expected activity data. + """ + return check_activity_verbose(activity, baseline)[0] + + +def check_activity_verbose( + activity: Activity, baseline: Activity | dict +) -> tuple[bool, Optional[AssertionErrorData]]: + """Asserts that the given activity matches the baseline activity. + + :param activity: The activity to be tested. + :param baseline: The baseline activity or a dictionary representing the expected activity data. + """ + actual_activity = normalize_activity_data(activity) + baseline = normalize_activity_data(baseline) + return _check(actual_activity, baseline, "activity") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py new file mode 100644 index 00000000..77c8938c --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -0,0 +1,97 @@ +import re +from typing import Any, Optional + +from .type_defs import FieldAssertionType, UNSET_FIELD + +_OPERATIONS = { + FieldAssertionType.EQUALS: lambda a, b: a == b or (a is UNSET_FIELD and b is None), + FieldAssertionType.NOT_EQUALS: lambda a, b: a != b + or (a is UNSET_FIELD and b is not None), + FieldAssertionType.GREATER_THAN: lambda a, b: a > b, + FieldAssertionType.LESS_THAN: lambda a, b: a < b, + FieldAssertionType.CONTAINS: lambda a, b: b in a, + FieldAssertionType.NOT_CONTAINS: lambda a, b: b not in a, + FieldAssertionType.RE_MATCH: lambda a, b: re.match(b, a) is not None, +} + + +def _parse_assertion(field: Any) -> tuple[Any, Optional[FieldAssertionType]]: + """Parses the assertion information and returns the assertion type and baseline value. + + :param assertion_info: The assertion information to be parsed. + :return: A tuple containing the assertion type and baseline value. + """ + + assertion_type = FieldAssertionType.EQUALS + assertion = None + + if ( + isinstance(field, dict) + and "assertion_type" in field + and "assertion" in field + and field["assertion_type"] in FieldAssertionType.__members__ + ): + # format: + # {"assertion_type": "__EQ__", "assertion": "value"} + assertion_type = FieldAssertionType[field["assertion_type"]] + assertion = field.get("assertion") + + elif ( + isinstance(field, list) + and len(field) >= 2 + and isinstance(field[0], str) + and field[0] in FieldAssertionType.__members__ + ): + # format: + # ["__EQ__", "assertion"] + assertion_type = FieldAssertionType[field[0]] + assertion = field[1] + elif isinstance(field, list) or isinstance(field, dict): + assertion_type = None + else: + # default format: direct value + assertion = field + + return assertion, assertion_type + + +def check_field( + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType +) -> bool: + """Checks if the actual value satisfies the given assertion based on the assertion type. + + :param actual_value: The value to be checked. + :param assertion: The expected value or pattern to check against. + :param assertion_type: The type of assertion to perform. + :return: True if the assertion is satisfied, False otherwise. + """ + + operation = _OPERATIONS.get(assertion_type) + if not operation: + raise ValueError(f"Unsupported assertion type: {assertion_type}") + return operation(actual_value, assertion) + + +def check_field_verbose( + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType +) -> tuple[bool, Optional[str]]: + """Checks if the actual value satisfies the given assertion based on the assertion type. + + :param actual_value: The value to be checked. + :param assertion: The expected value or pattern to check against. + :param assertion_type: The type of assertion to perform. + :return: A tuple containing a boolean indicating if the assertion is satisfied and an optional error message. + """ + + operation = _OPERATIONS.get(assertion_type) + if not operation: + raise ValueError(f"Unsupported assertion type: {assertion_type}") + + result = operation(actual_value, assertion) + if result: + return True, None + else: + return ( + False, + f"Assertion failed: actual value '{actual_value}' does not satisfy '{assertion_type.name}' with assertion '{assertion}'", + ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py new file mode 100644 index 00000000..666d6e77 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py @@ -0,0 +1,56 @@ +from typing import Optional + +from microsoft_agents.activity import Activity + +from .check_activity import check_activity +from .type_defs import SelectorQuantifier + + +def select_activity( + activities: list[Activity], + selector: Activity, + index: int = 0, +) -> Optional[Activity]: + """Selects a single activity from a list based on the provided selector and index. + + :param activities: List of activities to select from. + :param selector: Activity used as a selector. + :param index: Index of the activity to select when multiple match. + :return: The selected activity or None if no activity matches. + """ + res = select_activities( + activities, + {"selector": selector, "quantifier": SelectorQuantifier.ONE, "index": index}, + ) + return res[index] if res else None + + +def select_activities( + activities: list[Activity], selector_config: dict +) -> list[Activity]: + """Selects activities from a list based on the provided selector configuration. + + :param activities: List of activities to select from. + :param selector_config: Configuration dict containing 'selector', 'quantifier', and optionally ' + :return: List of selected activities. + """ + + selector = selector_config.get("selector", {}) + + index = selector_config.get("index", None) + if index is not None: + quantifier_name = selector_config.get("quantifier", SelectorQuantifier.ONE) + else: + quantifier_name = selector_config.get("quantifier", SelectorQuantifier.ALL) + quantifier_name = quantifier_name.upper() + + if quantifier_name not in SelectorQuantifier.__members__: + raise ValueError(f"Invalid quantifier: {quantifier_name}") + + quantifier = SelectorQuantifier(quantifier_name) + + if quantifier == SelectorQuantifier.ALL: + return list(filter(lambda a: check_activity(a, selector), activities)) + else: + first = next(filter(lambda a: check_activity(a, selector), activities), None) + return [first] if first is not None else [] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py new file mode 100644 index 00000000..72b614d8 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py @@ -0,0 +1,63 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from enum import Enum +from dataclasses import dataclass +from typing import Any + + +class UNSET_FIELD: + """Singleton to represent an unset field in activity comparisons.""" + + @staticmethod + def get(*args, **kwargs): + """Returns the singleton instance.""" + return UNSET_FIELD + + +class FieldAssertionType(str, Enum): + """Defines the types of assertions that can be made on fields.""" + + EQUALS = "EQUALS" + NOT_EQUALS = "NOT_EQUALS" + GREATER_THAN = "GREATER_THAN" + LESS_THAN = "LESS_THAN" + CONTAINS = "CONTAINS" + NOT_CONTAINS = "NOT_CONTAINS" + IN = "IN" + NOT_IN = "NOT_IN" + RE_MATCH = "RE_MATCH" + + +class SelectorQuantifier(str, Enum): + """Defines quantifiers for selecting activities.""" + + ALL = "ALL" + ONE = "ONE" + + +class AssertionQuantifier(str, Enum): + """Defines quantifiers for assertions on activities.""" + + ANY = "ANY" + ALL = "ALL" + ONE = "ONE" + NONE = "NONE" + + +@dataclass +class AssertionErrorData: + """Data class to hold information about assertion errors.""" + + field_path: str + actual_value: Any + assertion: Any + assertion_type: FieldAssertionType + + def __str__(self) -> str: + return ( + f"Assertion failed at '{self.field_path}': " + f"actual value '{self.actual_value}' " + f"does not satisfy assertion '{self.assertion}' " + f"of type '{self.assertion_type}'." + ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py index 3fe2a78f..80bb0402 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .generate_token import generate_token, generate_token_from_config __all__ = ["generate_token", "generate_token_from_config"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py index 83106639..f96f6d9e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import requests from microsoft_agents.hosting.core import AgentAuthConfiguration diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py index 3ad1e376..cdc0f0df 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .core import ( AgentClient, ApplicationRunner, diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py index 9c69a2ae..a1161336 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .application_runner import ApplicationRunner from .aiohttp import AiohttpEnvironment from .client import ( diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py index 82d2d1d0..4625620e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .aiohttp_environment import AiohttpEnvironment from .aiohttp_runner import AiohttpRunner diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py index 7ff83dc0..cd630697 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from aiohttp.web import Request, Response, Application from microsoft_agents.hosting.aiohttp import ( diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py index c8fe23c2..518c9b5a 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from typing import Optional from threading import Thread, Event import asyncio diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py index ebbc56f9..9c77d745 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import asyncio from abc import ABC, abstractmethod from typing import Any, Optional diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py index 1d59411e..7b778407 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .agent_client import AgentClient from .response_client import ResponseClient diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py index 73067207..0928dfff 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import json import asyncio from typing import Optional, cast diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py index b283efdf..e22a10df 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from __future__ import annotations import sys diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py index 0aa99f24..a351e735 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from abc import ABC, abstractmethod from typing import Awaitable, Callable @@ -34,7 +37,7 @@ async def init_env(self, environ_config: dict) -> None: @abstractmethod def create_runner(self, *args, **kwargs) -> ApplicationRunner: """Create an application runner for the environment. - + Subclasses may accept additional arguments as needed. """ raise NotImplementedError() diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py index 3b459617..1b0673f5 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import pytest import os diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py index 6dde3668..d97298cc 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from abc import ABC, abstractmethod from .environment import Environment diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py new file mode 100644 index 00000000..aab93742 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py @@ -0,0 +1,4 @@ +from .data_driven_test import DataDrivenTest +from .ddt import ddt + +__all__ = ["DataDrivenTest", "ddt"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py new file mode 100644 index 00000000..39637acc --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py @@ -0,0 +1,63 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License.s + +import asyncio + +from copy import deepcopy + +from microsoft_agents.activity import Activity + +from microsoft_agents.testing.assertions import ( + ActivityAssertion, + assert_activity, +) + + +class DataDrivenTest: + + def __init__(self, test_flow: dict) -> None: + self._description = test_flow.get("description", "") + + defaults = test_flow.get("defaults", {}) + self._input_defaults = defaults.get("input", {}) + self._assertion_defaults = defaults.get("assertion", {}) + self._sleep_defaults = defaults.get("sleep", {}) + + self._test = test_flow.get("test", []) + + def _load_input(self, input_data: dict) -> Activity: + data = deepcopy(self._input_defaults) + data.update(input_data) + return Activity.model_validate(data) + + def _load_assertion(self, assertion_data: dict) -> dict: + data = deepcopy(self._assertion_defaults) + data.update(assertion_data) + return data + + async def _sleep(self, sleep_data: dict) -> None: + duration = sleep_data.get("duration") + if duration is None: + duration = self._sleep_defaults.get("duration", 0) + await asyncio.sleep(duration) + + async def run(self, agent_client, response_client) -> None: + + responses = [] + for step in self._test: + step_type = step.get("type") + if not step_type: + raise ValueError("Each step must have a 'type' field.") + + if step_type == "input": + input_activity = self._load_input(step) + await agent_client.send_activity(input_activity) + elif step_type == "assertion": + + responses.extend(await response_client.pop()) + + activity_assertion = ActivityAssertion(step) + assert activity_assertion.check(responses) + + elif step_type == "sleep": + await self._sleep(step) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py new file mode 100644 index 00000000..a627e7e3 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py @@ -0,0 +1,41 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Callable + +import pytest + +from microsoft_agents.testing.integration.core import Integration + +from .data_driven_test import DataDrivenTest + +def _add_test_method(test_cls: Integration, test_path: str, base_dir: str) -> None: + + test_case_name = f"test_data_driven__{test_path.replace('/', '_').replace('.', '_')}" + + @pytest.mark.asyncio + async def _func(self, agent_client, response_client) -> None: + ddt = DataDrivenTest(f"{base_dir}/{test_path}") + await ddt.run(agent_client, response_client) + + setattr(test_cls, test_case_name, func) + +def ddt(test_path: str) -> Callable[[Integration], Integration]: + + def decorator(test_cls: Integration) -> Integration: + + test_case_name = f"test_data_driven__{test_path.replace('/', '_').replace('.', '_')}" + + async def func(self, agent_client, response_client) -> None: + ddt = DataDrivenTest(test_path) + + responses = [] + + await for step in ddt: + if isinstance(step, Activity): + await agent_client.send_activity(step) + elif isinstance(step, dict): + # assertion + responses.extend(await response_client.pop()) + + return decorator \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py new file mode 100644 index 00000000..f09f9bf5 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py @@ -0,0 +1,59 @@ +import asyncio + +from copy import deepcopy +from typing import Awaitable, Callable + +from microsoft_agents.activity import Activity + +from microsoft_agents.testing.assertions import assert_activity + + +class DataDrivenTest: + + def __init__(self, test_flow: dict) -> None: + self._description = test_flow.get("description", "") + + defaults = test_flow.get("defaults", {}) + self._input_defaults = defaults.get("input", {}) + self._assertion_defaults = defaults.get("assertion", {}) + self._sleep_defaults = defaults.get("sleep", {}) + + self._test = test_flow.get("test", []) + + def _load_input(self, input_data: dict) -> Activity: + data = deepcopy(self._input_defaults) + data.update(input_data) + return Activity.model_validate(data) + + def _load_assertion(self, assertion_data: dict) -> dict: + data = deepcopy(self._assertion_defaults) + data.update(assertion_data) + return data + + async def _sleep(self, sleep_data: dict) -> None: + duration = sleep_data.get("duration") + if duration is None: + duration = self._sleep_defaults.get("duration", 0) + await asyncio.sleep(duration) + + async def run(self, agent_client, response_client) -> None: + + for step in self._test: + step_type = step.get("type") + if not step_type: + raise ValueError("Each step must have a 'type' field.") + + if step_type == "input": + input_activity = self._load_input(step) + await agent_client.send_activity(input_activity) + elif step_type == "assertion": + assertion = self._load_assertion(step) + responses = await response_client.pop() + + selector = Selector(assertion.get("selector", {})) + selection = selector.select(responses) + + assert_activity(selection, assertion) + + elif step_type == "sleep": + await self._sleep(step) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py index c1824ae5..61e1def8 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import os from copy import deepcopy from dotenv import load_dotenv, dotenv_values diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py index 0c902992..fd1102ff 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -1,7 +1,11 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .populate_activity import populate_activity -from .urls import get_host_and_port +from .misc import get_host_and_port, normalize_activity_data __all__ = [ "populate_activity", "get_host_and_port", + "normalize_activity_data", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py new file mode 100644 index 00000000..ea6cda35 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from urllib.parse import urlparse + +from microsoft_agents.activity import Activity + + +def get_host_and_port(url: str) -> tuple[str, int]: + """Extract host and port from a URL.""" + + parsed_url = urlparse(url) + host = parsed_url.hostname or "localhost" + port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) + return host, port + + +def normalize_activity_data(source: Activity | dict) -> dict: + """Normalize Activity data to a dictionary format.""" + + if isinstance(source, Activity): + return source.model_dump(exclude_unset=True, mode="json") + return source diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py index a6b1c19f..7d1edea8 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from microsoft_agents.activity import Activity @@ -13,4 +16,4 @@ def populate_activity(original: Activity, defaults: Activity | dict) -> Activity if getattr(new_activity, key) is None: setattr(new_activity, key, defaults[key]) - return new_activity \ No newline at end of file + return new_activity diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py index d964ebd2..7cc8fe9e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from urllib.parse import urlparse diff --git a/dev/microsoft-agents-testing/pyproject.toml b/dev/microsoft-agents-testing/pyproject.toml index cf659e6f..5557ac38 100644 --- a/dev/microsoft-agents-testing/pyproject.toml +++ b/dev/microsoft-agents-testing/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "microsoft-agents-hosting-core" +name = "microsoft-agents-testing" dynamic = ["version", "dependencies"] description = "Core library for Microsoft Agents" readme = {file = "README.md", content-type = "text/markdown"} diff --git a/dev/microsoft-agents-testing/pytest.ini b/dev/microsoft-agents-testing/pytest.ini new file mode 100644 index 00000000..479894a8 --- /dev/null +++ b/dev/microsoft-agents-testing/pytest.ini @@ -0,0 +1,39 @@ +[pytest] +# Pytest configuration for Microsoft Agents for Python + +# Treat all warnings as errors by default +# This ensures that any code generating warnings will fail tests, +# promoting cleaner code and early detection of issues +filterwarnings = + error + # Ignore specific warnings that are not actionable or are from dependencies + ignore::DeprecationWarning:pkg_resources.* + ignore::DeprecationWarning:setuptools.* + ignore::PendingDeprecationWarning + # pytest-asyncio warnings that are safe to ignore + ignore:.*deprecated.*asyncio.*:DeprecationWarning:pytest_asyncio.* + +# Test discovery configuration +testpaths = tests +python_files = test_*.py *_test.py +python_classes = Test* +python_functions = test_* + +# Output configuration +addopts = + --strict-markers + --strict-config + --verbose + --tb=short + --durations=10 + +# Minimum version requirement +minversion = 6.0 + +# Markers for test categorization +markers = + unit: Unit tests + integration: Integration tests + slow: Slow tests that may take longer to run + requires_network: Tests that require network access + requires_auth: Tests that require authentication \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/__init__.py b/dev/microsoft-agents-testing/tests/assertions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/assertions/_common.py b/dev/microsoft-agents-testing/tests/assertions/_common.py new file mode 100644 index 00000000..0a69960c --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/_common.py @@ -0,0 +1,18 @@ +import pytest + +from microsoft_agents.activity import Activity + + +@pytest.fixture +def activity(): + return Activity(type="message", text="Hello, World!") + + +@pytest.fixture( + params=[ + Activity(type="message", text="Hello, World!"), + {"type": "message", "text": "Hello, World!"}, + ] +) +def baseline(request): + return request.param diff --git a/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py new file mode 100644 index 00000000..283e101e --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py @@ -0,0 +1,319 @@ +import pytest + +from microsoft_agents.activity import Activity, Attachment + +from microsoft_agents.testing.assertions.type_defs import FieldAssertionType +from microsoft_agents.testing.assertions.assertions import assert_activity +from microsoft_agents.testing.assertions.check_field import _parse_assertion + + +class TestParseAssertion: + + @pytest.fixture( + params=[ + FieldAssertionType.EQUALS, + FieldAssertionType.NOT_EQUALS, + FieldAssertionType.GREATER_THAN, + ] + ) + def assertion_type_str(self, request): + return request.param + + @pytest.fixture(params=["simple_value", {"key": "value"}, 42]) + def assertion_value(self, request): + return request.param + + def test_parse_assertion_dict(self, assertion_value, assertion_type_str): + + assertion, assertion_type = _parse_assertion( + {"assertion_type": assertion_type_str, "assertion": assertion_value} + ) + assert assertion == assertion_value + assert assertion_type == FieldAssertionType(assertion_type_str) + + def test_parse_assertion_list(self, assertion_value, assertion_type_str): + assertion, assertion_type = _parse_assertion( + [assertion_type_str, assertion_value] + ) + assert assertion == assertion_value + assert assertion_type.value == assertion_type_str + + @pytest.mark.parametrize( + "field", + ["value", 123, 12.34], + ) + def test_parse_assertion_default(self, field): + assertion, assertion_type = _parse_assertion(field) + assert assertion == field + assert assertion_type == FieldAssertionType.EQUALS + + @pytest.mark.parametrize( + "field", + [ + {"assertion_type": FieldAssertionType.IN}, + {"assertion_type": FieldAssertionType.IN, "key": "value"}, + [FieldAssertionType.RE_MATCH], + [], + {"assertion_type": "invalid", "assertion": "test"}, + ], + ) + def test_parse_assertion_none(self, field): + assertion, assertion_type = _parse_assertion(field) + assert assertion is None + assert assertion_type is None + + +class TestAssertActivity: + """Tests for assert_activity function.""" + + def test_assert_activity_with_matching_simple_fields(self): + """Test that activity matches baseline with simple equal fields.""" + activity = Activity(type="message", text="Hello, World!") + baseline = {"type": "message", "text": "Hello, World!"} + assert_activity(activity, baseline) + + def test_assert_activity_with_non_matching_fields(self): + """Test that activity doesn't match baseline with different field values.""" + activity = Activity(type="message", text="Hello") + baseline = {"type": "message", "text": "Goodbye"} + assert_activity(activity, baseline) + + def test_assert_activity_with_activity_baseline(self): + """Test that baseline can be an Activity object.""" + activity = Activity(type="message", text="Hello") + baseline = Activity(type="message", text="Hello") + assert_activity(activity, baseline) + + def test_assert_activity_with_partial_baseline(self): + """Test that only fields in baseline are checked.""" + activity = Activity( + type="message", + text="Hello", + channel_id="test-channel", + conversation={"id": "conv123"}, + ) + baseline = {"type": "message", "text": "Hello"} + assert_activity(activity, baseline) + + def test_assert_activity_with_missing_field(self): + """Test that activity with missing field doesn't match baseline.""" + activity = Activity(type="message") + baseline = {"type": "message", "text": "Hello"} + assert_activity(activity, baseline) + + def test_assert_activity_with_none_values(self): + """Test that None values are handled correctly.""" + activity = Activity(type="message") + baseline = {"type": "message", "text": None} + assert_activity(activity, baseline) + + def test_assert_activity_with_empty_baseline(self): + """Test that empty baseline always matches.""" + activity = Activity(type="message", text="Hello") + baseline = {} + assert_activity(activity, baseline) + + def test_assert_activity_with_dict_assertion_format(self): + """Test using dict format for assertions.""" + activity = Activity(type="message", text="Hello, World!") + baseline = { + "type": "message", + "text": {"assertion_type": "CONTAINS", "assertion": "Hello"}, + } + assert_activity(activity, baseline) + + def test_assert_activity_with_list_assertion_format(self): + """Test using list format for assertions.""" + activity = Activity(type="message", text="Hello, World!") + baseline = {"type": "message", "text": ["CONTAINS", "World"]} + assert_activity(activity, baseline) + + def test_assert_activity_with_not_equals_assertion(self): + """Test NOT_EQUALS assertion type.""" + activity = Activity(type="message", text="Hello") + baseline = { + "type": "message", + "text": {"assertion_type": "NOT_EQUALS", "assertion": "Goodbye"}, + } + assert_activity(activity, baseline) + + def test_assert_activity_with_contains_assertion(self): + """Test CONTAINS assertion type.""" + activity = Activity(type="message", text="Hello, World!") + baseline = {"text": {"assertion_type": "CONTAINS", "assertion": "World"}} + assert_activity(activity, baseline) + + def test_assert_activity_with_not_contains_assertion(self): + """Test NOT_CONTAINS assertion type.""" + activity = Activity(type="message", text="Hello") + baseline = {"text": {"assertion_type": "NOT_CONTAINS", "assertion": "Goodbye"}} + assert_activity(activity, baseline) + + def test_assert_activity_with_regex_assertion(self): + """Test RE_MATCH assertion type.""" + activity = Activity(type="message", text="msg_20250112_001") + baseline = { + "text": {"assertion_type": "RE_MATCH", "assertion": r"^msg_\d{8}_\d{3}$"} + } + assert_activity(activity, baseline) + + def test_assert_activity_with_multiple_fields_and_mixed_assertions(self): + """Test multiple fields with different assertion types.""" + activity = Activity( + type="message", text="Hello, World!", channel_id="test-channel" + ) + baseline = { + "type": "message", + "text": ["CONTAINS", "Hello"], + "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "prod-channel"}, + } + assert_activity(activity, baseline) + + def test_assert_activity_fails_on_any_field_mismatch(self): + """Test that activity check fails if any field doesn't match.""" + activity = Activity(type="message", text="Hello", channel_id="test-channel") + baseline = {"type": "message", "text": "Hello", "channel_id": "prod-channel"} + assert_activity(activity, baseline) + + def test_assert_activity_with_numeric_fields(self): + """Test with numeric field values.""" + activity = Activity(type="message", locale="en-US") + activity.channel_data = {"timestamp": 1234567890} + baseline = {"type": "message", "channel_data": {"timestamp": 1234567890}} + assert_activity(activity, baseline) + + def test_assert_activity_with_greater_than_assertion(self): + """Test GREATER_THAN assertion on numeric fields.""" + activity = Activity(type="message") + activity.channel_data = {"count": 100} + baseline = { + "channel_data": { + "count": {"assertion_type": "GREATER_THAN", "assertion": 50} + } + } + + # This test depends on how nested dicts are handled + # If channel_data is compared as a whole dict, this might not work as expected + # Keeping this test to illustrate the concept + assert_activity(activity, baseline) + + def test_assert_activity_with_complex_nested_structures(self): + """Test with complex nested structures in baseline.""" + activity = Activity( + type="message", conversation={"id": "conv123", "name": "Test Conversation"} + ) + baseline = { + "type": "message", + "conversation": {"id": "conv123", "name": "Test Conversation"}, + } + assert_activity(activity, baseline) + + def test_assert_activity_with_boolean_fields(self): + """Test with boolean field values.""" + activity = Activity(type="message") + activity.channel_data = {"is_active": True} + baseline = {"channel_data": {"is_active": True}} + assert_activity(activity, baseline) + + def test_assert_activity_type_mismatch(self): + """Test that different activity types don't match.""" + activity = Activity(type="message", text="Hello") + baseline = {"type": "event", "text": "Hello"} + assert_activity(activity, baseline) + + def test_assert_activity_with_list_fields(self): + """Test with list field values.""" + activity = Activity(type="message") + activity.attachments = [Attachment(content_type="text/plain", content="test")] + baseline = { + "type": "message", + "attachments": [{"content_type": "text/plain", "content": "test"}], + } + assert_activity(activity, baseline) + + +class TestAssertActivityRealWorldScenarios: + """Tests simulating real-world usage scenarios.""" + + def test_validate_bot_response_message(self): + """Test validating a typical bot response.""" + activity = Activity( + type="message", + text="I found 3 results for your query.", + from_property={"id": "bot123", "name": "HelpBot"}, + ) + baseline = { + "type": "message", + "text": ["RE_MATCH", r"I found \d+ results"], + "from_property": {"id": "bot123"}, + } + assert_activity(activity, baseline) + + def test_validate_user_message(self): + """Test validating a user message with flexible text matching.""" + activity = Activity( + type="message", + text="help me with something", + from_property={"id": "user456"}, + ) + baseline = { + "type": "message", + "text": {"assertion_type": "CONTAINS", "assertion": "help"}, + } + assert_activity(activity, baseline) + + def test_validate_event_activity(self): + """Test validating an event activity.""" + activity = Activity( + type="event", name="conversationUpdate", value={"action": "add"} + ) + baseline = {"type": "event", "name": "conversationUpdate"} + + assert assert_activity(activity, baseline) is True + + def test_partial_match_allows_extra_fields(self): + """Test that extra fields in activity don't cause failure.""" + activity = Activity( + type="message", + text="Hello", + channel_id="teams", + conversation={"id": "conv123"}, + from_property={"id": "user123"}, + timestamp="2025-01-12T10:00:00Z", + ) + baseline = {"type": "message", "text": "Hello"} + assert_activity(activity, baseline) + + def test_strict_match_with_multiple_fields(self): + """Test strict matching with multiple fields specified.""" + activity = Activity(type="message", text="Hello", channel_id="teams") + baseline = {"type": "message", "text": "Hello", "channel_id": "teams"} + assert_activity(activity, baseline) + + def test_flexible_text_matching_with_regex(self): + """Test flexible text matching using regex patterns.""" + activity = Activity(type="message", text="Order #12345 has been confirmed") + baseline = {"type": "message", "text": ["RE_MATCH", r"Order #\d+ has been"]} + assert_activity(activity, baseline) + + def test_negative_assertions(self): + """Test using negative assertions to ensure fields don't match.""" + activity = Activity(type="message", text="Success", channel_id="teams") + baseline = { + "type": "message", + "text": {"assertion_type": "NOT_CONTAINS", "assertion": "Error"}, + "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "slack"}, + } + assert_activity(activity, baseline) + + def test_combined_positive_and_negative_assertions(self): + """Test combining positive and negative assertions.""" + activity = Activity( + type="message", text="Operation completed successfully", channel_id="teams" + ) + baseline = { + "type": "message", + "text": ["CONTAINS", "completed"], + "channel_id": ["NOT_EQUALS", "slack"], + } + assert_activity(activity, baseline) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py new file mode 100644 index 00000000..0884e69b --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py @@ -0,0 +1,231 @@ +from microsoft_agents.testing.assertions.check_field import check_field +from microsoft_agents.testing.assertions.type_defs import FieldAssertionType + + +class TestCheckFieldEquals: + """Tests for EQUALS assertion type.""" + + def test_equals_with_matching_strings(self): + assert check_field("hello", "hello", FieldAssertionType.EQUALS) is True + + def test_equals_with_non_matching_strings(self): + assert check_field("hello", "world", FieldAssertionType.EQUALS) is False + + def test_equals_with_matching_integers(self): + assert check_field(42, 42, FieldAssertionType.EQUALS) is True + + def test_equals_with_non_matching_integers(self): + assert check_field(42, 43, FieldAssertionType.EQUALS) is False + + def test_equals_with_none_values(self): + assert check_field(None, None, FieldAssertionType.EQUALS) is True + + def test_equals_with_boolean_values(self): + assert check_field(True, True, FieldAssertionType.EQUALS) is True + assert check_field(False, False, FieldAssertionType.EQUALS) is True + assert check_field(True, False, FieldAssertionType.EQUALS) is False + + +class TestCheckFieldNotEquals: + """Tests for NOT_EQUALS assertion type.""" + + def test_not_equals_with_different_strings(self): + assert check_field("hello", "world", FieldAssertionType.NOT_EQUALS) is True + + def test_not_equals_with_matching_strings(self): + assert check_field("hello", "hello", FieldAssertionType.NOT_EQUALS) is False + + def test_not_equals_with_different_integers(self): + assert check_field(42, 43, FieldAssertionType.NOT_EQUALS) is True + + def test_not_equals_with_matching_integers(self): + assert check_field(42, 42, FieldAssertionType.NOT_EQUALS) is False + + +class TestCheckFieldGreaterThan: + """Tests for GREATER_THAN assertion type.""" + + def test_greater_than_with_larger_value(self): + assert check_field(10, 5, FieldAssertionType.GREATER_THAN) is True + + def test_greater_than_with_smaller_value(self): + assert check_field(5, 10, FieldAssertionType.GREATER_THAN) is False + + def test_greater_than_with_equal_value(self): + assert check_field(10, 10, FieldAssertionType.GREATER_THAN) is False + + def test_greater_than_with_floats(self): + assert check_field(10.5, 10.2, FieldAssertionType.GREATER_THAN) is True + assert check_field(10.2, 10.5, FieldAssertionType.GREATER_THAN) is False + + def test_greater_than_with_negative_numbers(self): + assert check_field(-5, -10, FieldAssertionType.GREATER_THAN) is True + assert check_field(-10, -5, FieldAssertionType.GREATER_THAN) is False + + +class TestCheckFieldLessThan: + """Tests for LESS_THAN assertion type.""" + + def test_less_than_with_smaller_value(self): + assert check_field(5, 10, FieldAssertionType.LESS_THAN) is True + + def test_less_than_with_larger_value(self): + assert check_field(10, 5, FieldAssertionType.LESS_THAN) is False + + def test_less_than_with_equal_value(self): + assert check_field(10, 10, FieldAssertionType.LESS_THAN) is False + + def test_less_than_with_floats(self): + assert check_field(10.2, 10.5, FieldAssertionType.LESS_THAN) is True + assert check_field(10.5, 10.2, FieldAssertionType.LESS_THAN) is False + + +class TestCheckFieldContains: + """Tests for CONTAINS assertion type.""" + + def test_contains_substring_in_string(self): + assert check_field("hello world", "world", FieldAssertionType.CONTAINS) is True + + def test_contains_substring_not_in_string(self): + assert check_field("hello world", "foo", FieldAssertionType.CONTAINS) is False + + def test_contains_element_in_list(self): + assert check_field([1, 2, 3, 4], 3, FieldAssertionType.CONTAINS) is True + + def test_contains_element_not_in_list(self): + assert check_field([1, 2, 3, 4], 5, FieldAssertionType.CONTAINS) is False + + def test_contains_key_in_dict(self): + assert check_field({"a": 1, "b": 2}, "a", FieldAssertionType.CONTAINS) is True + + def test_contains_key_not_in_dict(self): + assert check_field({"a": 1, "b": 2}, "c", FieldAssertionType.CONTAINS) is False + + def test_contains_empty_string(self): + assert check_field("hello", "", FieldAssertionType.CONTAINS) is True + + +class TestCheckFieldNotContains: + """Tests for NOT_CONTAINS assertion type.""" + + def test_not_contains_substring_not_in_string(self): + assert ( + check_field("hello world", "foo", FieldAssertionType.NOT_CONTAINS) is True + ) + + def test_not_contains_substring_in_string(self): + assert ( + check_field("hello world", "world", FieldAssertionType.NOT_CONTAINS) + is False + ) + + def test_not_contains_element_not_in_list(self): + assert check_field([1, 2, 3, 4], 5, FieldAssertionType.NOT_CONTAINS) is True + + def test_not_contains_element_in_list(self): + assert check_field([1, 2, 3, 4], 3, FieldAssertionType.NOT_CONTAINS) is False + + +class TestCheckFieldReMatch: + """Tests for RE_MATCH assertion type.""" + + def test_re_match_simple_pattern(self): + assert check_field("hello123", r"hello\d+", FieldAssertionType.RE_MATCH) is True + + def test_re_match_no_match(self): + assert check_field("hello", r"\d+", FieldAssertionType.RE_MATCH) is False + + def test_re_match_email_pattern(self): + pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + assert ( + check_field("test@example.com", pattern, FieldAssertionType.RE_MATCH) + is True + ) + assert ( + check_field("invalid-email", pattern, FieldAssertionType.RE_MATCH) is False + ) + + def test_re_match_anchored_pattern(self): + assert ( + check_field("hello world", r"^hello", FieldAssertionType.RE_MATCH) is True + ) + assert ( + check_field("hello world", r"^world", FieldAssertionType.RE_MATCH) is False + ) + + def test_re_match_full_string(self): + assert check_field("abc", r"^abc$", FieldAssertionType.RE_MATCH) is True + assert check_field("abcd", r"^abc$", FieldAssertionType.RE_MATCH) is False + + def test_re_match_case_sensitive(self): + assert check_field("Hello", r"hello", FieldAssertionType.RE_MATCH) is False + assert check_field("Hello", r"Hello", FieldAssertionType.RE_MATCH) is True + + +class TestCheckFieldEdgeCases: + """Tests for edge cases and error handling.""" + + def test_invalid_assertion_type(self): + # Passing an unsupported assertion type should return False + assert check_field("test", "test", "INVALID_TYPE") is False + + def test_none_actual_value_with_equals(self): + assert check_field(None, "test", FieldAssertionType.EQUALS) is False + assert check_field(None, None, FieldAssertionType.EQUALS) is True + + def test_empty_string_comparisons(self): + assert check_field("", "", FieldAssertionType.EQUALS) is True + assert check_field("", "test", FieldAssertionType.EQUALS) is False + + def test_empty_list_contains(self): + assert check_field([], "item", FieldAssertionType.CONTAINS) is False + + def test_zero_comparisons(self): + assert check_field(0, 0, FieldAssertionType.EQUALS) is True + assert check_field(0, 1, FieldAssertionType.LESS_THAN) is True + assert check_field(0, -1, FieldAssertionType.GREATER_THAN) is True + + def test_type_mismatch_comparisons(self): + # Different types should work with equality checks + assert check_field("42", 42, FieldAssertionType.EQUALS) is False + assert check_field("42", 42, FieldAssertionType.NOT_EQUALS) is True + + def test_complex_data_structures(self): + actual = {"nested": {"value": 123}} + expected = {"nested": {"value": 123}} + assert check_field(actual, expected, FieldAssertionType.EQUALS) is True + + def test_list_equality(self): + assert check_field([1, 2, 3], [1, 2, 3], FieldAssertionType.EQUALS) is True + assert check_field([1, 2, 3], [3, 2, 1], FieldAssertionType.EQUALS) is False + + +class TestCheckFieldWithRealWorldScenarios: + """Tests simulating real-world usage scenarios.""" + + def test_validate_response_status_code(self): + assert check_field(200, 200, FieldAssertionType.EQUALS) is True + assert check_field(404, 200, FieldAssertionType.NOT_EQUALS) is True + + def test_validate_response_contains_keyword(self): + response = "Success: Operation completed successfully" + assert check_field(response, "Success", FieldAssertionType.CONTAINS) is True + assert check_field(response, "Error", FieldAssertionType.NOT_CONTAINS) is True + + def test_validate_numeric_threshold(self): + temperature = 72.5 + assert check_field(temperature, 100, FieldAssertionType.LESS_THAN) is True + assert check_field(temperature, 0, FieldAssertionType.GREATER_THAN) is True + + def test_validate_message_format(self): + message_id = "msg_20250112_001" + pattern = r"^msg_\d{8}_\d{3}$" + assert check_field(message_id, pattern, FieldAssertionType.RE_MATCH) is True + + def test_validate_list_membership(self): + allowed_roles = ["admin", "user", "guest"] + assert check_field(allowed_roles, "admin", FieldAssertionType.CONTAINS) is True + assert ( + check_field(allowed_roles, "superuser", FieldAssertionType.NOT_CONTAINS) + is True + ) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_select_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_select_activity.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/integration/test_data_driven_tester.py b/dev/microsoft-agents-testing/tests/integration/test_data_driven_tester.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/activity/test_sub_channels.py b/tests/activity/test_sub_channels.py deleted file mode 100644 index 67c6d92c..00000000 --- a/tests/activity/test_sub_channels.py +++ /dev/null @@ -1,5 +0,0 @@ -from microsoft_agents.activity import Activity, ChannelId, Entity - - -class TestSubChannels: - pass