From cfb88476789368b0ddab057603e11e913b6077ab Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Tue, 21 Oct 2025 14:16:40 -0700 Subject: [PATCH 01/81] basic integration utility --- dev/integration/env.TEMPLATE | 3 ++ dev/integration/requirements.txt | 1 + dev/integration/src/__init__.py | 0 dev/integration/src/agent_client.py | 51 +++++++++++++++++++++ dev/integration/src/bot_client.py | 47 +++++++++++++++++++ dev/integration/src/bot_response.py | 14 ++++++ dev/integration/src/integration/__init__.py | 0 7 files changed, 116 insertions(+) create mode 100644 dev/integration/env.TEMPLATE create mode 100644 dev/integration/requirements.txt create mode 100644 dev/integration/src/__init__.py create mode 100644 dev/integration/src/agent_client.py create mode 100644 dev/integration/src/bot_client.py create mode 100644 dev/integration/src/bot_response.py create mode 100644 dev/integration/src/integration/__init__.py diff --git a/dev/integration/env.TEMPLATE b/dev/integration/env.TEMPLATE new file mode 100644 index 00000000..ca2114c8 --- /dev/null +++ b/dev/integration/env.TEMPLATE @@ -0,0 +1,3 @@ +aioresponses +microsoft-agents-activity +microsoft-agents-hosting-core \ No newline at end of file diff --git a/dev/integration/requirements.txt b/dev/integration/requirements.txt new file mode 100644 index 00000000..d524e63a --- /dev/null +++ b/dev/integration/requirements.txt @@ -0,0 +1 @@ +aioresponses \ No newline at end of file diff --git a/dev/integration/src/__init__.py b/dev/integration/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/agent_client.py b/dev/integration/src/agent_client.py new file mode 100644 index 00000000..5d73852a --- /dev/null +++ b/dev/integration/src/agent_client.py @@ -0,0 +1,51 @@ +from http import client +import os + +from aiohttp import ClientSession + +from microsoft_agents.activity import ( + Activity, + ActivityTypes, +) + +from msal import ConfidentialClientApplication + +class AgentClient: + + def __init__(self, messaging_endpoint: str, service_endpoint: str, cid: str, client_id: str, tenant_id: str, client_secret: str): + self.messaging_endpoint = messaging_endpoint + self.service_endpoint = service_endpoint + self.cid = cid + self.client_id = client_id + self.tenant_id = tenant_id + self.client_secret = client_secret + + async def send_request(self, activity: Activity): + + client_id = os.environ["CLIENT_ID"] + + msal_app = ConfidentialClientApplication( + client_id=client_id, + tenant_id=os.environ["TENANT_ID"], + client_credential=os.environ["CLIENT_SECRET"], + ) + + token = msal_app.acquire_token_for_client([f"{client_id}/.default"]) + + session = ClientSession() + activity.service_url = self.service_endpoint + activ + + async def send_activity(self, activity: Activity): + pass + + async def send_expect_replies_activity(self, activity: Activity): + pass + + async def send_stream_activity(self, activity: Activity): + pass + + async def send_invoke(self, activity: Activity): + if activity.type != ActivityTypes.invoke: + raise ValueError("Activity type must be 'invoke' for send_invoke method.") + return await self.send_request(activity) \ No newline at end of file diff --git a/dev/integration/src/bot_client.py b/dev/integration/src/bot_client.py new file mode 100644 index 00000000..2a547581 --- /dev/null +++ b/dev/integration/src/bot_client.py @@ -0,0 +1,47 @@ +from http import client +import os + +from microsoft_agents.activity import ( + Activity, + ActivityTypes, +) + +from msal import ConfidentialClientApplication + +class BotClient: + + def __init__(self, messaging_endpoint: str, service_endpoint: str, cid: str, client_id: str, tenant_id: str, client_secret: str): + self.messaging_endpoint = messaging_endpoint + self.service_endpoint = service_endpoint + self.cid = cid + self.client_id = client_id + self.tenant_id = tenant_id + self.client_secret = client_secret + + async def send_request(self, activity: Activity): + + client_id = os.environ["CLIENT_ID"] + + msal_app = ConfidentialClientApplication( + client_id=client_id, + tenant_id=os.environ["TENANT_ID"], + client_credential=os.environ["CLIENT_SECRET"], + ) + + token = msal_app.acquire_token_for_client([f"{client_id}/.default"]) + + http_client = + + async def send_activity(self, activity: Activity): + pass + + async def send_expect_replies_activity(self, activity: Activity): + pass + + async def send_stream_activity(self, activity: Activity): + pass + + async def send_invoke(self, activity: Activity): + if activity.type != ActivityTypes.invoke: + raise ValueError("Activity type must be 'invoke' for send_invoke method.") + return await self.send_request(activity) \ No newline at end of file diff --git a/dev/integration/src/bot_response.py b/dev/integration/src/bot_response.py new file mode 100644 index 00000000..07fb9282 --- /dev/null +++ b/dev/integration/src/bot_response.py @@ -0,0 +1,14 @@ +class BotResponse: + + def __init__(self): + pass + + def handle_streamed_activity(self, activity: Activity, sact: StreamInfo, cid: str) -> bool: + pass + + def dispose_async(self) -> ValueTask: + pass + + @property + def service_endpoint(self) -> str: + pass \ No newline at end of file diff --git a/dev/integration/src/integration/__init__.py b/dev/integration/src/integration/__init__.py new file mode 100644 index 00000000..e69de29b From 4d93a461c7c75afe386005314d0fc3ab671f6624 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 23 Oct 2025 11:59:05 -0700 Subject: [PATCH 02/81] Integration test suite factory implementation --- .../src/integration/integration_test.py | 40 +++++++++++++++++++ dev/integration/src/samples/__init__.py | 0 .../src/samples/quickstart_sample.py | 10 +++++ dev/integration/src/samples/sample.py | 17 ++++++++ 4 files changed, 67 insertions(+) create mode 100644 dev/integration/src/integration/integration_test.py create mode 100644 dev/integration/src/samples/__init__.py create mode 100644 dev/integration/src/samples/quickstart_sample.py create mode 100644 dev/integration/src/samples/sample.py diff --git a/dev/integration/src/integration/integration_test.py b/dev/integration/src/integration/integration_test.py new file mode 100644 index 00000000..1459538d --- /dev/null +++ b/dev/integration/src/integration/integration_test.py @@ -0,0 +1,40 @@ +from abc import ABC, abstractmethod +from typing import Callable, Awaitable, Generic, Protocol + +import pytest + +from microsoft_agents.hosting.core import ( + AgentApplication, + ChannelAdapter, + Connections +) + +from ..samples import Sample + +class SampleFactory(Protocol): + + def __call__(self, *args, **kwargs) -> Awaitable[Sample]: + ... + +def integration_suite_factory( + sample_factory: Callable[[...], Awaitable[Sample]] +): + class IntegrationTestSuite: + sample: Sample + + def setup_method(self, mocker): + self.sample = sample_factory(mocker=mocker) + + @pytest.fixture + def agent_application(self) -> AgentApplication: + return self.sample.agent_application + + @pytest.fixture + def adapter(self) -> ChannelAdapter: + return self.sample.adapter + + @pytest.fixture + def connections(self) -> Connections: + return self.sample.connections + + return IntegrationTestSuite \ No newline at end of file diff --git a/dev/integration/src/samples/__init__.py b/dev/integration/src/samples/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/samples/quickstart_sample.py b/dev/integration/src/samples/quickstart_sample.py new file mode 100644 index 00000000..bf4fa658 --- /dev/null +++ b/dev/integration/src/samples/quickstart_sample.py @@ -0,0 +1,10 @@ +from .sample import Sample + + +def create_quickstart() -> Sample: + + AgentApplication + + sample = Sample( + + ) \ No newline at end of file diff --git a/dev/integration/src/samples/sample.py b/dev/integration/src/samples/sample.py new file mode 100644 index 00000000..929cc34a --- /dev/null +++ b/dev/integration/src/samples/sample.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass + +from microsoft_agents.hosting.core import ( + AgentApplication, + ChannelAdapter, + Connections +) + +@dataclass +class Sample: + """A sample data object for integration tests.""" + + agent_application: AgentApplication + adapter: ChannelAdapter + connections: Connections + + config: dict \ No newline at end of file From be2fb1cebd7d8a5c6494451b5746c2b73ffeccc4 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 23 Oct 2025 13:59:48 -0700 Subject: [PATCH 03/81] Implementing core models for integration testing --- dev/integration/src/core/__init__.py | 13 +++++ .../src/{ => core}/agent_client.py | 0 dev/integration/src/{ => core}/bot_client.py | 0 .../src/{ => core}/bot_response.py | 0 .../src/core/environment/__init__.py | 7 +++ .../core/environment/create_aiohttp_env.py | 41 +++++++++++++++ .../environment/environment.py} | 11 ++-- .../core/integration_test_suite_factory.py | 52 +++++++++++++++++++ dev/integration/src/core/sample.py | 14 +++++ .../src/integration/integration_test.py | 40 -------------- dev/integration/src/samples/__init__.py | 5 ++ .../src/samples/quickstart_sample.py | 50 ++++++++++++++++-- .../src/{integration => tests}/__init__.py | 0 dev/integration/src/tests/test_quickstart.py | 12 +++++ 14 files changed, 197 insertions(+), 48 deletions(-) create mode 100644 dev/integration/src/core/__init__.py rename dev/integration/src/{ => core}/agent_client.py (100%) rename dev/integration/src/{ => core}/bot_client.py (100%) rename dev/integration/src/{ => core}/bot_response.py (100%) create mode 100644 dev/integration/src/core/environment/__init__.py create mode 100644 dev/integration/src/core/environment/create_aiohttp_env.py rename dev/integration/src/{samples/sample.py => core/environment/environment.py} (58%) create mode 100644 dev/integration/src/core/integration_test_suite_factory.py create mode 100644 dev/integration/src/core/sample.py delete mode 100644 dev/integration/src/integration/integration_test.py rename dev/integration/src/{integration => tests}/__init__.py (100%) create mode 100644 dev/integration/src/tests/test_quickstart.py diff --git a/dev/integration/src/core/__init__.py b/dev/integration/src/core/__init__.py new file mode 100644 index 00000000..df7ed39e --- /dev/null +++ b/dev/integration/src/core/__init__.py @@ -0,0 +1,13 @@ +from .environment import ( + Environment, + create_aiohttp_env +) +from .integration_test_suite_factory import integration_test_suite_factory +from .sample import Sample + +__all__ = [ + "Environment", + "create_aiohttp_env", + "integration_test_suite_factory", + "Sample", +] \ No newline at end of file diff --git a/dev/integration/src/agent_client.py b/dev/integration/src/core/agent_client.py similarity index 100% rename from dev/integration/src/agent_client.py rename to dev/integration/src/core/agent_client.py diff --git a/dev/integration/src/bot_client.py b/dev/integration/src/core/bot_client.py similarity index 100% rename from dev/integration/src/bot_client.py rename to dev/integration/src/core/bot_client.py diff --git a/dev/integration/src/bot_response.py b/dev/integration/src/core/bot_response.py similarity index 100% rename from dev/integration/src/bot_response.py rename to dev/integration/src/core/bot_response.py diff --git a/dev/integration/src/core/environment/__init__.py b/dev/integration/src/core/environment/__init__.py new file mode 100644 index 00000000..b2deb639 --- /dev/null +++ b/dev/integration/src/core/environment/__init__.py @@ -0,0 +1,7 @@ +from .create_aiohttp_env import create_aiohttp_env +from .environment import Environment + +__all__ = [ + "create_aiohttp_env", + "Environment" +] \ No newline at end of file diff --git a/dev/integration/src/core/environment/create_aiohttp_env.py b/dev/integration/src/core/environment/create_aiohttp_env.py new file mode 100644 index 00000000..4d463c0b --- /dev/null +++ b/dev/integration/src/core/environment/create_aiohttp_env.py @@ -0,0 +1,41 @@ +from typing import Optional + +from microsoft_agents.hosting.aiohttp import CloudAdapter +from microsoft_agents.hosting.core import ( + Authorization, + AgentApplication, + TurnState, + TurnContext, + MemoryStorage, +) +from microsoft_agents.authentication.msal import MsalConnectionManager +from microsoft_agents.activity import load_configuration_from_env + +from .environment import Environment + +def create_aiohttp_env(environ_dict: Optional[dict] = None) -> Environment: + + environ_dict = environ_dict or {} + + agents_sdk_config = load_configuration_from_env(environ_dict) + + storage = MemoryStorage() + connection_manager = MsalConnectionManager(**agents_sdk_config) + adapter = CloudAdapter(connection_manager=connection_manager) + authorization = Authorization(storage, connection_manager, **agents_sdk_config) + + agent_application = AgentApplication[TurnState]( + storage=storage, + adapter=adapter, + authorization=authorization, + **agents_sdk_config + ) + + return Environment( + agent_application=agent_application, + storage=storage, + connections=connection_manager, + adapter=adapter, + authorization=authorization, + config=agents_sdk_config + ) \ No newline at end of file diff --git a/dev/integration/src/samples/sample.py b/dev/integration/src/core/environment/environment.py similarity index 58% rename from dev/integration/src/samples/sample.py rename to dev/integration/src/core/environment/environment.py index 929cc34a..d428268d 100644 --- a/dev/integration/src/samples/sample.py +++ b/dev/integration/src/core/environment/environment.py @@ -3,15 +3,20 @@ from microsoft_agents.hosting.core import ( AgentApplication, ChannelAdapter, - Connections + Connections, + Authorization, + Storage, + TurnState, ) @dataclass -class Sample: +class Environment: """A sample data object for integration tests.""" - agent_application: AgentApplication + agent_application: AgentApplication[TurnState] + storage: Storage adapter: ChannelAdapter connections: Connections + authorization: Authorization config: dict \ No newline at end of file diff --git a/dev/integration/src/core/integration_test_suite_factory.py b/dev/integration/src/core/integration_test_suite_factory.py new file mode 100644 index 00000000..ae79575b --- /dev/null +++ b/dev/integration/src/core/integration_test_suite_factory.py @@ -0,0 +1,52 @@ +from typing import Optional + +import pytest + +from microsoft_agents.hosting.core import ( + AgentApplication, + ChannelAdapter, + Connections +) + +from .environment import Environment, create_aiohttp_env +from .sample import Sample + +def integration_test_suite_factory( + sample_cls: type[Sample], + environment: Optional[Environment] = None +): + """Factory function to create an integration test suite for a given sample application. + + :param environment: The environment to use for the sample application. + :param sample_cls: The sample class to create the test suite for. + :return: An integration test suite class. + """ + + if not environment: + environment = create_aiohttp_env() + + class IntegrationTestSuite: + """Integration test suite for a given sample application.""" + sample: Sample + + async def setup_method(self, mocker): + """Set up the test suite with the sample application.""" + self.sample = sample_cls(environment, mocker=mocker) + await self.sample.init_app() + + @pytest.fixture + def agent_application(self) -> AgentApplication: + """Get the agent application for the test suite.""" + return self.sample.env.agent_application + + @pytest.fixture + def adapter(self) -> ChannelAdapter: + """Get the channel adapter for the test suite.""" + return self.sample.env.adapter + + @pytest.fixture + def connections(self) -> Connections: + """Get the connections for the test suite.""" + return self.sample.env.connections + + return IntegrationTestSuite \ No newline at end of file diff --git a/dev/integration/src/core/sample.py b/dev/integration/src/core/sample.py new file mode 100644 index 00000000..524f1247 --- /dev/null +++ b/dev/integration/src/core/sample.py @@ -0,0 +1,14 @@ +from abc import ABC, abstractmethod + +from .environment import Environment + + +class Sample(ABC): + """Base class for all samples.""" + + def __init__(self, environment: Environment, **kwargs): + self.env = environment + + @abstractmethod + async def init_app(self): + """Initialize the application for the sample.""" \ No newline at end of file diff --git a/dev/integration/src/integration/integration_test.py b/dev/integration/src/integration/integration_test.py deleted file mode 100644 index 1459538d..00000000 --- a/dev/integration/src/integration/integration_test.py +++ /dev/null @@ -1,40 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Callable, Awaitable, Generic, Protocol - -import pytest - -from microsoft_agents.hosting.core import ( - AgentApplication, - ChannelAdapter, - Connections -) - -from ..samples import Sample - -class SampleFactory(Protocol): - - def __call__(self, *args, **kwargs) -> Awaitable[Sample]: - ... - -def integration_suite_factory( - sample_factory: Callable[[...], Awaitable[Sample]] -): - class IntegrationTestSuite: - sample: Sample - - def setup_method(self, mocker): - self.sample = sample_factory(mocker=mocker) - - @pytest.fixture - def agent_application(self) -> AgentApplication: - return self.sample.agent_application - - @pytest.fixture - def adapter(self) -> ChannelAdapter: - return self.sample.adapter - - @pytest.fixture - def connections(self) -> Connections: - return self.sample.connections - - return IntegrationTestSuite \ No newline at end of file diff --git a/dev/integration/src/samples/__init__.py b/dev/integration/src/samples/__init__.py index e69de29b..4963a4c7 100644 --- a/dev/integration/src/samples/__init__.py +++ b/dev/integration/src/samples/__init__.py @@ -0,0 +1,5 @@ +from .quickstart_sample import QuickstartSample + +__all__ = [ + "QuickstartSample" +] \ No newline at end of file diff --git a/dev/integration/src/samples/quickstart_sample.py b/dev/integration/src/samples/quickstart_sample.py index bf4fa658..05a5ebc9 100644 --- a/dev/integration/src/samples/quickstart_sample.py +++ b/dev/integration/src/samples/quickstart_sample.py @@ -1,10 +1,50 @@ -from .sample import Sample +import re +import sys +import traceback +from microsoft_agents.activity import ConversationUpdateTypes -def create_quickstart() -> Sample: +from microsoft_agents.hosting.core import ( + AgentApplication, + TurnContext, + TurnState +) - AgentApplication +from ..core.sample import Sample + +class QuickstartSample(Sample): + """A quickstart sample implementation.""" - sample = Sample( + async def init_app(self): + """Initialize the application for the quickstart sample.""" - ) \ No newline at end of file + app: AgentApplication[TurnState] = self.env.agent_application + + @app.conversation_update(ConversationUpdateTypes.MEMBERS_ADDED) + async def on_members_added(context: TurnContext, state: TurnState) -> None: + await context.send_activity( + "Welcome to the empty agent! " + "This agent is designed to be a starting point for your own agent development." + ) + + + @app.message(re.compile(r"^hello$")) + async def on_hello(context: TurnContext, state: TurnState) -> None: + await context.send_activity("Hello!") + + + @app.activity("message") + async def on_message(context: TurnContext, state: TurnState) -> None: + await context.send_activity(f"you said: {context.activity.text}") + + + @app.error + async def on_error(context: TurnContext, error: Exception): + # This check writes out errors to console log .vs. app insights. + # NOTE: In production environment, you should consider logging this to Azure + # application insights. + print(f"\n [on_turn_error] unhandled error: {error}", file=sys.stderr) + traceback.print_exc() + + # Send a message to the user + await context.send_activity("The bot encountered an error or bug.") diff --git a/dev/integration/src/integration/__init__.py b/dev/integration/src/tests/__init__.py similarity index 100% rename from dev/integration/src/integration/__init__.py rename to dev/integration/src/tests/__init__.py diff --git a/dev/integration/src/tests/test_quickstart.py b/dev/integration/src/tests/test_quickstart.py new file mode 100644 index 00000000..2dce2dd3 --- /dev/null +++ b/dev/integration/src/tests/test_quickstart.py @@ -0,0 +1,12 @@ +import pytest + +from ..core import integration_test_suite_factory +from ..samples import QuickstartSample + +TestSuiteBase = integration_test_suite_factory(QuickstartSample) + +class TestQuickstart(TestSuiteBase): + + @pytest.mark.asyncio + async def test_quickstart_functionality(self): + pass \ No newline at end of file From 0d5539a885ed511ed9a93eeb898adcfdaaa49af2 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 23 Oct 2025 14:10:29 -0700 Subject: [PATCH 04/81] AutoClient mockup --- .../src/core/auto_client/__init__.py | 0 .../src/core/auto_client/auto_client.py | 18 ++++++++++++++++++ .../src/core/environment/create_aiohttp_env.py | 2 ++ dev/integration/src/core/environment/driver.py | 4 ++++ .../src/core/environment/environment.py | 5 ++++- 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 dev/integration/src/core/auto_client/__init__.py create mode 100644 dev/integration/src/core/auto_client/auto_client.py create mode 100644 dev/integration/src/core/environment/driver.py diff --git a/dev/integration/src/core/auto_client/__init__.py b/dev/integration/src/core/auto_client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/core/auto_client/auto_client.py b/dev/integration/src/core/auto_client/auto_client.py new file mode 100644 index 00000000..8e423867 --- /dev/null +++ b/dev/integration/src/core/auto_client/auto_client.py @@ -0,0 +1,18 @@ +from microsoft_agents.activity import Activity + +from ..agent_client import AgentClient + +class AutoClient: + + def __init__(self, agent_client: AgentClient): + self._agent_client = agent_client + + async def generate_message(self) -> str: + pass + + async def run(self, max_turns: int = 10, time_between_turns: float = 2.0) -> None: + + for i in range(max_turns): + await self._agent_client.send_activity( + Activity(type="message", text=self.generate_message()) + ) \ No newline at end of file diff --git a/dev/integration/src/core/environment/create_aiohttp_env.py b/dev/integration/src/core/environment/create_aiohttp_env.py index 4d463c0b..e16e71fa 100644 --- a/dev/integration/src/core/environment/create_aiohttp_env.py +++ b/dev/integration/src/core/environment/create_aiohttp_env.py @@ -1,5 +1,6 @@ from typing import Optional +from click import Option from microsoft_agents.hosting.aiohttp import CloudAdapter from microsoft_agents.hosting.core import ( Authorization, @@ -13,6 +14,7 @@ from .environment import Environment + def create_aiohttp_env(environ_dict: Optional[dict] = None) -> Environment: environ_dict = environ_dict or {} diff --git a/dev/integration/src/core/environment/driver.py b/dev/integration/src/core/environment/driver.py new file mode 100644 index 00000000..9d421530 --- /dev/null +++ b/dev/integration/src/core/environment/driver.py @@ -0,0 +1,4 @@ +import threading + +async def aiohttp_driver() -> None: + diff --git a/dev/integration/src/core/environment/environment.py b/dev/integration/src/core/environment/environment.py index d428268d..1b06215e 100644 --- a/dev/integration/src/core/environment/environment.py +++ b/dev/integration/src/core/environment/environment.py @@ -1,3 +1,4 @@ +from typing import Awaitable, Callable from dataclasses import dataclass from microsoft_agents.hosting.core import ( @@ -19,4 +20,6 @@ class Environment: connections: Connections authorization: Authorization - config: dict \ No newline at end of file + config: dict + + driver: Callable[[], Awaitable[None]] \ No newline at end of file From bf4c00676f9e72b94bcd2f19f4fe9152492231c3 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 23 Oct 2025 14:18:07 -0700 Subject: [PATCH 05/81] Adding runner starter code --- .../src/core/environment/create_aiohttp_env.py | 15 +++++++++++++++ .../src/core/integration_test_suite_factory.py | 5 +++++ dev/integration/src/core/runner/__init__.py | 0 dev/integration/src/core/runner/driver.py | 0 4 files changed, 20 insertions(+) create mode 100644 dev/integration/src/core/runner/__init__.py create mode 100644 dev/integration/src/core/runner/driver.py diff --git a/dev/integration/src/core/environment/create_aiohttp_env.py b/dev/integration/src/core/environment/create_aiohttp_env.py index e16e71fa..429a1e05 100644 --- a/dev/integration/src/core/environment/create_aiohttp_env.py +++ b/dev/integration/src/core/environment/create_aiohttp_env.py @@ -14,6 +14,21 @@ from .environment import Environment +def start_server() -> None: + import asyncio + from threading import Thread + from contextlib import asynccontextmanager + from microsoft_agents.hosting.aiohttp import host_app + +@asynccontextmanager +def aiohttp_runner(timeout=10.0) -> None: + + thread = Thread(target=start_server) + thread.start() + + yield + + thread.join(timeout=timeout) def create_aiohttp_env(environ_dict: Optional[dict] = None) -> Environment: diff --git a/dev/integration/src/core/integration_test_suite_factory.py b/dev/integration/src/core/integration_test_suite_factory.py index ae79575b..24eec943 100644 --- a/dev/integration/src/core/integration_test_suite_factory.py +++ b/dev/integration/src/core/integration_test_suite_factory.py @@ -33,6 +33,11 @@ async def setup_method(self, mocker): """Set up the test suite with the sample application.""" self.sample = sample_cls(environment, mocker=mocker) await self.sample.init_app() + await self.sample.runner.start() + + async def teardown_method(self): + """Tear down the test suite and clean up resources.""" + await self.sample.runner.stop() @pytest.fixture def agent_application(self) -> AgentApplication: diff --git a/dev/integration/src/core/runner/__init__.py b/dev/integration/src/core/runner/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/core/runner/driver.py b/dev/integration/src/core/runner/driver.py new file mode 100644 index 00000000..e69de29b From ebe622df7c5d754d27a0cd94362c64ecb40e1470 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Sat, 25 Oct 2025 09:30:06 -0700 Subject: [PATCH 06/81] Adding foundational classes --- dev/integration/src/client/__init__.py | 0 dev/integration/src/client/agent_client.py | 0 .../src/client/listening_client.py | 4 + dev/integration/src/core/client/__init__.py | 9 ++ .../src/core/client/agent_client.py | 4 + .../src/core/client/agent_client_base.py | 4 + .../src/core/client/ai_agent_client.py | 4 + dev/integration/src/core/integration.py | 140 ++++++++++++++++++ .../src/core/integration_fixtures.py | 36 +++++ .../src/core/response_server/__init__.py | 3 + .../core/response_server/response_server.py | 11 ++ dev/integration/src/core/runner/__init__.py | 3 + dev/integration/src/core/runner/app_runner.py | 2 + dev/integration/src/core/sample.py | 5 + 14 files changed, 225 insertions(+) create mode 100644 dev/integration/src/client/__init__.py create mode 100644 dev/integration/src/client/agent_client.py create mode 100644 dev/integration/src/client/listening_client.py create mode 100644 dev/integration/src/core/client/__init__.py create mode 100644 dev/integration/src/core/client/agent_client.py create mode 100644 dev/integration/src/core/client/agent_client_base.py create mode 100644 dev/integration/src/core/client/ai_agent_client.py create mode 100644 dev/integration/src/core/integration.py create mode 100644 dev/integration/src/core/integration_fixtures.py create mode 100644 dev/integration/src/core/response_server/__init__.py create mode 100644 dev/integration/src/core/response_server/response_server.py create mode 100644 dev/integration/src/core/runner/app_runner.py diff --git a/dev/integration/src/client/__init__.py b/dev/integration/src/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/client/agent_client.py b/dev/integration/src/client/agent_client.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/client/listening_client.py b/dev/integration/src/client/listening_client.py new file mode 100644 index 00000000..ad71188b --- /dev/null +++ b/dev/integration/src/client/listening_client.py @@ -0,0 +1,4 @@ +from .agent_client import AgentClient + +class ListeningClient(AgentClient): + pass \ No newline at end of file diff --git a/dev/integration/src/core/client/__init__.py b/dev/integration/src/core/client/__init__.py new file mode 100644 index 00000000..5f5993f7 --- /dev/null +++ b/dev/integration/src/core/client/__init__.py @@ -0,0 +1,9 @@ +from .agent_client import AgentClient +from .ai_agent_client import AIAgentClient +from .agent_client_base import BaseClient + +__all__ = [ + "AgentClient", + "AIAgentClient", + "BaseClient", +] \ No newline at end of file diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py new file mode 100644 index 00000000..1ecdb488 --- /dev/null +++ b/dev/integration/src/core/client/agent_client.py @@ -0,0 +1,4 @@ +from .agent_client_base import AgentClientBase + +class AgentClient(AgentClientBase): + pass \ No newline at end of file diff --git a/dev/integration/src/core/client/agent_client_base.py b/dev/integration/src/core/client/agent_client_base.py new file mode 100644 index 00000000..62bd85ed --- /dev/null +++ b/dev/integration/src/core/client/agent_client_base.py @@ -0,0 +1,4 @@ +from abc import ABC + +class AgentClientBase(ABC): + pass \ No newline at end of file diff --git a/dev/integration/src/core/client/ai_agent_client.py b/dev/integration/src/core/client/ai_agent_client.py new file mode 100644 index 00000000..072fe004 --- /dev/null +++ b/dev/integration/src/core/client/ai_agent_client.py @@ -0,0 +1,4 @@ +from .agent_client import AgentClient + +class AIAgentClient(AgentClient): + pass \ No newline at end of file diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py new file mode 100644 index 00000000..4fb56670 --- /dev/null +++ b/dev/integration/src/core/integration.py @@ -0,0 +1,140 @@ +from random import sample +import pytest + +from typing import Optional, TypeVar, Union + +import aiohttp.web + +from .runner import AppRunner +from .environment import Environment +from .response_server import ResponseServer +from .sample import Sample + +T = TypeVar("T", bound=type) +AppT = TypeVar("AppT", bound=aiohttp.web.Application) # for future extension w/ Union + +async def start_response_server(): + pass + +class Integration: + + @staticmethod + def _with_response_server(target_cls: T, host_response: bool) -> T: + """Wraps the target class to include a response server if needed.""" + + _prev_setup_method = getattr(target_cls, "setup_method", None) + _prev_teardown_method = getattr(target_cls, "teardown_method", None) + + async def setup_method(self): + if host_response: + self._response_server = ResponseServer() + await self._response_server.__aenter__() + if _prev_setup_method: + await _prev_setup_method(self) + + async def teardown_method(self): + if host_response: + await self._response_server.__aexit__(None, None, None) + if _prev_teardown_method: + await _prev_teardown_method(self) + + target_cls.setup_method = setup_method + target_cls.teardown_method = teardown_method + + return target_cls + + @staticmethod + def from_service_url(target_cls: T, service_url: str, host_response: bool = False) -> T: + """Creates an Integration instance using a service URL.""" + + async def setup_method(self): + self._service_url = service_url + + async def teardown_method(self): + self._service_url = service_url + + target_cls.setup_method = setup_method + target_cls.teardown_method = teardown_method + + target_cls = Integration._with_response_server(target_cls, host_response) + + return target_cls + + @staticmethod + def from_sample( + target_cls: T, + sample_cls: type[Sample], + environment_cls: type[Environment], + host_agent: bool = False, + host_response: bool = False + ) -> T: + """Creates an Integration instance using a sample and environment.""" + + def setup_method(self): + self._environment = environment_cls(sample_cls.get_config()) + await self._environment.__aenter__() + + self._sample = sample_cls(self._environment) + await self._sample.__aenter__() + + def teardown_method(self): + await self._sample.__aexit__(None, None, None) + await self._environment.__aexit__(None, None, None) + + target_cls = Integration._with_response_server(target_cls, host_response) + + return target_cls + + @staticmethod + def from_app(target_cls: T, app: AppT, host_response: bool = True) -> T: + """Creates an Integration instance using an aiohttp application.""" + + async def setup_method(self): + + self._app = app + self._runner = AppRunner(self._app) + await self._runner.__aenter__() + + async def teardown_method(self): + await self._runner.__aexit__(None, None, None) + + target_cls = Integration._with_response_server(target_cls, host_response) + + return target_cls + +def integration( + cls: T, + service_url: Optional[str] = None, + sample_cls: Optional[type[Sample]] = None, + environment_cls: Optional[type[Environment]] = None, + app: Optional[AppT] = None, + host_agent: bool = False, + host_response: bool = True, +) -> T: + """Factory function to create an Integration instance based on provided parameters. + + Essentially resolves to one of the static methods of Integration: + `from_service_url`, `from_sample`, or `from_app`, + based on the provided parameters. + + If a service URL is provided, it creates the Integration using that. + If both sample and environment are provided, it creates the Integration using them. + If an aiohttp application is provided, it creates the Integration using that. + + :param cls: The Integration class type. + :param service_url: Optional service URL to connect to. + :param sample: Optional Sample instance. + :param environment: Optional Environment instance. + :param host_agent: Flag to indicate if the agent should be hosted. + :param app: Optional aiohttp application instance. + :return: An instance of the Integration class. + """ + + if service_url: + return Integration.from_service_url(cls, service_url, host_response=host_response) + elif sample_cls and environment_cls: + return Integration.from_sample(cls, sample_cls, environment_cls, host_agent=host_agent, host_response=host_response) + elif app: + return Integration.from_app(cls, app, host_response=host_response) + else: + raise ValueError("Insufficient parameters to create Integration instance.") \ No newline at end of file diff --git a/dev/integration/src/core/integration_fixtures.py b/dev/integration/src/core/integration_fixtures.py new file mode 100644 index 00000000..0fcf1fa6 --- /dev/null +++ b/dev/integration/src/core/integration_fixtures.py @@ -0,0 +1,36 @@ +from typing import TypeVar, Any +import pytest + +from .environment import Environment +from .sample import Sample +from .client import AgentClient, AIAgentClient +from .response_server import ResponseServer + +class IntegrationFixtures: + """Provides integration test fixtures.""" + + _environment: Environment + _sample: Sample + _response_server: ResponseServer + + @pytest.fixture + def environment(self) -> Environment: + """Provides the test environment instance.""" + return self._environment + + @pytest.fixture + def sample(self) -> Sample: + """Provides the sample instance.""" + return self._sample + + @pytest.fixture + def agent_client(self) -> AgentClient: + ... + + @pytest.fixture + def ai_agent_client(self) -> AIAgentClient: + ... + + @pytest.fixture + def response_server(self) -> ResponseServer: + return self._response_server \ No newline at end of file diff --git a/dev/integration/src/core/response_server/__init__.py b/dev/integration/src/core/response_server/__init__.py new file mode 100644 index 00000000..3aeae6bf --- /dev/null +++ b/dev/integration/src/core/response_server/__init__.py @@ -0,0 +1,3 @@ +from .response_server import ResponseServer + +__all__ = ["ResponseServer"] \ No newline at end of file diff --git a/dev/integration/src/core/response_server/response_server.py b/dev/integration/src/core/response_server/response_server.py new file mode 100644 index 00000000..a621537b --- /dev/null +++ b/dev/integration/src/core/response_server/response_server.py @@ -0,0 +1,11 @@ +class ResponseServer: + """A mock response server for handling responses during integration tests.""" + + def __init__(self): + pass + + async def __aenter__(self): + pass + + async def __aexit__(self, exc_type, exc_val, exc_tb): + pass \ No newline at end of file diff --git a/dev/integration/src/core/runner/__init__.py b/dev/integration/src/core/runner/__init__.py index e69de29b..2e244b56 100644 --- a/dev/integration/src/core/runner/__init__.py +++ b/dev/integration/src/core/runner/__init__.py @@ -0,0 +1,3 @@ +from .app_runner import AppRunner + +__all__ = ["AppRunner"] \ No newline at end of file diff --git a/dev/integration/src/core/runner/app_runner.py b/dev/integration/src/core/runner/app_runner.py new file mode 100644 index 00000000..9eedc887 --- /dev/null +++ b/dev/integration/src/core/runner/app_runner.py @@ -0,0 +1,2 @@ +class AppRunner: + pass \ No newline at end of file diff --git a/dev/integration/src/core/sample.py b/dev/integration/src/core/sample.py index 524f1247..31148791 100644 --- a/dev/integration/src/core/sample.py +++ b/dev/integration/src/core/sample.py @@ -9,6 +9,11 @@ class Sample(ABC): def __init__(self, environment: Environment, **kwargs): self.env = environment + @classmethod + async def get_config(cls) -> dict: + """Retrieve the configuration for the sample.""" + return {} + @abstractmethod async def init_app(self): """Initialize the application for the sample.""" \ No newline at end of file From 45678e9b3b86b85dacf3b044fa9c96c8d3d722ad Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 27 Oct 2025 11:11:56 -0700 Subject: [PATCH 07/81] Drafting AgentClient and ResponseClient implementations --- dev/integration/src/core/bot_client.py | 47 ----- dev/integration/src/core/bot_response.py | 14 -- dev/integration/src/core/client/__init__.py | 6 +- .../src/core/client/agent_client.py | 115 ++++++++++- .../src/core/client/agent_client_base.py | 4 - .../src/core/client/ai_agent_client.py | 4 - dev/integration/src/core/client/bot_client.cs | 172 +++++++++++++++++ dev/integration/src/core/client/bot_client.py | 181 +++++++++++++++++ .../src/core/client/bot_response.cs | 141 ++++++++++++++ .../src/core/client/bot_response.py | 182 ++++++++++++++++++ .../src/core/client/response_client.py | 67 +++++++ 11 files changed, 857 insertions(+), 76 deletions(-) delete mode 100644 dev/integration/src/core/bot_client.py delete mode 100644 dev/integration/src/core/bot_response.py delete mode 100644 dev/integration/src/core/client/agent_client_base.py delete mode 100644 dev/integration/src/core/client/ai_agent_client.py create mode 100644 dev/integration/src/core/client/bot_client.cs create mode 100644 dev/integration/src/core/client/bot_client.py create mode 100644 dev/integration/src/core/client/bot_response.cs create mode 100644 dev/integration/src/core/client/bot_response.py create mode 100644 dev/integration/src/core/client/response_client.py diff --git a/dev/integration/src/core/bot_client.py b/dev/integration/src/core/bot_client.py deleted file mode 100644 index 2a547581..00000000 --- a/dev/integration/src/core/bot_client.py +++ /dev/null @@ -1,47 +0,0 @@ -from http import client -import os - -from microsoft_agents.activity import ( - Activity, - ActivityTypes, -) - -from msal import ConfidentialClientApplication - -class BotClient: - - def __init__(self, messaging_endpoint: str, service_endpoint: str, cid: str, client_id: str, tenant_id: str, client_secret: str): - self.messaging_endpoint = messaging_endpoint - self.service_endpoint = service_endpoint - self.cid = cid - self.client_id = client_id - self.tenant_id = tenant_id - self.client_secret = client_secret - - async def send_request(self, activity: Activity): - - client_id = os.environ["CLIENT_ID"] - - msal_app = ConfidentialClientApplication( - client_id=client_id, - tenant_id=os.environ["TENANT_ID"], - client_credential=os.environ["CLIENT_SECRET"], - ) - - token = msal_app.acquire_token_for_client([f"{client_id}/.default"]) - - http_client = - - async def send_activity(self, activity: Activity): - pass - - async def send_expect_replies_activity(self, activity: Activity): - pass - - async def send_stream_activity(self, activity: Activity): - pass - - async def send_invoke(self, activity: Activity): - if activity.type != ActivityTypes.invoke: - raise ValueError("Activity type must be 'invoke' for send_invoke method.") - return await self.send_request(activity) \ No newline at end of file diff --git a/dev/integration/src/core/bot_response.py b/dev/integration/src/core/bot_response.py deleted file mode 100644 index 07fb9282..00000000 --- a/dev/integration/src/core/bot_response.py +++ /dev/null @@ -1,14 +0,0 @@ -class BotResponse: - - def __init__(self): - pass - - def handle_streamed_activity(self, activity: Activity, sact: StreamInfo, cid: str) -> bool: - pass - - def dispose_async(self) -> ValueTask: - pass - - @property - def service_endpoint(self) -> str: - pass \ No newline at end of file diff --git a/dev/integration/src/core/client/__init__.py b/dev/integration/src/core/client/__init__.py index 5f5993f7..01c71621 100644 --- a/dev/integration/src/core/client/__init__.py +++ b/dev/integration/src/core/client/__init__.py @@ -1,9 +1,7 @@ from .agent_client import AgentClient -from .ai_agent_client import AIAgentClient -from .agent_client_base import BaseClient +from .response_client import ResponseClient __all__ = [ "AgentClient", - "AIAgentClient", - "BaseClient", + "ResponseClient", ] \ No newline at end of file diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py index 1ecdb488..0a86b8eb 100644 --- a/dev/integration/src/core/client/agent_client.py +++ b/dev/integration/src/core/client/agent_client.py @@ -1,4 +1,113 @@ -from .agent_client_base import AgentClientBase +import os +import json +import asyncio +from re import A +from typing import Any, Optional -class AgentClient(AgentClientBase): - pass \ No newline at end of file +from aiohttp import ClientSession + +from microsoft_agents.activity import ( + Activity, + ActivityTypes, + DeliveryModes +) + +from msal import ConfidentialClientApplication + +class AgentClient: + + def __init__( + self, + messaging_endpoint: str, + service_endpoint: str, + cid: str, + client_id: str, + tenant_id: str, + client_secret: str, + default_timeout: float = 5.0 + ): + self._messaging_endpoint = messaging_endpoint + self.service_endpoint = service_endpoint + self.cid = cid + self.client_id = client_id + self.tenant_id = tenant_id + self.client_secret = client_secret + self._headers = None + self._default_timeout = default_timeout + + self._client = ClientSession( + base_url=self._messaging_endpoint, + headers={"Content-Type": "application/json"} + ) + + self._msal_app = ConfidentialClientApplication( + client_id=client_id, + client_credential=client_secret, + authority=f"https://login.microsoftonline.com/{tenant_id}" + ) + + async def get_access_token(self) -> str: + res = self._msal_app.acquire_token_for_client( + scopes=[f"{self.client_id}/.default"] + ) + token = res.get("access_token") if res else None + if not token: + raise Exception("Could not obtain access token") + return token + + async def _set_headers(self) -> None: + if not self._headers: + token = await self.get_access_token() + self._headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + async def send_request(self, activity: Activity) -> str: + + await self._set_headers() + + if activity.conversation: + activity.conversation.id = self.cid + + async with self._client.post( + self._messaging_endpoint, + headers=self._headers, + json=activity.model_dump(by_alias=True, exclude_none=True) + ) as response: + if not response.ok: + raise Exception(f"Failed to send activity: {response.status}") + content = await response.text() + return content + + async def send_activity(self, activity: Activity, timeout: Optional[float] = None) -> list[Activity]: + timeout = timeout or self._default_timeout + + content = await self.send_request(activity) + return content + + async def send_expect_replies_activity(self, activity: Activity) -> list[Activity]: + if activity.delivery_mode != DeliveryModes.expect_replies: + raise ValueError("Activity delivery_mode must be 'expectReplies' for this method.") + + content = await self.send_request(activity) + if not content: + raise RuntimeError("Expected replies but received no content.") + + activities_content = json.loads(content) + activities = [ + Activity.model_validate_json(json.dumps(act)) + for act in activities_content + ] + + return activities + + async def send_stream_activity(self, activity: Activity) -> list[Activity]: + raise NotImplementedError("send_stream_activity is not implemented yet.") + + async def send_invoke(self, activity: Activity) -> str: + if activity.type != ActivityTypes.invoke: + raise ValueError("Activity type must be 'invoke' for send_invoke method.") + + content = await self.send_request(activity) + return content \ No newline at end of file diff --git a/dev/integration/src/core/client/agent_client_base.py b/dev/integration/src/core/client/agent_client_base.py deleted file mode 100644 index 62bd85ed..00000000 --- a/dev/integration/src/core/client/agent_client_base.py +++ /dev/null @@ -1,4 +0,0 @@ -from abc import ABC - -class AgentClientBase(ABC): - pass \ No newline at end of file diff --git a/dev/integration/src/core/client/ai_agent_client.py b/dev/integration/src/core/client/ai_agent_client.py deleted file mode 100644 index 072fe004..00000000 --- a/dev/integration/src/core/client/ai_agent_client.py +++ /dev/null @@ -1,4 +0,0 @@ -from .agent_client import AgentClient - -class AIAgentClient(AgentClient): - pass \ No newline at end of file diff --git a/dev/integration/src/core/client/bot_client.cs b/dev/integration/src/core/client/bot_client.cs new file mode 100644 index 00000000..793e823e --- /dev/null +++ b/dev/integration/src/core/client/bot_client.cs @@ -0,0 +1,172 @@ +using Microsoft.Agents.Core.Models; +using Microsoft.Agents.Core.Serialization; +using Microsoft.Identity.Client; +using System.Collections.Concurrent; +using System.Text.Json; + +namespace Framework +{ + public class BotClient(string messagingEndpoint, string serviceEndpoint, string cid, string clientId, string tenantId, string clientSecret) + { + public static ConcurrentDictionary> TaskList = new(); + + public async Task SendRequest(Activity activity) + { + + // Create bearer authentication token + IConfidentialClientApplication app = ConfidentialClientApplicationBuilder + .Create(clientId) + .WithTenantId(tenantId) + .WithClientSecret(clientSecret) + .Build(); + + var token = await app.AcquireTokenForClient([clientId + "/.default"]).ExecuteAsync(); + + // Send request to agent + HttpClient http = new(); + + // Update activity to send to service endpoint + activity.ServiceUrl = serviceEndpoint; + + // Update activity conversation ID + activity.Conversation.Id = cid; + + var stringContent = new StringContent(activity.ToJson(), System.Text.Encoding.UTF8, "application/json"); + + HttpRequestMessage req = new(HttpMethod.Post, messagingEndpoint) + { + Headers = + { + { "Authorization", "Bearer " + token.AccessToken } + }, + Content = stringContent + + }; + + var resp = await http.SendAsync(req); + + // Check if request was successful + if (!resp.IsSuccessStatusCode) + { + throw new Exception("Failed to send activity: " + resp.StatusCode); + } + + var content = await resp.Content.ReadAsStringAsync(); + return content; + } + public async Task SendActivity(Activity activity) + { + + // Keep track of activities being sent to the web service + TaskCompletionSource tcs = new(); + + TaskList.TryAdd(cid, tcs); + + // Send activity to the agent + var content = await SendRequest(activity); + + // Receive response from the agent + var result = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(20)); + TaskList.TryRemove(cid, out _); + + return result; + } + + public async Task SendExpectRepliesActivity(Activity activity) + { + // Validate that the activity is of delivery mode expected replies + if (activity.DeliveryMode != DeliveryModes.ExpectReplies) + { + throw new InvalidOperationException("Activity type must be ExpectedReplies."); + } + + // Keep track of activities being sent to the web service + TaskCompletionSource tcs = new(); + + // Send activity to the agent + var content = await SendRequest(activity); + + // Receive response from the agent + if (content.Length == 0) + { + throw new InvalidOperationException("No response received from the agent."); + } + JsonDocument jsonDoc = JsonDocument.Parse(content); + JsonElement activitiesJson = jsonDoc.RootElement.GetProperty("activities"); + var activities = ProtocolJsonSerializer.ToObject(activitiesJson); + + return activities; + + } + + public async Task SendStreamActivity(Activity activity) + { + // Validate that the activity is of type Stream + if (activity.DeliveryMode != DeliveryModes.Stream) + { + throw new InvalidOperationException("Activity type must be Stream."); + } + + // Keep track of activities being sent to the web service + TaskCompletionSource tcs = new(); + + TaskList.TryAdd(cid, tcs); + + // Send activity to the agent + var content = await SendRequest(activity); + + // Check if error comes back + if (content.Length == 0) { + var result = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(6)); + if (result.Length > 0) + { + return result; + } + } + + // Receive response from the agent + if (content.Length == 0) + { + throw new InvalidOperationException("No response received from the agent."); + } + + var split = content.Split("\n\r\n"); + split = split.Where(s => !string.IsNullOrWhiteSpace(s)).ToArray(); + + Activity[] activities = new Activity[split.Length]; + + for (int i = 0; i < split.Length; i++) + { + if (split[i].Contains("event: activity")) + { + var format = split[i].Split("data: "); + var act = ProtocolJsonSerializer.ToObject(format[1]); + activities[i] = act; + } else + { + throw new Exception("Must receive server-sent events"); + } + } + + TaskList.TryRemove(cid, out _); + + return activities; + } + + + public async Task SendInvoke(Activity activity) + { + + // Validate that the activity is of type Invoke + if (activity.Type != ActivityTypes.Invoke) + { + throw new InvalidOperationException("Activity type must be Invoke."); + } + + // Send activity to the agent + var content = await SendRequest(activity); + + return content; + } + } +} \ No newline at end of file diff --git a/dev/integration/src/core/client/bot_client.py b/dev/integration/src/core/client/bot_client.py new file mode 100644 index 00000000..60c0613f --- /dev/null +++ b/dev/integration/src/core/client/bot_client.py @@ -0,0 +1,181 @@ +import asyncio +import json +from concurrent.futures import ThreadPoolExecutor +from typing import Dict, List, Optional +import aiohttp +from msal import ConfidentialClientApplication + +from microsoft_agents.activity import ( + Activity, + ActivityTypes, + DeliveryModes, +) + + +class BotClient: + """Python equivalent of the C# BotClient class for sending activities to agents.""" + + def __init__( + self, + messaging_endpoint: str, + service_endpoint: str, + cid: str, + client_id: str, + tenant_id: str, + client_secret: str + ): + self.messaging_endpoint = messaging_endpoint + self.service_endpoint = service_endpoint + self.cid = cid + self.client_id = client_id + self.tenant_id = tenant_id + self.client_secret = client_secret + + # Dictionary to track pending tasks (equivalent to C# ConcurrentDictionary) + self.task_list: Dict[str, asyncio.Future[List[Activity]]] = {} + + # MSAL app for authentication + self.msal_app = ConfidentialClientApplication( + client_id=client_id, + client_credential=client_secret, + authority=f"https://login.microsoftonline.com/{tenant_id}" + ) + + async def send_request(self, activity: Activity) -> str: + """Send an HTTP request with the activity to the messaging endpoint.""" + + # Acquire token for authentication + token_result = self.msal_app.acquire_token_for_client( + scopes=[f"{self.client_id}/.default"] + ) + + if "error" in token_result: + raise Exception(f"Failed to acquire token: {token_result['error_description']}") + + # Update activity properties + activity.service_url = self.service_endpoint + if activity.conversation: + activity.conversation.id = self.cid + + # Prepare the request + headers = { + "Authorization": f"Bearer {token_result['access_token']}", + "Content-Type": "application/json" + } + + # Serialize activity to JSON + activity_json = activity.model_dump(by_alias=True, exclude_none=True) + + async with aiohttp.ClientSession() as session: + async with session.post( + self.messaging_endpoint, + headers=headers, + json=activity_json + ) as response: + if not response.ok: + raise Exception(f"Failed to send activity: {response.status}") + + content = await response.text() + return content + + async def send_activity(self, activity: Activity) -> List[Activity]: + """Send an activity and wait for the response.""" + + # Create a future to track the response + future: asyncio.Future[List[Activity]] = asyncio.Future() + self.task_list[self.cid] = future + + try: + # Send activity to the agent + await self.send_request(activity) + + # Wait for response with timeout (20 seconds) + result = await asyncio.wait_for(future, timeout=20.0) + return result + finally: + # Clean up the task from the list + self.task_list.pop(self.cid, None) + + async def send_expect_replies_activity(self, activity: Activity) -> List[Activity]: + """Send an activity with delivery mode expect_replies.""" + + # Validate that the activity has the correct delivery mode + if activity.delivery_mode != DeliveryModes.expect_replies: + raise ValueError("Activity delivery mode must be expect_replies.") + + # Send activity to the agent + content = await self.send_request(activity) + + # Parse response + if not content: + raise ValueError("No response received from the agent.") + + response_data = json.loads(content) + activities_data = response_data.get("activities", []) + + # Convert JSON to Activity objects + activities = [Activity.model_validate(act_data) for act_data in activities_data] + return activities + + async def send_stream_activity(self, activity: Activity) -> List[Activity]: + """Send an activity with delivery mode stream.""" + + # Validate that the activity has the correct delivery mode + if activity.delivery_mode != DeliveryModes.stream: + raise ValueError("Activity delivery mode must be stream.") + + # Create a future to track the response + future: asyncio.Future[List[Activity]] = asyncio.Future() + self.task_list[self.cid] = future + + try: + # Send activity to the agent + content = await self.send_request(activity) + + # Check if we got an immediate response + if not content: + # Wait for streaming response + result = await asyncio.wait_for(future, timeout=6.0) + if result: + return result + raise ValueError("No response received from the agent.") + + # Parse server-sent events + lines = [line.strip() for line in content.split("\n\r\n") if line.strip()] + activities = [] + + for line in lines: + if "event: activity" in line: + # Extract the data part after "data: " + data_parts = line.split("data: ") + if len(data_parts) > 1: + activity_data = json.loads(data_parts[1]) + activity = Activity.model_validate(activity_data) + activities.append(activity) + else: + raise Exception("Must receive server-sent events") + + return activities + finally: + # Clean up the task from the list + self.task_list.pop(self.cid, None) + + async def send_invoke(self, activity: Activity) -> str: + """Send an invoke activity.""" + + # Validate that the activity is of type Invoke + if activity.type != ActivityTypes.invoke: + raise ValueError("Activity type must be invoke.") + + # Send activity to the agent + content = await self.send_request(activity) + return content + + def complete_task(self, conversation_id: str, activities: List[Activity]): + """Complete a pending task with the received activities. + + This method should be called by the webhook handler when activities are received. + """ + future = self.task_list.get(conversation_id) + if future and not future.done(): + future.set_result(activities) \ No newline at end of file diff --git a/dev/integration/src/core/client/bot_response.cs b/dev/integration/src/core/client/bot_response.cs new file mode 100644 index 00000000..460942bb --- /dev/null +++ b/dev/integration/src/core/client/bot_response.cs @@ -0,0 +1,141 @@ +using Microsoft.Agents.Core.Models; +using Microsoft.Agents.Core.Serialization; +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Identity.Web; +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text.Json; + +namespace Framework +{ + public sealed class BotResponse : IAsyncDisposable + { + readonly WebApplication _app; + ConcurrentDictionary> _multipleActivities = new(); + public string TestId { get; } = Guid.NewGuid().ToString(); + public BotResponse() + { + // Suppress console output + Console.SetOut(TextWriter.Null); + + // Create a web application builder and configure services + WebApplicationBuilder builder = WebApplication.CreateBuilder(); + + builder.Services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme) + .AddMicrosoftIdentityWebApi(builder.Configuration); + + // Add services to the web application + _app = builder.Build(); + _app.UseRouting(); + _app.MapPost("/v3/conversations/{*text}", async (ctx) => + { + + using StreamReader reader = new(ctx.Request.Body); + var resp = await reader.ReadToEndAsync(); + Activity act = ProtocolJsonSerializer.ToObject(resp); + string cid = act.Conversation.Id; + var activityList = _multipleActivities.GetOrAdd(cid!, _ => new List()); + + lock (activityList) + { + activityList.Add(act); + } + + var response = new + { + Id = Guid.NewGuid().ToString() + }; + + // Check if the activity is a streamed activity + if (act.Entities?.Any(e => e.Type == EntityTypes.StreamInfo) == true) + { + var entities = ProtocolJsonSerializer.ToJson(act.Entities[0]); + var sact = ProtocolJsonSerializer.ToObject(entities); + + bool handled = HandleStreamedActivity(act, sact, cid); + + ctx.Response.StatusCode = 200; + ctx.Response.ContentType = "application/json"; + await ctx.Response.WriteAsync(JsonSerializer.Serialize(response)); + + if (BotClient.TaskList.TryGetValue(cid!, out var tcs) && handled) + { + if (_multipleActivities.TryGetValue(cid, out var streamedActivity)) + { + tcs!.TrySetResult(streamedActivity.ToArray()); + _multipleActivities.TryRemove(cid, out _); + } + } + } + else + { + + if (act.Type != ActivityTypes.Typing) + { + _ = Task.Run(async () => + { + await Task.Delay(5000); + if (BotClient.TaskList.TryGetValue(cid!, out var tcs)) + { + _multipleActivities.TryGetValue(cid!, out var result); + tcs?.TrySetResult(result!.ToArray()); + } + }); + } + ctx.Response.StatusCode = 200; + ctx.Response.ContentType = "application/json"; + await ctx.Response.WriteAsync(JsonSerializer.Serialize(response)); + } + }); + + ServiceEndpoint = "http://localhost:9873"; + + _app.UseAuthentication(); + _app.UseAuthorization(); + _app.RunAsync(ServiceEndpoint); + } + + private bool HandleStreamedActivity(Activity act, StreamInfo sact, string cid) + { + + // Check if activity is the final message + if (sact.StreamType == StreamTypes.Final) + { + if (act.Type == ActivityTypes.Message) + { + return true; + } + else + { + throw new Exception("final streamed activity should be type message"); + } + } + + // Handler for streaming types which allows us to verify later if the text length has increased + else if (sact.StreamType == StreamTypes.Streaming) + { + if (sact.StreamSequence <= 0 && act.Type == ActivityTypes.Typing) + { + throw new Exception("streamed activity's stream sequence should be a positive number"); + } + } + // Activity is being streamed but isn't the final message + return false; + + } + + public async ValueTask DisposeAsync() + { + await _app.StopAsync(); + await _app.DisposeAsync(); + } + + public string ServiceEndpoint { get; private set; } + + } + +} \ No newline at end of file diff --git a/dev/integration/src/core/client/bot_response.py b/dev/integration/src/core/client/bot_response.py new file mode 100644 index 00000000..1ba46c0c --- /dev/null +++ b/dev/integration/src/core/client/bot_response.py @@ -0,0 +1,182 @@ +import asyncio +import json +import sys +import threading +import uuid +from io import StringIO +from typing import Dict, List, Optional +from threading import Lock +from collections import defaultdict + +from aiohttp import web, ClientSession +from aiohttp.web import Request, Response +import aiohttp_security +from microsoft_agents.core.models import Activity, EntityTypes, ActivityTypes, StreamInfo, StreamTypes +from microsoft_agents.core.serialization import ProtocolJsonSerializer + +class BotResponse: + """Python equivalent of the C# BotResponse class using aiohttp web framework.""" + + def __init__(self): + self._app: Optional[web.Application] = None + self._runner: Optional[web.AppRunner] = None + self._site: Optional[web.TCPSite] = None + self._multiple_activities: Dict[str, List[Activity]] = defaultdict(list) + self._activity_locks: Dict[str, Lock] = defaultdict(Lock) + self.test_id: str = str(uuid.uuid4()) + self.service_endpoint: str = "http://localhost:9873" + + # Suppress console output (equivalent to Console.SetOut(TextWriter.Null)) + sys.stdout = StringIO() + + # Initialize the web application + self._setup_app() + + def _setup_app(self): + """Setup the aiohttp web application with routes and middleware.""" + self._app = web.Application() + + # Add JWT authentication middleware (placeholder - would need proper implementation) + # self._app.middlewares.append(self._auth_middleware) + + # Add routes + self._app.router.add_post('/v3/conversations/{path:.*}', self._handle_conversation) + + async def _auth_middleware(self, request: Request, handler): + """JWT authentication middleware (placeholder implementation).""" + # TODO: Implement proper JWT authentication + return await handler(request) + + async def _handle_conversation(self, request: Request) -> Response: + """Handle POST requests to /v3/conversations/{*text}.""" + try: + # Read request body + body = await request.text() + act = ProtocolJsonSerializer.to_object(body, Activity) + cid = act.conversation.id if act.conversation else None + + if not cid: + return web.Response(status=400, text="Missing conversation ID") + + # Add activity to the list (thread-safe) + with self._activity_locks[cid]: + self._multiple_activities[cid].append(act) + + # Create response + response_data = { + "Id": str(uuid.uuid4()) + } + + # Check if the activity is a streamed activity + if (act.entities and + any(e.type == EntityTypes.STREAM_INFO for e in act.entities)): + + entities_json = ProtocolJsonSerializer.to_json(act.entities[0]) + sact = ProtocolJsonSerializer.to_object(entities_json, StreamInfo) + + handled = self._handle_streamed_activity(act, sact, cid) + + response = web.Response( + status=200, + content_type="application/json", + text=json.dumps(response_data) + ) + + # Handle task completion (would need BotClient implementation) + if handled: + await self._complete_streaming_task(cid) + + return response + else: + # Handle non-streamed activities + if act.type != ActivityTypes.TYPING: + # Start background task with 5-second delay + asyncio.create_task(self._delayed_task_completion(cid)) + + return web.Response( + status=200, + content_type="application/json", + text=json.dumps(response_data) + ) + + except Exception as e: + return web.Response(status=500, text=str(e)) + + def _handle_streamed_activity(self, act: Activity, sact: StreamInfo, cid: str) -> bool: + """Handle streamed activity logic.""" + + # Check if activity is the final message + if sact.stream_type == StreamTypes.FINAL: + if act.type == ActivityTypes.MESSAGE: + return True + else: + raise Exception("final streamed activity should be type message") + + # Handler for streaming types which allows us to verify later if the text length has increased + elif sact.stream_type == StreamTypes.STREAMING: + if sact.stream_sequence <= 0 and act.type == ActivityTypes.TYPING: + raise Exception("streamed activity's stream sequence should be a positive number") + + # Activity is being streamed but isn't the final message + return False + + async def _complete_streaming_task(self, cid: str): + """Complete streaming task (placeholder for BotClient.TaskList logic).""" + # TODO: Implement BotClient.TaskList equivalent + # This would require the BotClient class to be implemented + if cid in self._multiple_activities: + activities = self._multiple_activities[cid].copy() + # Complete the task with activities + # BotClient.complete_task(cid, activities) + # Clean up + del self._multiple_activities[cid] + if cid in self._activity_locks: + del self._activity_locks[cid] + + async def _delayed_task_completion(self, cid: str): + """Handle delayed task completion after 5 seconds.""" + await asyncio.sleep(5.0) + # TODO: Implement BotClient.TaskList equivalent + # if BotClient.has_task(cid): + # activities = self._multiple_activities.get(cid, []) + # BotClient.complete_task(cid, activities) + + async def start(self): + """Start the web server.""" + self._runner = web.AppRunner(self._app) + await self._runner.setup() + + # Extract port from service_endpoint + port = int(self.service_endpoint.split(':')[-1]) + self._site = web.TCPSite(self._runner, 'localhost', port) + await self._site.start() + + print(f"Bot server started at {self.service_endpoint}") + + async def dispose(self): + """Cleanup resources (equivalent to DisposeAsync).""" + if self._site: + await self._site.stop() + if self._runner: + await self._runner.cleanup() + + # Restore stdout + sys.stdout = sys.__stdout__ + + +# Example usage +async def main(): + bot_response = BotResponse() + try: + await bot_response.start() + # Keep the server running + while True: + await asyncio.sleep(1) + except KeyboardInterrupt: + print("Shutting down...") + finally: + await bot_response.dispose() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py new file mode 100644 index 00000000..5275b349 --- /dev/null +++ b/dev/integration/src/core/client/response_client.py @@ -0,0 +1,67 @@ +import sys +from io import StringIO +from typing import Optional +from threading import Lock + +from aiohttp import ClientSession +from aiohttp.web import Application, Request, Response + +from microsoft_agents.activity import ( + Activity, + ActivityTypes, +) + +class ResponseClient: + + def __init__( + self, + service_endpoint: str = "http://localhost:9873" + ): + self._app: Application = Application() + self._prev_stdout = None + self._service_endpoint = service_endpoint + self._activities_list = [] + self._activities_list_lock = [] + + self._app.router.add_post( + "/v3/conversations/{path:.*}", + self._handle_conversation + ) + + @property + def service_endpoint(self) -> str: + return self._service_endpoint + + def __aenter__(self): + self._prev_stdout = sys.stdout + sys.stdout = StringIO() + return self + + def __aexit__(self, exc_type, exc, tb): + if self._prev_stdout is not None: + sys.stdout = self._prev_stdout + + async def _handle_conversation(self, request: Request) -> Response: + try: + body = await request.text() + activity = Activity.model_validate(body) + + conversation_id = activity.conversation.id if activity.conversation else None + + with self._activities_list_lock: + self._activities_list.append(activity) + + if any(map(lambda x: x.type == "streaminfo", activity.entities or [])): + await self._handle_streamed_activity(activity) + else: + if activity.type != ActivityTypes.typing: + async with ClientSession() as session: + async with session.post( + f"{self._service_endpoint}/v3/conversations/{conversation_id}/activities", + json=activity.model_dump() + ) as resp: + resp_text = await resp.text() + return Response(status=resp.status, text=resp_text) + + async def _handle_streamed_activity(self, activity: Activity, *args, **kwargs) -> bool: + raise NotImplementedError("_handle_streamed_activity is not implemented yet.") \ No newline at end of file From c2b84cb5a8db66f365af40016bb7a60cdb83baa9 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 27 Oct 2025 12:28:25 -0700 Subject: [PATCH 08/81] Spec test --- .../src/core/client/agent_client_protocol.py | 17 ++++++++++++ .../core/client/response_client_protocol.py | 11 ++++++++ dev/integration/src/tests/test_quickstart.py | 26 +++++++++++++++++-- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 dev/integration/src/core/client/agent_client_protocol.py create mode 100644 dev/integration/src/core/client/response_client_protocol.py diff --git a/dev/integration/src/core/client/agent_client_protocol.py b/dev/integration/src/core/client/agent_client_protocol.py new file mode 100644 index 00000000..7e0aacec --- /dev/null +++ b/dev/integration/src/core/client/agent_client_protocol.py @@ -0,0 +1,17 @@ +from typing import Protocol + +from microsoft_agents.activity import Activity + +class AgentClientProtocol(Protocol): + + async def send_request(self, activity: Activity) -> str: + ... + + async def send_activity(self, activity: Activity) -> list[Activity]: + ... + + async def send_expect_replies_activity(self, activity: Activity) -> list[Activity]: + ... + + async def send_invoke_activity(self, activity: Activity) -> str: + ... \ No newline at end of file diff --git a/dev/integration/src/core/client/response_client_protocol.py b/dev/integration/src/core/client/response_client_protocol.py new file mode 100644 index 00000000..d104e492 --- /dev/null +++ b/dev/integration/src/core/client/response_client_protocol.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from typing import Protocol + +class ResponseClientProtocol(Protocol): + + async def __aenter__(self) -> ResponseClientProtocol: + ... + + async def __aexit__(self, exc_type, exc, tb) -> None: + ... \ No newline at end of file diff --git a/dev/integration/src/tests/test_quickstart.py b/dev/integration/src/tests/test_quickstart.py index 2dce2dd3..3fc72a0c 100644 --- a/dev/integration/src/tests/test_quickstart.py +++ b/dev/integration/src/tests/test_quickstart.py @@ -1,6 +1,6 @@ import pytest -from ..core import integration_test_suite_factory +from ..core import integration_test_suite_factory, integration from ..samples import QuickstartSample TestSuiteBase = integration_test_suite_factory(QuickstartSample) @@ -9,4 +9,26 @@ class TestQuickstart(TestSuiteBase): @pytest.mark.asyncio async def test_quickstart_functionality(self): - pass \ No newline at end of file + pass + +@integration(QuickstartSample, None) # env +class TestQuickstart: + + @pytest.mark.asyncio + async def test_hello(self, agent_client, env): + agent_client.send("hi") + + await asyncio.sleep(1) + + # assert env.auth... + +@integration(app=None) # (endpoint="alternative") +class TestQuickstartAlternative: + + @pytest.mark.asyncio + async def test_hello(self, agent_client, response_client): + + agent_client.send("hi") + await asyncio.sleep(10) + + assert receiver.has_activity("hello") \ No newline at end of file From 2816e68baa1850b3e03cd0379e60ea4a10266f74 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 27 Oct 2025 14:42:25 -0700 Subject: [PATCH 09/81] Filling in more implementation details --- .../src/core/client/agent_client_protocol.py | 17 ------ .../src/core/client/response_client.py | 4 +- .../core/environment/application_runner.py | 37 ++++++++++++ .../src/core/environment/driver.py | 4 -- .../src/core/integration_fixtures.py | 22 +++---- .../core/integration_test_suite_factory.py | 57 ------------------- .../src/core/response_server/__init__.py | 3 - .../core/response_server/response_server.py | 11 ---- dev/integration/src/core/runner/__init__.py | 3 - dev/integration/src/core/runner/app_runner.py | 2 - dev/integration/src/core/runner/driver.py | 0 11 files changed, 52 insertions(+), 108 deletions(-) delete mode 100644 dev/integration/src/core/client/agent_client_protocol.py create mode 100644 dev/integration/src/core/environment/application_runner.py delete mode 100644 dev/integration/src/core/environment/driver.py delete mode 100644 dev/integration/src/core/integration_test_suite_factory.py delete mode 100644 dev/integration/src/core/response_server/__init__.py delete mode 100644 dev/integration/src/core/response_server/response_server.py delete mode 100644 dev/integration/src/core/runner/__init__.py delete mode 100644 dev/integration/src/core/runner/app_runner.py delete mode 100644 dev/integration/src/core/runner/driver.py diff --git a/dev/integration/src/core/client/agent_client_protocol.py b/dev/integration/src/core/client/agent_client_protocol.py deleted file mode 100644 index 7e0aacec..00000000 --- a/dev/integration/src/core/client/agent_client_protocol.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Protocol - -from microsoft_agents.activity import Activity - -class AgentClientProtocol(Protocol): - - async def send_request(self, activity: Activity) -> str: - ... - - async def send_activity(self, activity: Activity) -> list[Activity]: - ... - - async def send_expect_replies_activity(self, activity: Activity) -> list[Activity]: - ... - - async def send_invoke_activity(self, activity: Activity) -> str: - ... \ No newline at end of file diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py index 5275b349..11cdbd7e 100644 --- a/dev/integration/src/core/client/response_client.py +++ b/dev/integration/src/core/client/response_client.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import sys from io import StringIO from typing import Optional @@ -32,7 +34,7 @@ def __init__( def service_endpoint(self) -> str: return self._service_endpoint - def __aenter__(self): + def __aenter__(self) -> ResponseClient: self._prev_stdout = sys.stdout sys.stdout = StringIO() return self diff --git a/dev/integration/src/core/environment/application_runner.py b/dev/integration/src/core/environment/application_runner.py new file mode 100644 index 00000000..2149e976 --- /dev/null +++ b/dev/integration/src/core/environment/application_runner.py @@ -0,0 +1,37 @@ +from typing import TypeVar +from threading import Thread + +import aiohttp.web + +AppT = TypeVar('AppT', bound=aiohttp.web.Application) + +class ApplicationRunner: + def __init__(self, app: AppT): + self._app = app + self._thread = None + + async def _start_server(self) -> None: + runner = aiohttp.web.AppRunner(self._app) + await runner.setup() + site = aiohttp.web.TCPSite(runner, 'localhost', 8080) + await site.start() + + async def __aenter__(self) -> None: + + if self._thread: + raise RuntimeError("Server is already running") + + self._thread = Thread(target=self._start_server, daemon=True) + self._thread.start() + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + + if self._thread: + + if isinstance(self._app, aiohttp.web.Application): + await self._app.shutdown() + + self._thread.join() + self._thread = None + else: + raise RuntimeError("Server is not running") \ No newline at end of file diff --git a/dev/integration/src/core/environment/driver.py b/dev/integration/src/core/environment/driver.py deleted file mode 100644 index 9d421530..00000000 --- a/dev/integration/src/core/environment/driver.py +++ /dev/null @@ -1,4 +0,0 @@ -import threading - -async def aiohttp_driver() -> None: - diff --git a/dev/integration/src/core/integration_fixtures.py b/dev/integration/src/core/integration_fixtures.py index 0fcf1fa6..e5a46321 100644 --- a/dev/integration/src/core/integration_fixtures.py +++ b/dev/integration/src/core/integration_fixtures.py @@ -1,17 +1,17 @@ -from typing import TypeVar, Any +from typing import TypeVar, Any, AsyncGenerator, Callable import pytest from .environment import Environment from .sample import Sample -from .client import AgentClient, AIAgentClient -from .response_server import ResponseServer +from .client import AgentClient, ResponseClient class IntegrationFixtures: """Provides integration test fixtures.""" _environment: Environment _sample: Sample - _response_server: ResponseServer + _agent_client: AgentClient + _response_client: ResponseClient @pytest.fixture def environment(self) -> Environment: @@ -25,12 +25,14 @@ def sample(self) -> Sample: @pytest.fixture def agent_client(self) -> AgentClient: - ... - + return self._agent_client + @pytest.fixture - def ai_agent_client(self) -> AIAgentClient: - ... + async def response_client(self) -> AsyncGenerator[ResponseClient, None]: + """Provides the response client instance.""" + async with ResponseClient() as response_client: + yield response_client @pytest.fixture - def response_server(self) -> ResponseServer: - return self._response_server \ No newline at end of file + def create_response_client(self) -> Callable[None, ResponseClient]: + return lambda: ResponseClient() \ No newline at end of file diff --git a/dev/integration/src/core/integration_test_suite_factory.py b/dev/integration/src/core/integration_test_suite_factory.py deleted file mode 100644 index 24eec943..00000000 --- a/dev/integration/src/core/integration_test_suite_factory.py +++ /dev/null @@ -1,57 +0,0 @@ -from typing import Optional - -import pytest - -from microsoft_agents.hosting.core import ( - AgentApplication, - ChannelAdapter, - Connections -) - -from .environment import Environment, create_aiohttp_env -from .sample import Sample - -def integration_test_suite_factory( - sample_cls: type[Sample], - environment: Optional[Environment] = None -): - """Factory function to create an integration test suite for a given sample application. - - :param environment: The environment to use for the sample application. - :param sample_cls: The sample class to create the test suite for. - :return: An integration test suite class. - """ - - if not environment: - environment = create_aiohttp_env() - - class IntegrationTestSuite: - """Integration test suite for a given sample application.""" - sample: Sample - - async def setup_method(self, mocker): - """Set up the test suite with the sample application.""" - self.sample = sample_cls(environment, mocker=mocker) - await self.sample.init_app() - await self.sample.runner.start() - - async def teardown_method(self): - """Tear down the test suite and clean up resources.""" - await self.sample.runner.stop() - - @pytest.fixture - def agent_application(self) -> AgentApplication: - """Get the agent application for the test suite.""" - return self.sample.env.agent_application - - @pytest.fixture - def adapter(self) -> ChannelAdapter: - """Get the channel adapter for the test suite.""" - return self.sample.env.adapter - - @pytest.fixture - def connections(self) -> Connections: - """Get the connections for the test suite.""" - return self.sample.env.connections - - return IntegrationTestSuite \ No newline at end of file diff --git a/dev/integration/src/core/response_server/__init__.py b/dev/integration/src/core/response_server/__init__.py deleted file mode 100644 index 3aeae6bf..00000000 --- a/dev/integration/src/core/response_server/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .response_server import ResponseServer - -__all__ = ["ResponseServer"] \ No newline at end of file diff --git a/dev/integration/src/core/response_server/response_server.py b/dev/integration/src/core/response_server/response_server.py deleted file mode 100644 index a621537b..00000000 --- a/dev/integration/src/core/response_server/response_server.py +++ /dev/null @@ -1,11 +0,0 @@ -class ResponseServer: - """A mock response server for handling responses during integration tests.""" - - def __init__(self): - pass - - async def __aenter__(self): - pass - - async def __aexit__(self, exc_type, exc_val, exc_tb): - pass \ No newline at end of file diff --git a/dev/integration/src/core/runner/__init__.py b/dev/integration/src/core/runner/__init__.py deleted file mode 100644 index 2e244b56..00000000 --- a/dev/integration/src/core/runner/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .app_runner import AppRunner - -__all__ = ["AppRunner"] \ No newline at end of file diff --git a/dev/integration/src/core/runner/app_runner.py b/dev/integration/src/core/runner/app_runner.py deleted file mode 100644 index 9eedc887..00000000 --- a/dev/integration/src/core/runner/app_runner.py +++ /dev/null @@ -1,2 +0,0 @@ -class AppRunner: - pass \ No newline at end of file diff --git a/dev/integration/src/core/runner/driver.py b/dev/integration/src/core/runner/driver.py deleted file mode 100644 index e69de29b..00000000 From 2214827c00185ca4b41ca3b0e5cbeaccf19c74a9 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 27 Oct 2025 14:55:04 -0700 Subject: [PATCH 10/81] More files --- dev/integration/src/auto_client/__init__.py | 0 .../src/auto_client/auto_client.py | 4 + dev/integration/src/core/__init__.py | 13 +- dev/integration/src/core/client/bot_client.py | 181 ------------------ dev/integration/src/mocks/__init__.py | 0 5 files changed, 15 insertions(+), 183 deletions(-) create mode 100644 dev/integration/src/auto_client/__init__.py create mode 100644 dev/integration/src/auto_client/auto_client.py delete mode 100644 dev/integration/src/core/client/bot_client.py create mode 100644 dev/integration/src/mocks/__init__.py diff --git a/dev/integration/src/auto_client/__init__.py b/dev/integration/src/auto_client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/auto_client/auto_client.py b/dev/integration/src/auto_client/auto_client.py new file mode 100644 index 00000000..8aee4368 --- /dev/null +++ b/dev/integration/src/auto_client/auto_client.py @@ -0,0 +1,4 @@ +from ..core import AgentClient + +class AutoClient(AgentClient): + pass \ No newline at end of file diff --git a/dev/integration/src/core/__init__.py b/dev/integration/src/core/__init__.py index df7ed39e..a13fcd60 100644 --- a/dev/integration/src/core/__init__.py +++ b/dev/integration/src/core/__init__.py @@ -2,12 +2,21 @@ Environment, create_aiohttp_env ) -from .integration_test_suite_factory import integration_test_suite_factory +from .client import ( + AgentClient, + ResponseClient, +) +from .integration import integration +from .integration_fixtures import IntegrationFixtures from .sample import Sample + __all__ = [ + "AgentClient", + "ResponseClient", "Environment", "create_aiohttp_env", - "integration_test_suite_factory", + "integration", + "IntegrationFixtures", "Sample", ] \ No newline at end of file diff --git a/dev/integration/src/core/client/bot_client.py b/dev/integration/src/core/client/bot_client.py deleted file mode 100644 index 60c0613f..00000000 --- a/dev/integration/src/core/client/bot_client.py +++ /dev/null @@ -1,181 +0,0 @@ -import asyncio -import json -from concurrent.futures import ThreadPoolExecutor -from typing import Dict, List, Optional -import aiohttp -from msal import ConfidentialClientApplication - -from microsoft_agents.activity import ( - Activity, - ActivityTypes, - DeliveryModes, -) - - -class BotClient: - """Python equivalent of the C# BotClient class for sending activities to agents.""" - - def __init__( - self, - messaging_endpoint: str, - service_endpoint: str, - cid: str, - client_id: str, - tenant_id: str, - client_secret: str - ): - self.messaging_endpoint = messaging_endpoint - self.service_endpoint = service_endpoint - self.cid = cid - self.client_id = client_id - self.tenant_id = tenant_id - self.client_secret = client_secret - - # Dictionary to track pending tasks (equivalent to C# ConcurrentDictionary) - self.task_list: Dict[str, asyncio.Future[List[Activity]]] = {} - - # MSAL app for authentication - self.msal_app = ConfidentialClientApplication( - client_id=client_id, - client_credential=client_secret, - authority=f"https://login.microsoftonline.com/{tenant_id}" - ) - - async def send_request(self, activity: Activity) -> str: - """Send an HTTP request with the activity to the messaging endpoint.""" - - # Acquire token for authentication - token_result = self.msal_app.acquire_token_for_client( - scopes=[f"{self.client_id}/.default"] - ) - - if "error" in token_result: - raise Exception(f"Failed to acquire token: {token_result['error_description']}") - - # Update activity properties - activity.service_url = self.service_endpoint - if activity.conversation: - activity.conversation.id = self.cid - - # Prepare the request - headers = { - "Authorization": f"Bearer {token_result['access_token']}", - "Content-Type": "application/json" - } - - # Serialize activity to JSON - activity_json = activity.model_dump(by_alias=True, exclude_none=True) - - async with aiohttp.ClientSession() as session: - async with session.post( - self.messaging_endpoint, - headers=headers, - json=activity_json - ) as response: - if not response.ok: - raise Exception(f"Failed to send activity: {response.status}") - - content = await response.text() - return content - - async def send_activity(self, activity: Activity) -> List[Activity]: - """Send an activity and wait for the response.""" - - # Create a future to track the response - future: asyncio.Future[List[Activity]] = asyncio.Future() - self.task_list[self.cid] = future - - try: - # Send activity to the agent - await self.send_request(activity) - - # Wait for response with timeout (20 seconds) - result = await asyncio.wait_for(future, timeout=20.0) - return result - finally: - # Clean up the task from the list - self.task_list.pop(self.cid, None) - - async def send_expect_replies_activity(self, activity: Activity) -> List[Activity]: - """Send an activity with delivery mode expect_replies.""" - - # Validate that the activity has the correct delivery mode - if activity.delivery_mode != DeliveryModes.expect_replies: - raise ValueError("Activity delivery mode must be expect_replies.") - - # Send activity to the agent - content = await self.send_request(activity) - - # Parse response - if not content: - raise ValueError("No response received from the agent.") - - response_data = json.loads(content) - activities_data = response_data.get("activities", []) - - # Convert JSON to Activity objects - activities = [Activity.model_validate(act_data) for act_data in activities_data] - return activities - - async def send_stream_activity(self, activity: Activity) -> List[Activity]: - """Send an activity with delivery mode stream.""" - - # Validate that the activity has the correct delivery mode - if activity.delivery_mode != DeliveryModes.stream: - raise ValueError("Activity delivery mode must be stream.") - - # Create a future to track the response - future: asyncio.Future[List[Activity]] = asyncio.Future() - self.task_list[self.cid] = future - - try: - # Send activity to the agent - content = await self.send_request(activity) - - # Check if we got an immediate response - if not content: - # Wait for streaming response - result = await asyncio.wait_for(future, timeout=6.0) - if result: - return result - raise ValueError("No response received from the agent.") - - # Parse server-sent events - lines = [line.strip() for line in content.split("\n\r\n") if line.strip()] - activities = [] - - for line in lines: - if "event: activity" in line: - # Extract the data part after "data: " - data_parts = line.split("data: ") - if len(data_parts) > 1: - activity_data = json.loads(data_parts[1]) - activity = Activity.model_validate(activity_data) - activities.append(activity) - else: - raise Exception("Must receive server-sent events") - - return activities - finally: - # Clean up the task from the list - self.task_list.pop(self.cid, None) - - async def send_invoke(self, activity: Activity) -> str: - """Send an invoke activity.""" - - # Validate that the activity is of type Invoke - if activity.type != ActivityTypes.invoke: - raise ValueError("Activity type must be invoke.") - - # Send activity to the agent - content = await self.send_request(activity) - return content - - def complete_task(self, conversation_id: str, activities: List[Activity]): - """Complete a pending task with the received activities. - - This method should be called by the webhook handler when activities are received. - """ - future = self.task_list.get(conversation_id) - if future and not future.done(): - future.set_result(activities) \ No newline at end of file diff --git a/dev/integration/src/mocks/__init__.py b/dev/integration/src/mocks/__init__.py new file mode 100644 index 00000000..e69de29b From c90c8e425b56fbecde4f398c7d23feadee4c872a Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Wed, 29 Oct 2025 08:46:43 -0700 Subject: [PATCH 11/81] Cleaning up implementations --- dev/integration/src/auto_client/__init__.py | 0 .../src/auto_client/auto_client.py | 4 - dev/integration/src/client/__init__.py | 0 dev/integration/src/client/agent_client.py | 0 .../src/client/listening_client.py | 4 - dev/integration/src/core/__init__.py | 8 +- dev/integration/src/core/agent_client.py | 51 ------ .../src/core/application_runner.py | 35 ++++ .../src/core/auto_client/__init__.py | 0 .../src/core/auto_client/auto_client.py | 18 -- .../src/core/client/agent_client.py | 41 ++--- .../src/core/client/auto_client.py | 18 ++ dev/integration/src/core/client/bot_client.cs | 172 ------------------ .../src/core/client/bot_response.cs | 141 -------------- .../core/client/response_client_protocol.py | 11 -- dev/integration/src/core/environment.py | 41 +++++ .../src/core/environment/__init__.py | 7 - .../core/environment/application_runner.py | 37 ---- .../core/environment/create_aiohttp_env.py | 58 ------ .../src/core/environment/environment.py | 25 --- dev/integration/src/environments/__init__.py | 5 + .../src/environments/aiohttp_environment.py | 71 ++++++++ dev/integration/src/mocks/__init__.py | 0 23 files changed, 185 insertions(+), 562 deletions(-) delete mode 100644 dev/integration/src/auto_client/__init__.py delete mode 100644 dev/integration/src/auto_client/auto_client.py delete mode 100644 dev/integration/src/client/__init__.py delete mode 100644 dev/integration/src/client/agent_client.py delete mode 100644 dev/integration/src/client/listening_client.py delete mode 100644 dev/integration/src/core/agent_client.py create mode 100644 dev/integration/src/core/application_runner.py delete mode 100644 dev/integration/src/core/auto_client/__init__.py delete mode 100644 dev/integration/src/core/auto_client/auto_client.py create mode 100644 dev/integration/src/core/client/auto_client.py delete mode 100644 dev/integration/src/core/client/bot_client.cs delete mode 100644 dev/integration/src/core/client/bot_response.cs delete mode 100644 dev/integration/src/core/client/response_client_protocol.py create mode 100644 dev/integration/src/core/environment.py delete mode 100644 dev/integration/src/core/environment/__init__.py delete mode 100644 dev/integration/src/core/environment/application_runner.py delete mode 100644 dev/integration/src/core/environment/create_aiohttp_env.py delete mode 100644 dev/integration/src/core/environment/environment.py create mode 100644 dev/integration/src/environments/__init__.py create mode 100644 dev/integration/src/environments/aiohttp_environment.py delete mode 100644 dev/integration/src/mocks/__init__.py diff --git a/dev/integration/src/auto_client/__init__.py b/dev/integration/src/auto_client/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/integration/src/auto_client/auto_client.py b/dev/integration/src/auto_client/auto_client.py deleted file mode 100644 index 8aee4368..00000000 --- a/dev/integration/src/auto_client/auto_client.py +++ /dev/null @@ -1,4 +0,0 @@ -from ..core import AgentClient - -class AutoClient(AgentClient): - pass \ No newline at end of file diff --git a/dev/integration/src/client/__init__.py b/dev/integration/src/client/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/integration/src/client/agent_client.py b/dev/integration/src/client/agent_client.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/integration/src/client/listening_client.py b/dev/integration/src/client/listening_client.py deleted file mode 100644 index ad71188b..00000000 --- a/dev/integration/src/client/listening_client.py +++ /dev/null @@ -1,4 +0,0 @@ -from .agent_client import AgentClient - -class ListeningClient(AgentClient): - pass \ No newline at end of file diff --git a/dev/integration/src/core/__init__.py b/dev/integration/src/core/__init__.py index a13fcd60..ff94868f 100644 --- a/dev/integration/src/core/__init__.py +++ b/dev/integration/src/core/__init__.py @@ -1,11 +1,9 @@ -from .environment import ( - Environment, - create_aiohttp_env -) +from .application_runner import ApplicationRunner from .client import ( AgentClient, ResponseClient, ) +from .environment import Environment from .integration import integration from .integration_fixtures import IntegrationFixtures from .sample import Sample @@ -13,9 +11,9 @@ __all__ = [ "AgentClient", + "ApplicationRunner", "ResponseClient", "Environment", - "create_aiohttp_env", "integration", "IntegrationFixtures", "Sample", diff --git a/dev/integration/src/core/agent_client.py b/dev/integration/src/core/agent_client.py deleted file mode 100644 index 5d73852a..00000000 --- a/dev/integration/src/core/agent_client.py +++ /dev/null @@ -1,51 +0,0 @@ -from http import client -import os - -from aiohttp import ClientSession - -from microsoft_agents.activity import ( - Activity, - ActivityTypes, -) - -from msal import ConfidentialClientApplication - -class AgentClient: - - def __init__(self, messaging_endpoint: str, service_endpoint: str, cid: str, client_id: str, tenant_id: str, client_secret: str): - self.messaging_endpoint = messaging_endpoint - self.service_endpoint = service_endpoint - self.cid = cid - self.client_id = client_id - self.tenant_id = tenant_id - self.client_secret = client_secret - - async def send_request(self, activity: Activity): - - client_id = os.environ["CLIENT_ID"] - - msal_app = ConfidentialClientApplication( - client_id=client_id, - tenant_id=os.environ["TENANT_ID"], - client_credential=os.environ["CLIENT_SECRET"], - ) - - token = msal_app.acquire_token_for_client([f"{client_id}/.default"]) - - session = ClientSession() - activity.service_url = self.service_endpoint - activ - - async def send_activity(self, activity: Activity): - pass - - async def send_expect_replies_activity(self, activity: Activity): - pass - - async def send_stream_activity(self, activity: Activity): - pass - - async def send_invoke(self, activity: Activity): - if activity.type != ActivityTypes.invoke: - raise ValueError("Activity type must be 'invoke' for send_invoke method.") - return await self.send_request(activity) \ No newline at end of file diff --git a/dev/integration/src/core/application_runner.py b/dev/integration/src/core/application_runner.py new file mode 100644 index 00000000..d6d73fb1 --- /dev/null +++ b/dev/integration/src/core/application_runner.py @@ -0,0 +1,35 @@ +from abc import ABC, abstractmethod +from typing import Any, Optional +from threading import Thread + +class ApplicationRunner(ABC): + """Base class for application runners.""" + + def __init__(self, app: Any): + self._app = app + self._thread: Optional[Thread] = None + + @abstractmethod + def _start_server(self) -> None: + raise NotImplementedError("Start server method must be implemented by subclasses") + + def _stop_server(self) -> None: + pass + + async def __aenter__(self) -> None: + + if self._thread: + raise RuntimeError("Server is already running") + + self._thread = Thread(target=self._start_server, daemon=True) + self._thread.start() + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + + if self._thread: + self._stop_server() + + self._thread.join() + self._thread = None + else: + raise RuntimeError("Server is not running") \ No newline at end of file diff --git a/dev/integration/src/core/auto_client/__init__.py b/dev/integration/src/core/auto_client/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/integration/src/core/auto_client/auto_client.py b/dev/integration/src/core/auto_client/auto_client.py deleted file mode 100644 index 8e423867..00000000 --- a/dev/integration/src/core/auto_client/auto_client.py +++ /dev/null @@ -1,18 +0,0 @@ -from microsoft_agents.activity import Activity - -from ..agent_client import AgentClient - -class AutoClient: - - def __init__(self, agent_client: AgentClient): - self._agent_client = agent_client - - async def generate_message(self) -> str: - pass - - async def run(self, max_turns: int = 10, time_between_turns: float = 2.0) -> None: - - for i in range(max_turns): - await self._agent_client.send_activity( - Activity(type="message", text=self.generate_message()) - ) \ No newline at end of file diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py index 0a86b8eb..9c6f8e20 100644 --- a/dev/integration/src/core/client/agent_client.py +++ b/dev/integration/src/core/client/agent_client.py @@ -1,8 +1,7 @@ import os import json import asyncio -from re import A -from typing import Any, Optional +from typing import Any, Optional, cast from aiohttp import ClientSession @@ -79,35 +78,19 @@ async def send_request(self, activity: Activity) -> str: raise Exception(f"Failed to send activity: {response.status}") content = await response.text() return content - - async def send_activity(self, activity: Activity, timeout: Optional[float] = None) -> list[Activity]: - timeout = timeout or self._default_timeout - - content = await self.send_request(activity) - return content - - async def send_expect_replies_activity(self, activity: Activity) -> list[Activity]: - if activity.delivery_mode != DeliveryModes.expect_replies: - raise ValueError("Activity delivery_mode must be 'expectReplies' for this method.") - content = await self.send_request(activity) - if not content: - raise RuntimeError("Expected replies but received no content.") - - activities_content = json.loads(content) - activities = [ - Activity.model_validate_json(json.dumps(act)) - for act in activities_content - ] - - return activities + async def send_activity(self, activity_or_text: Activity | str, timeout: Optional[float] = None) -> str: + timeout = timeout or self._default_timeout - async def send_stream_activity(self, activity: Activity) -> list[Activity]: - raise NotImplementedError("send_stream_activity is not implemented yet.") - - async def send_invoke(self, activity: Activity) -> str: - if activity.type != ActivityTypes.invoke: - raise ValueError("Activity type must be 'invoke' for send_invoke method.") + if isinstance(activity_or_text, str): + activity = Activity( + type=ActivityTypes.message, + # delivery_mode=DeliveryModes.expect_replies, + text=activity_or_text, + input_hint=input_hint or InputHints.accepting_input, + ) + else: + activity = cast(Activity, activity_or_text) content = await self.send_request(activity) return content \ No newline at end of file diff --git a/dev/integration/src/core/client/auto_client.py b/dev/integration/src/core/client/auto_client.py new file mode 100644 index 00000000..721994a6 --- /dev/null +++ b/dev/integration/src/core/client/auto_client.py @@ -0,0 +1,18 @@ +# from microsoft_agents.activity import Activity + +# from ..agent_client import AgentClient + +# class AutoClient: + +# def __init__(self, agent_client: AgentClient): +# self._agent_client = agent_client + +# async def generate_message(self) -> str: +# pass + +# async def run(self, max_turns: int = 10, time_between_turns: float = 2.0) -> None: + +# for i in range(max_turns): +# await self._agent_client.send_activity( +# Activity(type="message", text=self.generate_message()) +# ) \ No newline at end of file diff --git a/dev/integration/src/core/client/bot_client.cs b/dev/integration/src/core/client/bot_client.cs deleted file mode 100644 index 793e823e..00000000 --- a/dev/integration/src/core/client/bot_client.cs +++ /dev/null @@ -1,172 +0,0 @@ -using Microsoft.Agents.Core.Models; -using Microsoft.Agents.Core.Serialization; -using Microsoft.Identity.Client; -using System.Collections.Concurrent; -using System.Text.Json; - -namespace Framework -{ - public class BotClient(string messagingEndpoint, string serviceEndpoint, string cid, string clientId, string tenantId, string clientSecret) - { - public static ConcurrentDictionary> TaskList = new(); - - public async Task SendRequest(Activity activity) - { - - // Create bearer authentication token - IConfidentialClientApplication app = ConfidentialClientApplicationBuilder - .Create(clientId) - .WithTenantId(tenantId) - .WithClientSecret(clientSecret) - .Build(); - - var token = await app.AcquireTokenForClient([clientId + "/.default"]).ExecuteAsync(); - - // Send request to agent - HttpClient http = new(); - - // Update activity to send to service endpoint - activity.ServiceUrl = serviceEndpoint; - - // Update activity conversation ID - activity.Conversation.Id = cid; - - var stringContent = new StringContent(activity.ToJson(), System.Text.Encoding.UTF8, "application/json"); - - HttpRequestMessage req = new(HttpMethod.Post, messagingEndpoint) - { - Headers = - { - { "Authorization", "Bearer " + token.AccessToken } - }, - Content = stringContent - - }; - - var resp = await http.SendAsync(req); - - // Check if request was successful - if (!resp.IsSuccessStatusCode) - { - throw new Exception("Failed to send activity: " + resp.StatusCode); - } - - var content = await resp.Content.ReadAsStringAsync(); - return content; - } - public async Task SendActivity(Activity activity) - { - - // Keep track of activities being sent to the web service - TaskCompletionSource tcs = new(); - - TaskList.TryAdd(cid, tcs); - - // Send activity to the agent - var content = await SendRequest(activity); - - // Receive response from the agent - var result = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(20)); - TaskList.TryRemove(cid, out _); - - return result; - } - - public async Task SendExpectRepliesActivity(Activity activity) - { - // Validate that the activity is of delivery mode expected replies - if (activity.DeliveryMode != DeliveryModes.ExpectReplies) - { - throw new InvalidOperationException("Activity type must be ExpectedReplies."); - } - - // Keep track of activities being sent to the web service - TaskCompletionSource tcs = new(); - - // Send activity to the agent - var content = await SendRequest(activity); - - // Receive response from the agent - if (content.Length == 0) - { - throw new InvalidOperationException("No response received from the agent."); - } - JsonDocument jsonDoc = JsonDocument.Parse(content); - JsonElement activitiesJson = jsonDoc.RootElement.GetProperty("activities"); - var activities = ProtocolJsonSerializer.ToObject(activitiesJson); - - return activities; - - } - - public async Task SendStreamActivity(Activity activity) - { - // Validate that the activity is of type Stream - if (activity.DeliveryMode != DeliveryModes.Stream) - { - throw new InvalidOperationException("Activity type must be Stream."); - } - - // Keep track of activities being sent to the web service - TaskCompletionSource tcs = new(); - - TaskList.TryAdd(cid, tcs); - - // Send activity to the agent - var content = await SendRequest(activity); - - // Check if error comes back - if (content.Length == 0) { - var result = await tcs.Task.WaitAsync(TimeSpan.FromSeconds(6)); - if (result.Length > 0) - { - return result; - } - } - - // Receive response from the agent - if (content.Length == 0) - { - throw new InvalidOperationException("No response received from the agent."); - } - - var split = content.Split("\n\r\n"); - split = split.Where(s => !string.IsNullOrWhiteSpace(s)).ToArray(); - - Activity[] activities = new Activity[split.Length]; - - for (int i = 0; i < split.Length; i++) - { - if (split[i].Contains("event: activity")) - { - var format = split[i].Split("data: "); - var act = ProtocolJsonSerializer.ToObject(format[1]); - activities[i] = act; - } else - { - throw new Exception("Must receive server-sent events"); - } - } - - TaskList.TryRemove(cid, out _); - - return activities; - } - - - public async Task SendInvoke(Activity activity) - { - - // Validate that the activity is of type Invoke - if (activity.Type != ActivityTypes.Invoke) - { - throw new InvalidOperationException("Activity type must be Invoke."); - } - - // Send activity to the agent - var content = await SendRequest(activity); - - return content; - } - } -} \ No newline at end of file diff --git a/dev/integration/src/core/client/bot_response.cs b/dev/integration/src/core/client/bot_response.cs deleted file mode 100644 index 460942bb..00000000 --- a/dev/integration/src/core/client/bot_response.cs +++ /dev/null @@ -1,141 +0,0 @@ -using Microsoft.Agents.Core.Models; -using Microsoft.Agents.Core.Serialization; -using Microsoft.AspNetCore.Authentication.JwtBearer; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Identity.Web; -using System.Collections.Concurrent; -using System.Security.Cryptography; -using System.Text.Json; - -namespace Framework -{ - public sealed class BotResponse : IAsyncDisposable - { - readonly WebApplication _app; - ConcurrentDictionary> _multipleActivities = new(); - public string TestId { get; } = Guid.NewGuid().ToString(); - public BotResponse() - { - // Suppress console output - Console.SetOut(TextWriter.Null); - - // Create a web application builder and configure services - WebApplicationBuilder builder = WebApplication.CreateBuilder(); - - builder.Services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme) - .AddMicrosoftIdentityWebApi(builder.Configuration); - - // Add services to the web application - _app = builder.Build(); - _app.UseRouting(); - _app.MapPost("/v3/conversations/{*text}", async (ctx) => - { - - using StreamReader reader = new(ctx.Request.Body); - var resp = await reader.ReadToEndAsync(); - Activity act = ProtocolJsonSerializer.ToObject(resp); - string cid = act.Conversation.Id; - var activityList = _multipleActivities.GetOrAdd(cid!, _ => new List()); - - lock (activityList) - { - activityList.Add(act); - } - - var response = new - { - Id = Guid.NewGuid().ToString() - }; - - // Check if the activity is a streamed activity - if (act.Entities?.Any(e => e.Type == EntityTypes.StreamInfo) == true) - { - var entities = ProtocolJsonSerializer.ToJson(act.Entities[0]); - var sact = ProtocolJsonSerializer.ToObject(entities); - - bool handled = HandleStreamedActivity(act, sact, cid); - - ctx.Response.StatusCode = 200; - ctx.Response.ContentType = "application/json"; - await ctx.Response.WriteAsync(JsonSerializer.Serialize(response)); - - if (BotClient.TaskList.TryGetValue(cid!, out var tcs) && handled) - { - if (_multipleActivities.TryGetValue(cid, out var streamedActivity)) - { - tcs!.TrySetResult(streamedActivity.ToArray()); - _multipleActivities.TryRemove(cid, out _); - } - } - } - else - { - - if (act.Type != ActivityTypes.Typing) - { - _ = Task.Run(async () => - { - await Task.Delay(5000); - if (BotClient.TaskList.TryGetValue(cid!, out var tcs)) - { - _multipleActivities.TryGetValue(cid!, out var result); - tcs?.TrySetResult(result!.ToArray()); - } - }); - } - ctx.Response.StatusCode = 200; - ctx.Response.ContentType = "application/json"; - await ctx.Response.WriteAsync(JsonSerializer.Serialize(response)); - } - }); - - ServiceEndpoint = "http://localhost:9873"; - - _app.UseAuthentication(); - _app.UseAuthorization(); - _app.RunAsync(ServiceEndpoint); - } - - private bool HandleStreamedActivity(Activity act, StreamInfo sact, string cid) - { - - // Check if activity is the final message - if (sact.StreamType == StreamTypes.Final) - { - if (act.Type == ActivityTypes.Message) - { - return true; - } - else - { - throw new Exception("final streamed activity should be type message"); - } - } - - // Handler for streaming types which allows us to verify later if the text length has increased - else if (sact.StreamType == StreamTypes.Streaming) - { - if (sact.StreamSequence <= 0 && act.Type == ActivityTypes.Typing) - { - throw new Exception("streamed activity's stream sequence should be a positive number"); - } - } - // Activity is being streamed but isn't the final message - return false; - - } - - public async ValueTask DisposeAsync() - { - await _app.StopAsync(); - await _app.DisposeAsync(); - } - - public string ServiceEndpoint { get; private set; } - - } - -} \ No newline at end of file diff --git a/dev/integration/src/core/client/response_client_protocol.py b/dev/integration/src/core/client/response_client_protocol.py deleted file mode 100644 index d104e492..00000000 --- a/dev/integration/src/core/client/response_client_protocol.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations - -from typing import Protocol - -class ResponseClientProtocol(Protocol): - - async def __aenter__(self) -> ResponseClientProtocol: - ... - - async def __aexit__(self, exc_type, exc, tb) -> None: - ... \ No newline at end of file diff --git a/dev/integration/src/core/environment.py b/dev/integration/src/core/environment.py new file mode 100644 index 00000000..1472c7c7 --- /dev/null +++ b/dev/integration/src/core/environment.py @@ -0,0 +1,41 @@ +from abc import ABC, abstractmethod +from typing import Awaitable, Callable + +from microsoft_agents.hosting.core import ( + AgentApplication, + ChannelAdapter, + Connections, + Authorization, + Storage, + TurnState, +) + +from .application_runner import ApplicationRunner + +class Environment(ABC): + """A sample data object for integration tests.""" + + agent_application: AgentApplication[TurnState] + storage: Storage + adapter: ChannelAdapter + connections: Connections + authorization: Authorization + + config: dict + + driver: Callable[[], Awaitable[None]] + + @abstractmethod + async def init_env(self) -> None: + """Initialize the environment.""" + raise NotImplementedError() + + @abstractmethod + def create_runner(self) -> ApplicationRunner: + """Create an application runner for the environment.""" + raise NotImplementedError() + + @abstractmethod + def create_app(self): + """Create the application for the environment.""" + raise NotImplementedError() \ No newline at end of file diff --git a/dev/integration/src/core/environment/__init__.py b/dev/integration/src/core/environment/__init__.py deleted file mode 100644 index b2deb639..00000000 --- a/dev/integration/src/core/environment/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .create_aiohttp_env import create_aiohttp_env -from .environment import Environment - -__all__ = [ - "create_aiohttp_env", - "Environment" -] \ No newline at end of file diff --git a/dev/integration/src/core/environment/application_runner.py b/dev/integration/src/core/environment/application_runner.py deleted file mode 100644 index 2149e976..00000000 --- a/dev/integration/src/core/environment/application_runner.py +++ /dev/null @@ -1,37 +0,0 @@ -from typing import TypeVar -from threading import Thread - -import aiohttp.web - -AppT = TypeVar('AppT', bound=aiohttp.web.Application) - -class ApplicationRunner: - def __init__(self, app: AppT): - self._app = app - self._thread = None - - async def _start_server(self) -> None: - runner = aiohttp.web.AppRunner(self._app) - await runner.setup() - site = aiohttp.web.TCPSite(runner, 'localhost', 8080) - await site.start() - - async def __aenter__(self) -> None: - - if self._thread: - raise RuntimeError("Server is already running") - - self._thread = Thread(target=self._start_server, daemon=True) - self._thread.start() - - async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: - - if self._thread: - - if isinstance(self._app, aiohttp.web.Application): - await self._app.shutdown() - - self._thread.join() - self._thread = None - else: - raise RuntimeError("Server is not running") \ No newline at end of file diff --git a/dev/integration/src/core/environment/create_aiohttp_env.py b/dev/integration/src/core/environment/create_aiohttp_env.py deleted file mode 100644 index 429a1e05..00000000 --- a/dev/integration/src/core/environment/create_aiohttp_env.py +++ /dev/null @@ -1,58 +0,0 @@ -from typing import Optional - -from click import Option -from microsoft_agents.hosting.aiohttp import CloudAdapter -from microsoft_agents.hosting.core import ( - Authorization, - AgentApplication, - TurnState, - TurnContext, - MemoryStorage, -) -from microsoft_agents.authentication.msal import MsalConnectionManager -from microsoft_agents.activity import load_configuration_from_env - -from .environment import Environment - -def start_server() -> None: - import asyncio - from threading import Thread - from contextlib import asynccontextmanager - from microsoft_agents.hosting.aiohttp import host_app - -@asynccontextmanager -def aiohttp_runner(timeout=10.0) -> None: - - thread = Thread(target=start_server) - thread.start() - - yield - - thread.join(timeout=timeout) - -def create_aiohttp_env(environ_dict: Optional[dict] = None) -> Environment: - - environ_dict = environ_dict or {} - - agents_sdk_config = load_configuration_from_env(environ_dict) - - storage = MemoryStorage() - connection_manager = MsalConnectionManager(**agents_sdk_config) - adapter = CloudAdapter(connection_manager=connection_manager) - authorization = Authorization(storage, connection_manager, **agents_sdk_config) - - agent_application = AgentApplication[TurnState]( - storage=storage, - adapter=adapter, - authorization=authorization, - **agents_sdk_config - ) - - return Environment( - agent_application=agent_application, - storage=storage, - connections=connection_manager, - adapter=adapter, - authorization=authorization, - config=agents_sdk_config - ) \ No newline at end of file diff --git a/dev/integration/src/core/environment/environment.py b/dev/integration/src/core/environment/environment.py deleted file mode 100644 index 1b06215e..00000000 --- a/dev/integration/src/core/environment/environment.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Awaitable, Callable -from dataclasses import dataclass - -from microsoft_agents.hosting.core import ( - AgentApplication, - ChannelAdapter, - Connections, - Authorization, - Storage, - TurnState, -) - -@dataclass -class Environment: - """A sample data object for integration tests.""" - - agent_application: AgentApplication[TurnState] - storage: Storage - adapter: ChannelAdapter - connections: Connections - authorization: Authorization - - config: dict - - driver: Callable[[], Awaitable[None]] \ No newline at end of file diff --git a/dev/integration/src/environments/__init__.py b/dev/integration/src/environments/__init__.py new file mode 100644 index 00000000..b71cc2b4 --- /dev/null +++ b/dev/integration/src/environments/__init__.py @@ -0,0 +1,5 @@ +from .sdk_sample_environment import SDKSampleEnvironment + +__all__ = [ + "SDKSampleEnvironment" +] \ No newline at end of file diff --git a/dev/integration/src/environments/aiohttp_environment.py b/dev/integration/src/environments/aiohttp_environment.py new file mode 100644 index 00000000..27255d88 --- /dev/null +++ b/dev/integration/src/environments/aiohttp_environment.py @@ -0,0 +1,71 @@ + +from aiohttp.web import Request, Response, Application, run_app + +from microsoft_agents.hosting.aiohttp import ( + CloudAdapter, + jwt_authorization_middleware, + start_agent_process +) +from microsoft_agents.hosting.core import ( + Authorization, + AgentApplication, + TurnState, + TurnContext, + MemoryStorage, +) +from microsoft_agents.authentication.msal import MsalConnectionManager +from microsoft_agents.activity import load_configuration_from_env + +from .application_runner import ApplicationRunner +from .environment import Environment + +class AiohttpRunner(ApplicationRunner): + """A runner for aiohttp applications.""" + + def _start_server(self) -> None: + try: + assert isinstance(self._app, Application) + run_app(self._app, host="localhost", port=3978) + except Exception as error: + raise error + + def _stop_server(self) -> None: + pass + +class AiohttpEnvironment(Environment): + + async def init_env(self, environ_dict: dict) -> None: + environ_dict = environ_dict or {} + + agents_sdk_config = load_configuration_from_env(environ_dict) + + storage = MemoryStorage() + connection_manager = MsalConnectionManager(**agents_sdk_config) + adapter = CloudAdapter(connection_manager=connection_manager) + authorization = Authorization(storage, connection_manager, **agents_sdk_config) + + agent_application = AgentApplication[TurnState]( + storage=storage, + adapter=adapter, + authorization=authorization, + **agents_sdk_config + ) + + def create_runner(self) -> ApplicationRunner: + + async def entry_point(req: Request) -> Response: + agent: AgentApplication = req.app["agent_app"] + adapter: CloudAdapter = req.app["adapter"] + return await start_agent_process( + req, + agent, + adapter + ) + + APP = Application(middlewares=[jwt_authorization_middleware]) + APP.router.add_post("/api/messages", entry_point) + APP["agent_configuration"] = self.connections.get_default_connection() + APP["agent_app"] = self.agent_application + APP["adapter"] = self.adapter + + return ApplicationRunner(APP) \ No newline at end of file diff --git a/dev/integration/src/mocks/__init__.py b/dev/integration/src/mocks/__init__.py deleted file mode 100644 index e69de29b..00000000 From 36f5c3b8fea20556823f1955891340b14ba6055f Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Wed, 29 Oct 2025 14:28:18 -0700 Subject: [PATCH 12/81] Adding expect replies sending method --- .../src/core/client/agent_client.py | 27 ++++-- dev/integration/src/core/environment.py | 9 +- dev/integration/src/core/integration.py | 95 +++++++++++-------- dev/integration/src/environments/__init__.py | 4 +- .../src/environments/aiohttp_environment.py | 37 ++++---- dev/integration/src/tests/test_quickstart.py | 51 ++++++---- 6 files changed, 126 insertions(+), 97 deletions(-) diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py index 9c6f8e20..237a4f7d 100644 --- a/dev/integration/src/core/client/agent_client.py +++ b/dev/integration/src/core/client/agent_client.py @@ -78,19 +78,30 @@ async def send_request(self, activity: Activity) -> str: raise Exception(f"Failed to send activity: {response.status}") content = await response.text() return content - - async def send_activity(self, activity_or_text: Activity | str, timeout: Optional[float] = None) -> str: - timeout = timeout or self._default_timeout - + + def _to_activity(self, activity_or_text: Activity | str) -> Activity: if isinstance(activity_or_text, str): activity = Activity( type=ActivityTypes.message, - # delivery_mode=DeliveryModes.expect_replies, text=activity_or_text, - input_hint=input_hint or InputHints.accepting_input, ) + return activity else: - activity = cast(Activity, activity_or_text) + return cast(Activity, activity_or_text) + async def send_activity(self, activity_or_text: Activity | str, timeout: Optional[float] = None) -> str: + timeout = timeout or self._default_timeout + activity = self._to_activity(activity_or_text) content = await self.send_request(activity) - return content \ No newline at end of file + return content + + async def send_expect_replies(self, activity_or_text: Activity | str, timeout: Optional[float] = None) -> list[Activity]: + timeout = timeout or self._default_timeout + activity = self._to_activity(activity_or_text) + activity.delivery_mode = DeliveryModes.expect_replies + + content = await self.send_request(activity) + + activities_data = json.loads(content).get("activities", []) + activities = [Activity.model_validate(act) for act in activities_data] + return activities \ No newline at end of file diff --git a/dev/integration/src/core/environment.py b/dev/integration/src/core/environment.py index 1472c7c7..628460a6 100644 --- a/dev/integration/src/core/environment.py +++ b/dev/integration/src/core/environment.py @@ -18,7 +18,7 @@ class Environment(ABC): agent_application: AgentApplication[TurnState] storage: Storage adapter: ChannelAdapter - connections: Connections + connection_manager: Connections authorization: Authorization config: dict @@ -26,16 +26,11 @@ class Environment(ABC): driver: Callable[[], Awaitable[None]] @abstractmethod - async def init_env(self) -> None: + async def init_env(self, environ_config: dict) -> None: """Initialize the environment.""" raise NotImplementedError() @abstractmethod def create_runner(self) -> ApplicationRunner: """Create an application runner for the environment.""" - raise NotImplementedError() - - @abstractmethod - def create_app(self): - """Create the application for the environment.""" raise NotImplementedError() \ No newline at end of file diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index 4fb56670..76659021 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -1,7 +1,4 @@ -from random import sample -import pytest - -from typing import Optional, TypeVar, Union +from typing import Optional, TypeVar, Union, Callable, Any import aiohttp.web @@ -16,7 +13,7 @@ async def start_response_server(): pass -class Integration: +class _Integration: @staticmethod def _with_response_server(target_cls: T, host_response: bool) -> T: @@ -44,73 +41,83 @@ async def teardown_method(self): return target_cls @staticmethod - def from_service_url(target_cls: T, service_url: str, host_response: bool = False) -> T: + def from_service_url(service_url: str, host_response: bool = False) -> Callable[[T], T]: """Creates an Integration instance using a service URL.""" - async def setup_method(self): - self._service_url = service_url + def decorator(target_cls: T) -> T: - async def teardown_method(self): - self._service_url = service_url + async def setup_method(self): + self._service_url = service_url - target_cls.setup_method = setup_method - target_cls.teardown_method = teardown_method + async def teardown_method(self): + self._service_url = service_url - target_cls = Integration._with_response_server(target_cls, host_response) + target_cls.setup_method = setup_method + target_cls.teardown_method = teardown_method - return target_cls + target_cls = Integration._with_response_server(target_cls, host_response) + + return target_cls + + return decorator @staticmethod def from_sample( - target_cls: T, sample_cls: type[Sample], environment_cls: type[Environment], host_agent: bool = False, host_response: bool = False - ) -> T: + ) -> Callable[[T], T]: """Creates an Integration instance using a sample and environment.""" - def setup_method(self): - self._environment = environment_cls(sample_cls.get_config()) - await self._environment.__aenter__() + def decorator(target_cls: T) -> T: - self._sample = sample_cls(self._environment) - await self._sample.__aenter__() + def setup_method(self): + self._environment = environment_cls(sample_cls.get_config()) + await self._environment.__aenter__() - def teardown_method(self): - await self._sample.__aexit__(None, None, None) - await self._environment.__aexit__(None, None, None) + self._sample = sample_cls(self._environment) + await self._sample.__aenter__() - target_cls = Integration._with_response_server(target_cls, host_response) + def teardown_method(self): + await self._sample.__aexit__(None, None, None) + await self._environment.__aexit__(None, None, None) - return target_cls + target_cls = Integration._with_response_server(target_cls, host_response) + + return target_cls + + return decorator @staticmethod - def from_app(target_cls: T, app: AppT, host_response: bool = True) -> T: + def from_app(app: Any, host_response: bool = True) -> Callable[[T], T]: """Creates an Integration instance using an aiohttp application.""" - async def setup_method(self): + def decorator(target_cls: T) -> T: - self._app = app - self._runner = AppRunner(self._app) - await self._runner.__aenter__() + async def setup_method(self): - async def teardown_method(self): - await self._runner.__aexit__(None, None, None) + self._app = app + self._runner = AppRunner(self._app) + await self._runner.__aenter__() - target_cls = Integration._with_response_server(target_cls, host_response) + async def teardown_method(self): + await self._runner.__aexit__(None, None, None) - return target_cls + target_cls = Integration._with_response_server(target_cls, host_response) + + return target_cls + + return decorator def integration( - cls: T, service_url: Optional[str] = None, sample_cls: Optional[type[Sample]] = None, environment_cls: Optional[type[Environment]] = None, app: Optional[AppT] = None, host_agent: bool = False, host_response: bool = True, -) -> T: +) -> Callable[[T], T]: """Factory function to create an Integration instance based on provided parameters. Essentially resolves to one of the static methods of Integration: @@ -129,12 +136,16 @@ def integration( :param app: Optional aiohttp application instance. :return: An instance of the Integration class. """ - + + decorator: Callable[[T], T] + if service_url: - return Integration.from_service_url(cls, service_url, host_response=host_response) + decorator = _Integration.from_service_url(service_url, host_response=host_response) elif sample_cls and environment_cls: - return Integration.from_sample(cls, sample_cls, environment_cls, host_agent=host_agent, host_response=host_response) + decorator = _Integration.from_sample(sample_cls, environment_cls, host_agent=host_agent, host_response=host_response) elif app: - return Integration.from_app(cls, app, host_response=host_response) + decorator = _Integration.from_app(app, host_response=host_response) else: - raise ValueError("Insufficient parameters to create Integration instance.") \ No newline at end of file + raise ValueError("Insufficient parameters to create Integration instance.") + + return decorator \ No newline at end of file diff --git a/dev/integration/src/environments/__init__.py b/dev/integration/src/environments/__init__.py index b71cc2b4..db599f97 100644 --- a/dev/integration/src/environments/__init__.py +++ b/dev/integration/src/environments/__init__.py @@ -1,5 +1,5 @@ -from .sdk_sample_environment import SDKSampleEnvironment +from .aiohttp_environment import AiohttpEnvironment __all__ = [ - "SDKSampleEnvironment" + "AiohttpEnvironment" ] \ No newline at end of file diff --git a/dev/integration/src/environments/aiohttp_environment.py b/dev/integration/src/environments/aiohttp_environment.py index 27255d88..7525c9ea 100644 --- a/dev/integration/src/environments/aiohttp_environment.py +++ b/dev/integration/src/environments/aiohttp_environment.py @@ -1,4 +1,5 @@ +from tkinter import E from aiohttp.web import Request, Response, Application, run_app from microsoft_agents.hosting.aiohttp import ( @@ -10,14 +11,15 @@ Authorization, AgentApplication, TurnState, - TurnContext, MemoryStorage, ) from microsoft_agents.authentication.msal import MsalConnectionManager from microsoft_agents.activity import load_configuration_from_env -from .application_runner import ApplicationRunner -from .environment import Environment +from ..core import ( + ApplicationRunner, + Environment +) class AiohttpRunner(ApplicationRunner): """A runner for aiohttp applications.""" @@ -33,22 +35,23 @@ def _stop_server(self) -> None: pass class AiohttpEnvironment(Environment): + """An environment for aiohttp-hosted agents.""" - async def init_env(self, environ_dict: dict) -> None: - environ_dict = environ_dict or {} + async def init_env(self, environ_config: dict) -> None: + environ_config = environ_config or {} - agents_sdk_config = load_configuration_from_env(environ_dict) + self.config = load_configuration_from_env(environ_config) - storage = MemoryStorage() - connection_manager = MsalConnectionManager(**agents_sdk_config) - adapter = CloudAdapter(connection_manager=connection_manager) - authorization = Authorization(storage, connection_manager, **agents_sdk_config) + self.storage = MemoryStorage() + self.connection_manager = MsalConnectionManager(**self.config) + self.adapter = CloudAdapter(connection_manager=self.connection_manager) + self.authorization = Authorization(self.storage, self.connection_manager, **self.config) - agent_application = AgentApplication[TurnState]( - storage=storage, - adapter=adapter, - authorization=authorization, - **agents_sdk_config + self.agent_application = AgentApplication[TurnState]( + storage=self.storage, + adapter=self.adapter, + authorization=self.authorization, + **self.agents_sdk_config ) def create_runner(self) -> ApplicationRunner: @@ -64,8 +67,8 @@ async def entry_point(req: Request) -> Response: APP = Application(middlewares=[jwt_authorization_middleware]) APP.router.add_post("/api/messages", entry_point) - APP["agent_configuration"] = self.connections.get_default_connection() + APP["agent_configuration"] = self.connection_manager.get_default_connection() APP["agent_app"] = self.agent_application APP["adapter"] = self.adapter - return ApplicationRunner(APP) \ No newline at end of file + return AiohttpRunner(APP) \ No newline at end of file diff --git a/dev/integration/src/tests/test_quickstart.py b/dev/integration/src/tests/test_quickstart.py index 3fc72a0c..04a28946 100644 --- a/dev/integration/src/tests/test_quickstart.py +++ b/dev/integration/src/tests/test_quickstart.py @@ -1,34 +1,43 @@ import pytest +import asyncio -from ..core import integration_test_suite_factory, integration -from ..samples import QuickstartSample +from ..core import integration -TestSuiteBase = integration_test_suite_factory(QuickstartSample) -class TestQuickstart(TestSuiteBase): - - @pytest.mark.asyncio - async def test_quickstart_functionality(self): - pass - -@integration(QuickstartSample, None) # env +@integration(service_url="http://localhost:3978/api/messages") class TestQuickstart: @pytest.mark.asyncio - async def test_hello(self, agent_client, env): - agent_client.send("hi") + async def test_quickstart_functionality(self, agent_client, response_client): + await agent_client.send("hi") + await asyncio.sleep(2) + response = (await response_client.pop())[0] + assert "hello" in response.text.lower() + +# class TestQuickstart(TestSuiteBase): + +# @pytest.mark.asyncio +# async def test_quickstart_functionality(self): +# pass + +# @integration(QuickstartSample, None) # env +# class TestQuickstart: - await asyncio.sleep(1) +# @pytest.mark.asyncio +# async def test_hello(self, agent_client, env): +# agent_client.send("hi") - # assert env.auth... +# await asyncio.sleep(1) -@integration(app=None) # (endpoint="alternative") -class TestQuickstartAlternative: +# # assert env.auth... + +# @integration(app=None) # (endpoint="alternative") +# class TestQuickstartAlternative: - @pytest.mark.asyncio - async def test_hello(self, agent_client, response_client): +# @pytest.mark.asyncio +# async def test_hello(self, agent_client, response_client): - agent_client.send("hi") - await asyncio.sleep(10) +# agent_client.send("hi") +# await asyncio.sleep(10) - assert receiver.has_activity("hello") \ No newline at end of file +# assert receiver.has_activity("hello") \ No newline at end of file From dfdd959052f49cdb5050ac439c8f0cc426452e1a Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 30 Oct 2025 15:04:22 -0700 Subject: [PATCH 13/81] Beginning unit tests --- dev/integration/integration_tests/__init__.py | 0 .../test_quickstart.py | 0 dev/integration/src/core/integration.py | 57 +++++++++---------- dev/integration/src/tests/core/__init__.py | 0 .../src/tests/core/client/__init__.py | 0 .../tests/core/client/test_agent_client.py | 18 ++++++ .../tests/core/client/test_response_client.py | 19 +++++++ .../src/tests/core/test_application_runner.py | 49 ++++++++++++++++ .../core/test_integration_from_sample.py | 6 ++ .../core/test_integration_from_service_url.py | 7 +++ .../src/tests/environments/__init__.py | 0 dev/integration/src/tests/samples/__init__.py | 0 12 files changed, 127 insertions(+), 29 deletions(-) create mode 100644 dev/integration/integration_tests/__init__.py rename dev/integration/{src/tests => integration_tests}/test_quickstart.py (100%) create mode 100644 dev/integration/src/tests/core/__init__.py create mode 100644 dev/integration/src/tests/core/client/__init__.py create mode 100644 dev/integration/src/tests/core/client/test_agent_client.py create mode 100644 dev/integration/src/tests/core/client/test_response_client.py create mode 100644 dev/integration/src/tests/core/test_application_runner.py create mode 100644 dev/integration/src/tests/core/test_integration_from_sample.py create mode 100644 dev/integration/src/tests/core/test_integration_from_service_url.py create mode 100644 dev/integration/src/tests/environments/__init__.py create mode 100644 dev/integration/src/tests/samples/__init__.py diff --git a/dev/integration/integration_tests/__init__.py b/dev/integration/integration_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/test_quickstart.py b/dev/integration/integration_tests/test_quickstart.py similarity index 100% rename from dev/integration/src/tests/test_quickstart.py rename to dev/integration/integration_tests/test_quickstart.py diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index 76659021..87f21c5e 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -2,36 +2,33 @@ import aiohttp.web -from .runner import AppRunner +from .application_runner import ApplicationRunner from .environment import Environment -from .response_server import ResponseServer +from .client import AgentClient, ResponseClient from .sample import Sample T = TypeVar("T", bound=type) AppT = TypeVar("AppT", bound=aiohttp.web.Application) # for future extension w/ Union -async def start_response_server(): - pass - class _Integration: @staticmethod - def _with_response_server(target_cls: T, host_response: bool) -> T: - """Wraps the target class to include a response server if needed.""" + def _with_response_client(target_cls: T, host_response: bool) -> T: + """Wraps the target class to include a response client if needed.""" _prev_setup_method = getattr(target_cls, "setup_method", None) _prev_teardown_method = getattr(target_cls, "teardown_method", None) async def setup_method(self): if host_response: - self._response_server = ResponseServer() - await self._response_server.__aenter__() + self._response_client = ResponseClient() + await self._response_client.__aenter__() if _prev_setup_method: await _prev_setup_method(self) async def teardown_method(self): if host_response: - await self._response_server.__aexit__(None, None, None) + await self._response_client.__aexit__(None, None, None) if _prev_teardown_method: await _prev_teardown_method(self) @@ -55,7 +52,7 @@ async def teardown_method(self): target_cls.setup_method = setup_method target_cls.teardown_method = teardown_method - target_cls = Integration._with_response_server(target_cls, host_response) + target_cls = _Integration._with_response_client(target_cls, host_response) return target_cls @@ -72,43 +69,45 @@ def from_sample( def decorator(target_cls: T) -> T: - def setup_method(self): + async def setup_method(self): self._environment = environment_cls(sample_cls.get_config()) await self._environment.__aenter__() self._sample = sample_cls(self._environment) await self._sample.__aenter__() - def teardown_method(self): + async def teardown_method(self): await self._sample.__aexit__(None, None, None) await self._environment.__aexit__(None, None, None) - target_cls = Integration._with_response_server(target_cls, host_response) + target_cls = _Integration._with_response_client(target_cls, host_response) return target_cls return decorator - @staticmethod - def from_app(app: Any, host_response: bool = True) -> Callable[[T], T]: - """Creates an Integration instance using an aiohttp application.""" + # not supported yet + # @staticmethod + # def from_app(app: Any, host_response: bool = True) -> Callable[[T], T]: + # """Creates an Integration instance using an aiohttp application.""" - def decorator(target_cls: T) -> T: + # def decorator(target_cls: T) -> T: - async def setup_method(self): + # async def setup_method(self): - self._app = app - self._runner = AppRunner(self._app) - await self._runner.__aenter__() + # self._app = app - async def teardown_method(self): - await self._runner.__aexit__(None, None, None) + # self._runner = self._environment.create_runner() + # await self._runner.__aenter__() - target_cls = Integration._with_response_server(target_cls, host_response) + # async def teardown_method(self): + # await self._runner.__aexit__(None, None, None) - return target_cls + # target_cls = _Integration._with_response_client(target_cls, host_response) + + # return target_cls - return decorator + # return decorator def integration( service_url: Optional[str] = None, @@ -143,8 +142,8 @@ def integration( decorator = _Integration.from_service_url(service_url, host_response=host_response) elif sample_cls and environment_cls: decorator = _Integration.from_sample(sample_cls, environment_cls, host_agent=host_agent, host_response=host_response) - elif app: - decorator = _Integration.from_app(app, host_response=host_response) + # elif app: + # decorator = _Integration.from_app(app, host_response=host_response) else: raise ValueError("Insufficient parameters to create Integration instance.") diff --git a/dev/integration/src/tests/core/__init__.py b/dev/integration/src/tests/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/core/client/__init__.py b/dev/integration/src/tests/core/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/core/client/test_agent_client.py b/dev/integration/src/tests/core/client/test_agent_client.py new file mode 100644 index 00000000..d4d363b4 --- /dev/null +++ b/dev/integration/src/tests/core/client/test_agent_client.py @@ -0,0 +1,18 @@ +import pytest +from aioresponses import aioresponses + +from src.core import AgentClient + +class TestAgentClient: + + @pytest.fixture + def agent_client(self) -> AgentClient: + return AgentClient(base_url="") + + @pytest.fixture + async def service_url(self): + with aioresponses() as mocked: + mocked.get("https://example.com/service-url", payload={"serviceUrl": "https://service.example.com"}) + client = AgentClient(base_url="https://example.com") + service_url = await client.get_service_url() + yield service_url \ No newline at end of file diff --git a/dev/integration/src/tests/core/client/test_response_client.py b/dev/integration/src/tests/core/client/test_response_client.py new file mode 100644 index 00000000..81794474 --- /dev/null +++ b/dev/integration/src/tests/core/client/test_response_client.py @@ -0,0 +1,19 @@ +from re import A +import pytest +from aioresponses import aioresponses + +from src.core import ResponseClient + +class TestResponseClient: + + @pytest.fixture + def response_client(self): + return ResponseClient(base_url="") + + @pytest.fixture + def service_url(self): + with aioresponses() as mocked: + mocked.get("https://example.com/service-url", payload={"serviceUrl": "https://service.example.com"}) + client = ResponseClient(base_url="https://example.com") + service_url = client.get_service_url() + yield service_url \ No newline at end of file diff --git a/dev/integration/src/tests/core/test_application_runner.py b/dev/integration/src/tests/core/test_application_runner.py new file mode 100644 index 00000000..4d4b9c2d --- /dev/null +++ b/dev/integration/src/tests/core/test_application_runner.py @@ -0,0 +1,49 @@ +import pytest +from time import sleep + +from src.core import ApplicationRunner + +class SimpleRunner(ApplicationRunner): + def _start_server(self) -> None: + self._app["running"] = True + +class OtherSimpleRunner(SimpleRunner): + def _stop_server(self) -> None: + self._app["running"] = False + +class TestApplicationRunner: + + @pytest.mark.asyncio + async def test_simple_runner(self): + + app = {} + runner = SimpleRunner(app) + async with runner as r: + sleep(0.1) + assert runner is r + assert app["running"] is True + + assert app["running"] is True + + @pytest.mark.asyncio + async def test_other_simple_runner(self): + + app = {} + runner = OtherSimpleRunner(app) + async with runner as r: + sleep(0.1) + assert runner is r + assert app["running"] is True + + assert app["running"] is False + + @pytest.mark.asyncio + async def test_double_start(self): + + app = {} + runner = SimpleRunner(app) + async with runner: + sleep(0.1) + with pytest.raises(RuntimeError, match="Server is already running"): + async with runner: + pass \ No newline at end of file diff --git a/dev/integration/src/tests/core/test_integration_from_sample.py b/dev/integration/src/tests/core/test_integration_from_sample.py new file mode 100644 index 00000000..fb86be68 --- /dev/null +++ b/dev/integration/src/tests/core/test_integration_from_sample.py @@ -0,0 +1,6 @@ +import pytest + +from src.core import integration + +class TestIntegrationFromSample: + pass \ No newline at end of file diff --git a/dev/integration/src/tests/core/test_integration_from_service_url.py b/dev/integration/src/tests/core/test_integration_from_service_url.py new file mode 100644 index 00000000..9c298776 --- /dev/null +++ b/dev/integration/src/tests/core/test_integration_from_service_url.py @@ -0,0 +1,7 @@ +import pytest + +from src.core import integration + +class TestIntegrationFromServiceURL: + + pass \ No newline at end of file diff --git a/dev/integration/src/tests/environments/__init__.py b/dev/integration/src/tests/environments/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/samples/__init__.py b/dev/integration/src/tests/samples/__init__.py new file mode 100644 index 00000000..e69de29b From d4828fb088aff03833cc7947e26b369fbd86946a Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 30 Oct 2025 15:20:37 -0700 Subject: [PATCH 14/81] Adding integration decor from sample test cases --- dev/integration/src/core/integration.py | 8 +-- dev/integration/src/tests/core/_common.py | 13 +++++ .../src/tests/core/test_application_runner.py | 12 +--- .../core/test_integration_from_sample.py | 57 ++++++++++++++++++- .../src/tests/environments/__init__.py | 0 dev/integration/src/tests/samples/__init__.py | 0 6 files changed, 73 insertions(+), 17 deletions(-) create mode 100644 dev/integration/src/tests/core/_common.py delete mode 100644 dev/integration/src/tests/environments/__init__.py delete mode 100644 dev/integration/src/tests/samples/__init__.py diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index 87f21c5e..df7483e8 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -111,8 +111,8 @@ async def teardown_method(self): def integration( service_url: Optional[str] = None, - sample_cls: Optional[type[Sample]] = None, - environment_cls: Optional[type[Environment]] = None, + sample: Optional[type[Sample]] = None, + environment: Optional[type[Environment]] = None, app: Optional[AppT] = None, host_agent: bool = False, host_response: bool = True, @@ -140,8 +140,8 @@ def integration( if service_url: decorator = _Integration.from_service_url(service_url, host_response=host_response) - elif sample_cls and environment_cls: - decorator = _Integration.from_sample(sample_cls, environment_cls, host_agent=host_agent, host_response=host_response) + elif sample and environment: + decorator = _Integration.from_sample(sample, environment, host_agent=host_agent, host_response=host_response) # elif app: # decorator = _Integration.from_app(app, host_response=host_response) else: diff --git a/dev/integration/src/tests/core/_common.py b/dev/integration/src/tests/core/_common.py new file mode 100644 index 00000000..895caef2 --- /dev/null +++ b/dev/integration/src/tests/core/_common.py @@ -0,0 +1,13 @@ +from src.core import ApplicationRunner + +class SimpleRunner(ApplicationRunner): + def _start_server(self) -> None: + self._app["running"] = True + + @property + def app(self): + return self._app + +class OtherSimpleRunner(SimpleRunner): + def _stop_server(self) -> None: + self._app["running"] = False \ No newline at end of file diff --git a/dev/integration/src/tests/core/test_application_runner.py b/dev/integration/src/tests/core/test_application_runner.py index 4d4b9c2d..dd1d0c28 100644 --- a/dev/integration/src/tests/core/test_application_runner.py +++ b/dev/integration/src/tests/core/test_application_runner.py @@ -1,15 +1,7 @@ import pytest from time import sleep -from src.core import ApplicationRunner - -class SimpleRunner(ApplicationRunner): - def _start_server(self) -> None: - self._app["running"] = True - -class OtherSimpleRunner(SimpleRunner): - def _stop_server(self) -> None: - self._app["running"] = False +from ._common import SimpleRunner, OtherSimpleRunner class TestApplicationRunner: @@ -44,6 +36,6 @@ async def test_double_start(self): runner = SimpleRunner(app) async with runner: sleep(0.1) - with pytest.raises(RuntimeError, match="Server is already running"): + with pytest.raises(RuntimeError): async with runner: pass \ No newline at end of file diff --git a/dev/integration/src/tests/core/test_integration_from_sample.py b/dev/integration/src/tests/core/test_integration_from_sample.py index fb86be68..65a160aa 100644 --- a/dev/integration/src/tests/core/test_integration_from_sample.py +++ b/dev/integration/src/tests/core/test_integration_from_sample.py @@ -1,6 +1,57 @@ import pytest +import asyncio +from copy import copy -from src.core import integration +from src.core import ( + ApplicationRunner, + Environment, + integration, + IntegrationFixtures, + Sample +) -class TestIntegrationFromSample: - pass \ No newline at end of file +from ._common import SimpleRunner, OtherSimpleRunner + +class SimpleEnvironment(Environment): + """A simple implementation of the Environment for testing.""" + + async def init_env(self, environ_config: dict) -> None: + self.config = environ_config + # Initialize other components as needed + + def create_runner(self) -> ApplicationRunner: + return SimpleRunner(copy(self.config)) + +class SimpleSample(Sample): + """A simple implementation of the Sample for testing.""" + + def __init__(self, environment: Environment, **kwargs): + super().__init__(environment, **kwargs) + self.data = kwargs.get("data", "default_data") + self.other_data = None + + @classmethod + async def get_config(cls) -> dict: + return {"sample_key": "sample_value"} + + async def init_app(self): + await asyncio.sleep(0.1) # Simulate some initialization delay + self.other_data = len(self.env.config) + +@integration(sample=SimpleSample, environment=SimpleEnvironment) +class TestIntegrationFromSample(IntegrationFixtures): + + @pytest.mark.asyncio + async def test_sample_integration(self, sample, environment): + """Test the integration of SimpleSample with SimpleEnvironment.""" + + assert environment.config == { + "sample_key": "sample_value" + } + + assert sample.env is environment + assert sample.data == "default_data" + assert sample.other_data == 1 + + runner = environment.create_runner() + assert runner.app == {"running": True} \ No newline at end of file diff --git a/dev/integration/src/tests/environments/__init__.py b/dev/integration/src/tests/environments/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/integration/src/tests/samples/__init__.py b/dev/integration/src/tests/samples/__init__.py deleted file mode 100644 index e69de29b..00000000 From 91b3c44169562c91ca94b75debd0b3e49ec4c9bf Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 30 Oct 2025 17:04:07 -0700 Subject: [PATCH 15/81] Integration from service url tests --- .../core/test_integration_from_service_url.py | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/dev/integration/src/tests/core/test_integration_from_service_url.py b/dev/integration/src/tests/core/test_integration_from_service_url.py index 9c298776..773f5b76 100644 --- a/dev/integration/src/tests/core/test_integration_from_service_url.py +++ b/dev/integration/src/tests/core/test_integration_from_service_url.py @@ -1,7 +1,41 @@ import pytest +import asyncio +from copy import copy +from aioresponses import aioresponses -from src.core import integration +from src.core import ( + integration, + IntegrationFixtures +) -class TestIntegrationFromServiceURL: +@integration(service_url="http://localhost:8000/api/messages") +class TestIntegrationFromServiceURL(IntegrationFixtures): - pass \ No newline at end of file + @pytest.mark.asyncio + async def test_service_url_integration(self, agent_client): + """Test the integration using a service URL.""" + + with aioresponses() as mocked: + + mocked.post("http://localhost:8000/api/messages", status=200, body="Service response") + + response = await agent_client.send_activity("Hello, service!") + assert response.status_code == 200 + assert "service" in response.text.lower() + + @pytest.mark.asyncio + async def test_service_url_integration_with_response_side_effect(self, agent_client, response_client): + """Test the integration using a service URL.""" + + with aioresponses() as mocked: + + mocked.post("http://localhost:8000/api/messages", status=200, body="Service response") + + response = await agent_client.send_activity("Hello, service!") + assert response.status_code == 200 + assert "service" in response.text.lower() + + await asyncio.sleep(1) + + res = await response_client.pop() + assert len(res) == 1 \ No newline at end of file From 1eefe445444d8a0b4bb99393b5337fb14d2dda52 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 30 Oct 2025 17:07:43 -0700 Subject: [PATCH 16/81] _handle_conversation implementation for response_client --- .../src/core/client/response_client.py | 2 ++ .../tests/core/client/test_agent_client.py | 26 ++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py index 11cdbd7e..45c7b562 100644 --- a/dev/integration/src/core/client/response_client.py +++ b/dev/integration/src/core/client/response_client.py @@ -64,6 +64,8 @@ async def _handle_conversation(self, request: Request) -> Response: ) as resp: resp_text = await resp.text() return Response(status=resp.status, text=resp_text) + except Exception as e: + return Response(status=500, text=str(e)) async def _handle_streamed_activity(self, activity: Activity, *args, **kwargs) -> bool: raise NotImplementedError("_handle_streamed_activity is not implemented yet.") \ No newline at end of file diff --git a/dev/integration/src/tests/core/client/test_agent_client.py b/dev/integration/src/tests/core/client/test_agent_client.py index d4d363b4..38ac1a85 100644 --- a/dev/integration/src/tests/core/client/test_agent_client.py +++ b/dev/integration/src/tests/core/client/test_agent_client.py @@ -10,9 +10,23 @@ def agent_client(self) -> AgentClient: return AgentClient(base_url="") @pytest.fixture - async def service_url(self): - with aioresponses() as mocked: - mocked.get("https://example.com/service-url", payload={"serviceUrl": "https://service.example.com"}) - client = AgentClient(base_url="https://example.com") - service_url = await client.get_service_url() - yield service_url \ No newline at end of file + def mock_service(self): + + async def context_manager(): + with aioresponses() as mocked: + mocked.get("https://example.com/service-url", payload={"serviceUrl": "https://service.example.com"}) + client = AgentClient(base_url="https://example.com") + service_url = await client.get_service_url() + yield service_url + + return context_manager + + @pytest.mark.asyncio + async def test_get_service_url(self, agent_client: AgentClient, service_url: str): + assert service_url == "https://service.example.com" + + @pytest.mark.asyncio + async def test_send_activity(self, agent_client, mock_service): + with mock_service as service_url: + response = await agent_client.send_activity("Hello, World!") + assert response == "Response from service" \ No newline at end of file From a06da9a1d3c70f179a3e7b83b8539d236d0ef638 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Fri, 31 Oct 2025 15:12:40 -0700 Subject: [PATCH 17/81] AgentClient tests completed --- .../src/core/client/agent_client.py | 66 ++++++++++------ .../src/core/client/response_client.py | 5 +- .../tests/core/client/test_agent_client.py | 76 ++++++++++++++----- 3 files changed, 104 insertions(+), 43 deletions(-) diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py index 237a4f7d..97084137 100644 --- a/dev/integration/src/core/client/agent_client.py +++ b/dev/integration/src/core/client/agent_client.py @@ -18,56 +18,71 @@ class AgentClient: def __init__( self, messaging_endpoint: str, - service_endpoint: str, cid: str, client_id: str, tenant_id: str, client_secret: str, + service_url: Optional[str] = None, default_timeout: float = 5.0 ): self._messaging_endpoint = messaging_endpoint - self.service_endpoint = service_endpoint - self.cid = cid - self.client_id = client_id - self.tenant_id = tenant_id - self.client_secret = client_secret + self._cid = cid + self._client_id = client_id + self._tenant_id = tenant_id + self._client_secret = client_secret + self._service_url = service_url self._headers = None self._default_timeout = default_timeout - self._client = ClientSession( - base_url=self._messaging_endpoint, - headers={"Content-Type": "application/json"} - ) + self._client: Optional[ClientSession] = None - self._msal_app = ConfidentialClientApplication( - client_id=client_id, - client_credential=client_secret, - authority=f"https://login.microsoftonline.com/{tenant_id}" - ) + @property + def messaging_endpoint(self) -> str: + return self._messaging_endpoint + + @property + def service_url(self) -> Optional[str]: + return self._service_url async def get_access_token(self) -> str: - res = self._msal_app.acquire_token_for_client( - scopes=[f"{self.client_id}/.default"] + + msal_app = ConfidentialClientApplication( + client_id=self._client_id, + client_credential=self._client_secret, + authority=f"https://login.microsoftonline.com/{self._tenant_id}" + ) + + res = msal_app.acquire_token_for_client( + scopes=[f"{self._client_id}/.default"] ) token = res.get("access_token") if res else None if not token: raise Exception("Could not obtain access token") return token - - async def _set_headers(self) -> None: - if not self._headers: + + async def _init_client(self) -> None: + if not self._client: token = await self.get_access_token() self._headers = { "Authorization": f"Bearer {token}", "Content-Type": "application/json" } + self._client = ClientSession( + base_url=self._messaging_endpoint, + headers=self._headers + ) + async def send_request(self, activity: Activity) -> str: - await self._set_headers() + await self._init_client() + assert self._client if activity.conversation: - activity.conversation.id = self.cid + activity.conversation.id = self._cid + + if self.service_url: + activity.service_url = self.service_url async with self._client.post( self._messaging_endpoint, @@ -104,4 +119,9 @@ async def send_expect_replies(self, activity_or_text: Activity | str, timeout: O activities_data = json.loads(content).get("activities", []) activities = [Activity.model_validate(act) for act in activities_data] - return activities \ No newline at end of file + return activities + + async def close(self) -> None: + if self._client: + await self._client.close() + self._client = None \ No newline at end of file diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py index 45c7b562..ae03a155 100644 --- a/dev/integration/src/core/client/response_client.py +++ b/dev/integration/src/core/client/response_client.py @@ -2,7 +2,6 @@ import sys from io import StringIO -from typing import Optional from threading import Lock from aiohttp import ClientSession @@ -23,7 +22,7 @@ def __init__( self._prev_stdout = None self._service_endpoint = service_endpoint self._activities_list = [] - self._activities_list_lock = [] + self._activities_list_lock = Lock() self._app.router.add_post( "/v3/conversations/{path:.*}", @@ -55,6 +54,7 @@ async def _handle_conversation(self, request: Request) -> Response: if any(map(lambda x: x.type == "streaminfo", activity.entities or [])): await self._handle_streamed_activity(activity) + return Response(status=200, text="Stream info handled") else: if activity.type != ActivityTypes.typing: async with ClientSession() as session: @@ -64,6 +64,7 @@ async def _handle_conversation(self, request: Request) -> Response: ) as resp: resp_text = await resp.text() return Response(status=resp.status, text=resp_text) + return Response(status=200, text="Activity received") except Exception as e: return Response(status=500, text=str(e)) diff --git a/dev/integration/src/tests/core/client/test_agent_client.py b/dev/integration/src/tests/core/client/test_agent_client.py index 38ac1a85..868f65a5 100644 --- a/dev/integration/src/tests/core/client/test_agent_client.py +++ b/dev/integration/src/tests/core/client/test_agent_client.py @@ -1,32 +1,72 @@ +import json +from contextlib import contextmanager +import re + import pytest from aioresponses import aioresponses +from msal import ConfidentialClientApplication + +from microsoft_agents.activity import Activity from src.core import AgentClient +class DEFAULTS: + + messaging_endpoint = "http://localhost:8000" + service_url = "http://localhost:8001" + cid = "test-cid" + client_id = "test-client-id" + tenant_id = "test-tenant-id" + client_secret = "test-client-secret" + class TestAgentClient: @pytest.fixture - def agent_client(self) -> AgentClient: - return AgentClient(base_url="") - + async def agent_client(self): + client = AgentClient( + messaging_endpoint=DEFAULTS.messaging_endpoint, + cid=DEFAULTS.cid, + client_id=DEFAULTS.client_id, + tenant_id=DEFAULTS.tenant_id, + client_secret=DEFAULTS.client_secret, + service_url=DEFAULTS.service_url + ) + yield client + await client.close() + @pytest.fixture - def mock_service(self): + def aioresponses_mock(self): + with aioresponses() as mocked: + yield mocked - async def context_manager(): - with aioresponses() as mocked: - mocked.get("https://example.com/service-url", payload={"serviceUrl": "https://service.example.com"}) - client = AgentClient(base_url="https://example.com") - service_url = await client.get_service_url() - yield service_url + @pytest.mark.asyncio + async def test_send_activity(self, mocker, agent_client, aioresponses_mock): + mocker.patch.object(AgentClient, 'get_access_token', return_value="mocked_token") + mocker.patch.object(ConfidentialClientApplication, "__new__", return_value=mocker.Mock(spec=ConfidentialClientApplication)) - return context_manager + assert agent_client.messaging_endpoint + aioresponses_mock.post(agent_client.messaging_endpoint, payload={"response": "Response from service"}) + + response = await agent_client.send_activity("Hello, World!") + data = json.loads(response) + assert data == {"response": "Response from service"} @pytest.mark.asyncio - async def test_get_service_url(self, agent_client: AgentClient, service_url: str): - assert service_url == "https://service.example.com" + async def test_send_expect_replies(self, mocker, agent_client, aioresponses_mock): + mocker.patch.object(AgentClient, 'get_access_token', return_value="mocked_token") + mocker.patch.object(ConfidentialClientApplication, "__new__", return_value=mocker.Mock(spec=ConfidentialClientApplication)) - @pytest.mark.asyncio - async def test_send_activity(self, agent_client, mock_service): - with mock_service as service_url: - response = await agent_client.send_activity("Hello, World!") - assert response == "Response from service" \ No newline at end of file + assert agent_client.messaging_endpoint + activities = [ + Activity(type="message", text="Response from service"), + Activity(type="message", text="Another response"), + ] + aioresponses_mock.post(agent_client.messaging_endpoint, payload={ + "activities": [activity.model_dump(by_alias=True, exclude_none=True) for activity in activities], + }) + + replies = await agent_client.send_expect_replies("Hello, World!") + assert len(replies) == 2 + assert replies[0].text == "Response from service" + assert replies[1].text == "Another response" + assert replies[0].type == replies[1].type == "message" \ No newline at end of file From d36ec7aaf1e3b74c098c48618a901e147e84a37b Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 08:14:36 -0800 Subject: [PATCH 18/81] Creating response client tests --- .../src/core/client/response_client.py | 15 +++----- .../src/tests/core/client/_common.py | 8 +++++ .../tests/core/client/test_agent_client.py | 9 +---- .../tests/core/client/test_response_client.py | 36 +++++++++++++------ 4 files changed, 39 insertions(+), 29 deletions(-) create mode 100644 dev/integration/src/tests/core/client/_common.py diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py index ae03a155..aa09f8c9 100644 --- a/dev/integration/src/core/client/response_client.py +++ b/dev/integration/src/core/client/response_client.py @@ -3,6 +3,7 @@ import sys from io import StringIO from threading import Lock +import asyncio from aiohttp import ClientSession from aiohttp.web import Application, Request, Response @@ -33,12 +34,12 @@ def __init__( def service_endpoint(self) -> str: return self._service_endpoint - def __aenter__(self) -> ResponseClient: + async def __aenter__(self) -> ResponseClient: self._prev_stdout = sys.stdout sys.stdout = StringIO() return self - - def __aexit__(self, exc_type, exc, tb): + + async def __aexit__(self, exc_type, exc, tb): if self._prev_stdout is not None: sys.stdout = self._prev_stdout @@ -57,13 +58,7 @@ async def _handle_conversation(self, request: Request) -> Response: return Response(status=200, text="Stream info handled") else: if activity.type != ActivityTypes.typing: - async with ClientSession() as session: - async with session.post( - f"{self._service_endpoint}/v3/conversations/{conversation_id}/activities", - json=activity.model_dump() - ) as resp: - resp_text = await resp.text() - return Response(status=resp.status, text=resp_text) + await asyncio.sleep(0.1) # Simulate processing delay return Response(status=200, text="Activity received") except Exception as e: return Response(status=500, text=str(e)) diff --git a/dev/integration/src/tests/core/client/_common.py b/dev/integration/src/tests/core/client/_common.py new file mode 100644 index 00000000..66d6adbf --- /dev/null +++ b/dev/integration/src/tests/core/client/_common.py @@ -0,0 +1,8 @@ +class DEFAULTS: + + messaging_endpoint = "http://localhost:8000" + service_url = "http://localhost:8001" + cid = "test-cid" + client_id = "test-client-id" + tenant_id = "test-tenant-id" + client_secret = "test-client-secret" \ No newline at end of file diff --git a/dev/integration/src/tests/core/client/test_agent_client.py b/dev/integration/src/tests/core/client/test_agent_client.py index 868f65a5..a5a0ea7e 100644 --- a/dev/integration/src/tests/core/client/test_agent_client.py +++ b/dev/integration/src/tests/core/client/test_agent_client.py @@ -10,14 +10,7 @@ from src.core import AgentClient -class DEFAULTS: - - messaging_endpoint = "http://localhost:8000" - service_url = "http://localhost:8001" - cid = "test-cid" - client_id = "test-client-id" - tenant_id = "test-tenant-id" - client_secret = "test-client-secret" +from ._common import DEFAULTS class TestAgentClient: diff --git a/dev/integration/src/tests/core/client/test_response_client.py b/dev/integration/src/tests/core/client/test_response_client.py index 81794474..d9877b7b 100644 --- a/dev/integration/src/tests/core/client/test_response_client.py +++ b/dev/integration/src/tests/core/client/test_response_client.py @@ -1,19 +1,33 @@ from re import A import pytest -from aioresponses import aioresponses +from aiohttp import ClientSession + +from microsoft_agents.activity import Activity from src.core import ResponseClient +from ._common import DEFAULTS class TestResponseClient: - - @pytest.fixture - def response_client(self): - return ResponseClient(base_url="") @pytest.fixture - def service_url(self): - with aioresponses() as mocked: - mocked.get("https://example.com/service-url", payload={"serviceUrl": "https://service.example.com"}) - client = ResponseClient(base_url="https://example.com") - service_url = client.get_service_url() - yield service_url \ No newline at end of file + async def response_client(self): + async with ResponseClient(DEFAULTS.service_url) as client: + yield client + + @pytest.mark.asyncio + async def test_init(self, response_client): + assert response_client.service_endpoint == DEFAULTS.service_url + + @pytest.mark.asyncio + async def test_endpoint(self, response_client): + + activity = Activity(type="message", text="Hello, World!") + + async with ClientSession() as session: + async with session.post( + f"{response_client.service_endpoint}/v3/conversations/test-conv", + json=activity.model_dump(by_alias=True, exclude_none=True) + ) as resp: + assert resp.status == 200 + text = await resp.text() + assert text == "" \ No newline at end of file From c096048d642e0382020b8a586f3504cd0bd299ab Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 08:51:53 -0800 Subject: [PATCH 19/81] Hosting server for response client --- .../src/core/client/response_client.py | 28 ++++++++++++++++--- .../tests/core/client/test_response_client.py | 1 - 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py index aa09f8c9..a0567db1 100644 --- a/dev/integration/src/core/client/response_client.py +++ b/dev/integration/src/core/client/response_client.py @@ -2,11 +2,11 @@ import sys from io import StringIO -from threading import Lock +from threading import Lock, to_thread, Thread import asyncio from aiohttp import ClientSession -from aiohttp.web import Application, Request, Response +from aiohttp.web import Application, Request, Response, run_app from microsoft_agents.activity import ( Activity, @@ -17,13 +17,20 @@ class ResponseClient: def __init__( self, - service_endpoint: str = "http://localhost:9873" + host: str = "localhost", + port: int = 9873, ): self._app: Application = Application() self._prev_stdout = None + service_endpoint = f"{host}:{port}" + self._host = host + self._port = port + if "http" not in service_endpoint: + service_endpoint = f"http://{service_endpoint}" self._service_endpoint = service_endpoint self._activities_list = [] self._activities_list_lock = Lock() + self._server_thread: Optional[Thread] = None self._app.router.add_post( "/v3/conversations/{path:.*}", @@ -33,16 +40,29 @@ def __init__( @property def service_endpoint(self) -> str: return self._service_endpoint + + def start(self) -> None: + run_app(self._app, host=self._host, port=self._port) async def __aenter__(self) -> ResponseClient: + if self._server_thread: + raise RuntimeError("ResponseClient is already running.") + self._prev_stdout = sys.stdout sys.stdout = StringIO() + self._server_thread = Thread(target=self.start) + self._server_thread.start() return self async def __aexit__(self, exc_type, exc, tb): + if not self._server_thread: + raise RuntimeError("ResponseClient is not running.") if self._prev_stdout is not None: sys.stdout = self._prev_stdout + self._server_thread.join() + self._server_thread = None + async def _handle_conversation(self, request: Request) -> Response: try: body = await request.text() @@ -64,4 +84,4 @@ async def _handle_conversation(self, request: Request) -> Response: return Response(status=500, text=str(e)) async def _handle_streamed_activity(self, activity: Activity, *args, **kwargs) -> bool: - raise NotImplementedError("_handle_streamed_activity is not implemented yet.") \ No newline at end of file + raise NotImplementedError("_handle_streamed_activity is not implemented yet.") diff --git a/dev/integration/src/tests/core/client/test_response_client.py b/dev/integration/src/tests/core/client/test_response_client.py index d9877b7b..a5076ad9 100644 --- a/dev/integration/src/tests/core/client/test_response_client.py +++ b/dev/integration/src/tests/core/client/test_response_client.py @@ -1,4 +1,3 @@ -from re import A import pytest from aiohttp import ClientSession From 5451ddfd6e2bb2ecba7caed767c51418654c7cfe Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 09:43:52 -0800 Subject: [PATCH 20/81] Response client tests completed --- .../src/core/client/response_client.py | 64 +++++++++++++++++-- .../src/tests/core/client/_common.py | 6 +- .../tests/core/client/test_response_client.py | 14 +++- 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py index a0567db1..3d69d17b 100644 --- a/dev/integration/src/core/client/response_client.py +++ b/dev/integration/src/core/client/response_client.py @@ -1,12 +1,16 @@ from __future__ import annotations import sys +import requests from io import StringIO -from threading import Lock, to_thread, Thread +from typing import Optional +from threading import Lock, Thread, Event import asyncio +import weakref from aiohttp import ClientSession from aiohttp.web import Application, Request, Response, run_app +from aiohttp.web_runner import AppRunner, TCPSite from microsoft_agents.activity import ( Activity, @@ -31,6 +35,14 @@ def __init__( self._activities_list = [] self._activities_list_lock = Lock() self._server_thread: Optional[Thread] = None + self._shutdown_event = Event() + self._runner: Optional[AppRunner] = None + self._site: Optional[TCPSite] = None + + self._app.router.add_get( + "/shutdown", + self._shutdown + ) self._app.router.add_post( "/v3/conversations/{path:.*}", @@ -42,7 +54,28 @@ def service_endpoint(self) -> str: return self._service_endpoint def start(self) -> None: - run_app(self._app, host=self._host, port=self._port) + """Start the server in the current thread""" + async def _run_server(): + self._runner = AppRunner(self._app) + await self._runner.setup() + self._site = TCPSite(self._runner, self._host, self._port) + await self._site.start() + + # Wait for shutdown signal + while not self._shutdown_event.is_set(): + await asyncio.sleep(0.1) + + # Cleanup + await self._site.stop() + await self._runner.cleanup() + + # Run the server + asyncio.run(_run_server()) + + async def _shutdown(self, request: Request) -> Response: + """Handle shutdown request by setting the shutdown event""" + self._shutdown_event.set() + return Response(status=200, text="Shutdown initiated") async def __aenter__(self) -> ResponseClient: if self._server_thread: @@ -50,8 +83,12 @@ async def __aenter__(self) -> ResponseClient: self._prev_stdout = sys.stdout sys.stdout = StringIO() + self._shutdown_event.clear() self._server_thread = Thread(target=self.start) self._server_thread.start() + + # Wait a bit for the server to start + await asyncio.sleep(0.2) return self async def __aexit__(self, exc_type, exc, tb): @@ -60,13 +97,24 @@ async def __aexit__(self, exc_type, exc, tb): if self._prev_stdout is not None: sys.stdout = self._prev_stdout - self._server_thread.join() + try: + async with ClientSession() as session: + async with session.get(f"http://{self._host}:{self._port}/shutdown") as response: + pass # Just trigger the shutdown + except Exception: + pass # Ignore errors during shutdown request + + # Set shutdown event as fallback + self._shutdown_event.set() + + # Wait for the server thread to finish + self._server_thread.join(timeout=5.0) self._server_thread = None async def _handle_conversation(self, request: Request) -> Response: try: - body = await request.text() - activity = Activity.model_validate(body) + data = await request.json() + activity = Activity.model_validate(data) conversation_id = activity.conversation.id if activity.conversation else None @@ -85,3 +133,9 @@ async def _handle_conversation(self, request: Request) -> Response: async def _handle_streamed_activity(self, activity: Activity, *args, **kwargs) -> bool: raise NotImplementedError("_handle_streamed_activity is not implemented yet.") + + async def pop(self) -> list[Activity]: + with self._activities_list_lock: + activities = self._activities_list[:] + self._activities_list.clear() + return activities \ No newline at end of file diff --git a/dev/integration/src/tests/core/client/_common.py b/dev/integration/src/tests/core/client/_common.py index 66d6adbf..e6df31e4 100644 --- a/dev/integration/src/tests/core/client/_common.py +++ b/dev/integration/src/tests/core/client/_common.py @@ -1,7 +1,9 @@ class DEFAULTS: - messaging_endpoint = "http://localhost:8000" - service_url = "http://localhost:8001" + host = "localhost" + response_port = 9873 + messaging_endpoint = f"http://{host}:8000" + service_url = f"http://{host}:{response_port}" cid = "test-cid" client_id = "test-client-id" tenant_id = "test-tenant-id" diff --git a/dev/integration/src/tests/core/client/test_response_client.py b/dev/integration/src/tests/core/client/test_response_client.py index a5076ad9..73f4a105 100644 --- a/dev/integration/src/tests/core/client/test_response_client.py +++ b/dev/integration/src/tests/core/client/test_response_client.py @@ -1,4 +1,5 @@ import pytest +import asyncio from aiohttp import ClientSession from microsoft_agents.activity import Activity @@ -10,7 +11,7 @@ class TestResponseClient: @pytest.fixture async def response_client(self): - async with ResponseClient(DEFAULTS.service_url) as client: + async with ResponseClient(host=DEFAULTS.host, port=DEFAULTS.response_port) as client: yield client @pytest.mark.asyncio @@ -29,4 +30,13 @@ async def test_endpoint(self, response_client): ) as resp: assert resp.status == 200 text = await resp.text() - assert text == "" \ No newline at end of file + assert text == "Activity received" + + await asyncio.sleep(0.1) # Give some time for the server to process + + activities = await response_client.pop() + assert len(activities) == 1 + assert activities[0].type == "message" + assert activities[0].text == "Hello, World!" + + assert (await response_client.pop()) == [] \ No newline at end of file From c2cdd40ac9f46a5540e333a9b99ea20c28be797e Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 10:24:55 -0800 Subject: [PATCH 21/81] Beginning refactor of aiohttp runner --- dev/integration/src/core/aiohttp/__init__.py | 7 + .../aiohttp}/aiohttp_environment.py | 15 +- .../src/core/aiohttp/aiohttp_runner.py | 77 ++++++++ .../src/core/application_runner.py | 12 +- .../src/core/client/bot_response.py | 182 ------------------ .../src/core/client/response_client.py | 13 +- dev/integration/src/environments/__init__.py | 5 - 7 files changed, 98 insertions(+), 213 deletions(-) create mode 100644 dev/integration/src/core/aiohttp/__init__.py rename dev/integration/src/{environments => core/aiohttp}/aiohttp_environment.py (83%) create mode 100644 dev/integration/src/core/aiohttp/aiohttp_runner.py delete mode 100644 dev/integration/src/core/client/bot_response.py delete mode 100644 dev/integration/src/environments/__init__.py diff --git a/dev/integration/src/core/aiohttp/__init__.py b/dev/integration/src/core/aiohttp/__init__.py new file mode 100644 index 00000000..236bc136 --- /dev/null +++ b/dev/integration/src/core/aiohttp/__init__.py @@ -0,0 +1,7 @@ +from .aiohttp_environment import AiohttpEnvironment +from .aiohttp_runner import AiohttpRunner + +__all__ = [ + "AiohttpEnvironment", + "AiohttpRunner" +] \ No newline at end of file diff --git a/dev/integration/src/environments/aiohttp_environment.py b/dev/integration/src/core/aiohttp/aiohttp_environment.py similarity index 83% rename from dev/integration/src/environments/aiohttp_environment.py rename to dev/integration/src/core/aiohttp/aiohttp_environment.py index 7525c9ea..70df2842 100644 --- a/dev/integration/src/environments/aiohttp_environment.py +++ b/dev/integration/src/core/aiohttp/aiohttp_environment.py @@ -16,24 +16,11 @@ from microsoft_agents.authentication.msal import MsalConnectionManager from microsoft_agents.activity import load_configuration_from_env -from ..core import ( +from . import ( ApplicationRunner, Environment ) -class AiohttpRunner(ApplicationRunner): - """A runner for aiohttp applications.""" - - def _start_server(self) -> None: - try: - assert isinstance(self._app, Application) - run_app(self._app, host="localhost", port=3978) - except Exception as error: - raise error - - def _stop_server(self) -> None: - pass - class AiohttpEnvironment(Environment): """An environment for aiohttp-hosted agents.""" diff --git a/dev/integration/src/core/aiohttp/aiohttp_runner.py b/dev/integration/src/core/aiohttp/aiohttp_runner.py new file mode 100644 index 00000000..a4520b2f --- /dev/null +++ b/dev/integration/src/core/aiohttp/aiohttp_runner.py @@ -0,0 +1,77 @@ +from typing import Optional +from typing import Optional +from threading import Thread, Event +import asyncio + +from aiohttp.web import Application, Request, Response +from aiohttp.web_runner import AppRunner, TCPSite + +from ..application_runner import ApplicationRunner + +class AiohttpRunner(ApplicationRunner): + """A runner for aiohttp applications.""" + + def __init__(self, app: Application): + assert isinstance(app, Application) + super().__init__(app) + + self._app.router.add_get( + "/shutdown", + self._shutdown + ) + + self._server_thread: Optional[Thread] = None + self._shutdown_event = Event() + self._runner: Optional[AppRunner] = None + self._site: Optional[TCPSite] = None + + + async def _start_server(self) -> None: + try: + assert isinstance(self._app, Application) + async def _run_server(): + self._runner = AppRunner(self._app) + await self._runner.setup() + self._site = TCPSite(self._runner, self._host, self._port) + await self._site.start() + + # Wait for shutdown signal + while not self._shutdown_event.is_set(): + await asyncio.sleep(0.1) + + # Cleanup + await self._site.stop() + await self._runner.cleanup() + + # Run the server + asyncio.run(_run_server()) + except Exception as error: + raise error + + async def _stop_server(self) -> None: + pass + + async def _shutdown(self, request: Request) -> Response: + """Handle shutdown request by setting the shutdown event""" + self._shutdown_event.set() + return Response(status=200, text="Shutdown initiated") + + async def __aexit__(self, exc_type, exc, tb): + if not self._server_thread: + raise RuntimeError("ResponseClient is not running.") + if self._prev_stdout is not None: + sys.stdout = self._prev_stdout + + try: + async with ClientSession() as session: + async with session.get(f"http://{self._host}:{self._port}/shutdown") as response: + pass # Just trigger the shutdown + except Exception: + pass # Ignore errors during shutdown request + + # Set shutdown event as fallback + self._shutdown_event.set() + + # Wait for the server thread to finish + self._server_thread.join(timeout=5.0) + self._server_thread = None \ No newline at end of file diff --git a/dev/integration/src/core/application_runner.py b/dev/integration/src/core/application_runner.py index d6d73fb1..cea6157c 100644 --- a/dev/integration/src/core/application_runner.py +++ b/dev/integration/src/core/application_runner.py @@ -1,3 +1,4 @@ +import asyncio from abc import ABC, abstractmethod from typing import Any, Optional from threading import Thread @@ -10,10 +11,10 @@ def __init__(self, app: Any): self._thread: Optional[Thread] = None @abstractmethod - def _start_server(self) -> None: + async def _start_server(self) -> None: raise NotImplementedError("Start server method must be implemented by subclasses") - def _stop_server(self) -> None: + async def _stop_server(self) -> None: pass async def __aenter__(self) -> None: @@ -21,13 +22,16 @@ async def __aenter__(self) -> None: if self._thread: raise RuntimeError("Server is already running") - self._thread = Thread(target=self._start_server, daemon=True) + def target(): + asyncio.run(self._start_server()) + + self._thread = Thread(target=target, daemon=True) self._thread.start() async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: if self._thread: - self._stop_server() + await self._stop_server() self._thread.join() self._thread = None diff --git a/dev/integration/src/core/client/bot_response.py b/dev/integration/src/core/client/bot_response.py deleted file mode 100644 index 1ba46c0c..00000000 --- a/dev/integration/src/core/client/bot_response.py +++ /dev/null @@ -1,182 +0,0 @@ -import asyncio -import json -import sys -import threading -import uuid -from io import StringIO -from typing import Dict, List, Optional -from threading import Lock -from collections import defaultdict - -from aiohttp import web, ClientSession -from aiohttp.web import Request, Response -import aiohttp_security -from microsoft_agents.core.models import Activity, EntityTypes, ActivityTypes, StreamInfo, StreamTypes -from microsoft_agents.core.serialization import ProtocolJsonSerializer - -class BotResponse: - """Python equivalent of the C# BotResponse class using aiohttp web framework.""" - - def __init__(self): - self._app: Optional[web.Application] = None - self._runner: Optional[web.AppRunner] = None - self._site: Optional[web.TCPSite] = None - self._multiple_activities: Dict[str, List[Activity]] = defaultdict(list) - self._activity_locks: Dict[str, Lock] = defaultdict(Lock) - self.test_id: str = str(uuid.uuid4()) - self.service_endpoint: str = "http://localhost:9873" - - # Suppress console output (equivalent to Console.SetOut(TextWriter.Null)) - sys.stdout = StringIO() - - # Initialize the web application - self._setup_app() - - def _setup_app(self): - """Setup the aiohttp web application with routes and middleware.""" - self._app = web.Application() - - # Add JWT authentication middleware (placeholder - would need proper implementation) - # self._app.middlewares.append(self._auth_middleware) - - # Add routes - self._app.router.add_post('/v3/conversations/{path:.*}', self._handle_conversation) - - async def _auth_middleware(self, request: Request, handler): - """JWT authentication middleware (placeholder implementation).""" - # TODO: Implement proper JWT authentication - return await handler(request) - - async def _handle_conversation(self, request: Request) -> Response: - """Handle POST requests to /v3/conversations/{*text}.""" - try: - # Read request body - body = await request.text() - act = ProtocolJsonSerializer.to_object(body, Activity) - cid = act.conversation.id if act.conversation else None - - if not cid: - return web.Response(status=400, text="Missing conversation ID") - - # Add activity to the list (thread-safe) - with self._activity_locks[cid]: - self._multiple_activities[cid].append(act) - - # Create response - response_data = { - "Id": str(uuid.uuid4()) - } - - # Check if the activity is a streamed activity - if (act.entities and - any(e.type == EntityTypes.STREAM_INFO for e in act.entities)): - - entities_json = ProtocolJsonSerializer.to_json(act.entities[0]) - sact = ProtocolJsonSerializer.to_object(entities_json, StreamInfo) - - handled = self._handle_streamed_activity(act, sact, cid) - - response = web.Response( - status=200, - content_type="application/json", - text=json.dumps(response_data) - ) - - # Handle task completion (would need BotClient implementation) - if handled: - await self._complete_streaming_task(cid) - - return response - else: - # Handle non-streamed activities - if act.type != ActivityTypes.TYPING: - # Start background task with 5-second delay - asyncio.create_task(self._delayed_task_completion(cid)) - - return web.Response( - status=200, - content_type="application/json", - text=json.dumps(response_data) - ) - - except Exception as e: - return web.Response(status=500, text=str(e)) - - def _handle_streamed_activity(self, act: Activity, sact: StreamInfo, cid: str) -> bool: - """Handle streamed activity logic.""" - - # Check if activity is the final message - if sact.stream_type == StreamTypes.FINAL: - if act.type == ActivityTypes.MESSAGE: - return True - else: - raise Exception("final streamed activity should be type message") - - # Handler for streaming types which allows us to verify later if the text length has increased - elif sact.stream_type == StreamTypes.STREAMING: - if sact.stream_sequence <= 0 and act.type == ActivityTypes.TYPING: - raise Exception("streamed activity's stream sequence should be a positive number") - - # Activity is being streamed but isn't the final message - return False - - async def _complete_streaming_task(self, cid: str): - """Complete streaming task (placeholder for BotClient.TaskList logic).""" - # TODO: Implement BotClient.TaskList equivalent - # This would require the BotClient class to be implemented - if cid in self._multiple_activities: - activities = self._multiple_activities[cid].copy() - # Complete the task with activities - # BotClient.complete_task(cid, activities) - # Clean up - del self._multiple_activities[cid] - if cid in self._activity_locks: - del self._activity_locks[cid] - - async def _delayed_task_completion(self, cid: str): - """Handle delayed task completion after 5 seconds.""" - await asyncio.sleep(5.0) - # TODO: Implement BotClient.TaskList equivalent - # if BotClient.has_task(cid): - # activities = self._multiple_activities.get(cid, []) - # BotClient.complete_task(cid, activities) - - async def start(self): - """Start the web server.""" - self._runner = web.AppRunner(self._app) - await self._runner.setup() - - # Extract port from service_endpoint - port = int(self.service_endpoint.split(':')[-1]) - self._site = web.TCPSite(self._runner, 'localhost', port) - await self._site.start() - - print(f"Bot server started at {self.service_endpoint}") - - async def dispose(self): - """Cleanup resources (equivalent to DisposeAsync).""" - if self._site: - await self._site.stop() - if self._runner: - await self._runner.cleanup() - - # Restore stdout - sys.stdout = sys.__stdout__ - - -# Example usage -async def main(): - bot_response = BotResponse() - try: - await bot_response.start() - # Keep the server running - while True: - await asyncio.sleep(1) - except KeyboardInterrupt: - print("Shutting down...") - finally: - await bot_response.dispose() - - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py index 3d69d17b..6e44c31e 100644 --- a/dev/integration/src/core/client/response_client.py +++ b/dev/integration/src/core/client/response_client.py @@ -1,12 +1,10 @@ from __future__ import annotations import sys -import requests from io import StringIO from typing import Optional from threading import Lock, Thread, Event import asyncio -import weakref from aiohttp import ClientSession from aiohttp.web import Application, Request, Response, run_app @@ -17,6 +15,8 @@ ActivityTypes, ) +from ..aiohttp import AiohttpRunner + class ResponseClient: def __init__( @@ -39,16 +39,13 @@ def __init__( self._runner: Optional[AppRunner] = None self._site: Optional[TCPSite] = None - self._app.router.add_get( - "/shutdown", - self._shutdown - ) - self._app.router.add_post( "/v3/conversations/{path:.*}", self._handle_conversation ) + self._app_runner = AiohttpRunner(self._app) + @property def service_endpoint(self) -> str: return self._service_endpoint @@ -91,7 +88,7 @@ async def __aenter__(self) -> ResponseClient: await asyncio.sleep(0.2) return self - async def __aexit__(self, exc_type, exc, tb): + async def _stop_server(self): if not self._server_thread: raise RuntimeError("ResponseClient is not running.") if self._prev_stdout is not None: diff --git a/dev/integration/src/environments/__init__.py b/dev/integration/src/environments/__init__.py deleted file mode 100644 index db599f97..00000000 --- a/dev/integration/src/environments/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .aiohttp_environment import AiohttpEnvironment - -__all__ = [ - "AiohttpEnvironment" -] \ No newline at end of file From 7d91ef920c5a40ca310d12f0f217e8359341829b Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 10:47:04 -0800 Subject: [PATCH 22/81] Unit test updates --- .../src/core/aiohttp/aiohttp_environment.py | 6 +- .../src/core/aiohttp/aiohttp_runner.py | 58 ++++++++++++--- .../src/core/client/response_client.py | 70 ++++--------------- 3 files changed, 63 insertions(+), 71 deletions(-) diff --git a/dev/integration/src/core/aiohttp/aiohttp_environment.py b/dev/integration/src/core/aiohttp/aiohttp_environment.py index 70df2842..5616235c 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_environment.py +++ b/dev/integration/src/core/aiohttp/aiohttp_environment.py @@ -16,10 +16,8 @@ from microsoft_agents.authentication.msal import MsalConnectionManager from microsoft_agents.activity import load_configuration_from_env -from . import ( - ApplicationRunner, - Environment -) +from ..application_runner import ApplicationRunner +from ..environment import Environment class AiohttpEnvironment(Environment): """An environment for aiohttp-hosted agents.""" diff --git a/dev/integration/src/core/aiohttp/aiohttp_runner.py b/dev/integration/src/core/aiohttp/aiohttp_runner.py index a4520b2f..f07e6c95 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_runner.py +++ b/dev/integration/src/core/aiohttp/aiohttp_runner.py @@ -3,6 +3,7 @@ from threading import Thread, Event import asyncio +from aiohttp import ClientSession from aiohttp.web import Application, Request, Response from aiohttp.web_runner import AppRunner, TCPSite @@ -11,20 +12,34 @@ class AiohttpRunner(ApplicationRunner): """A runner for aiohttp applications.""" - def __init__(self, app: Application): + def __init__(self, + app: Application, + host: str = "localhost", + port: int = 8000 + ): assert isinstance(app, Application) super().__init__(app) + url = f"{host}:{port}" + self._host = host + self._port = port + if "http" not in url: + url = f"http://{url}" + self._url = url + self._app.router.add_get( "/shutdown", - self._shutdown + self._shutdown_route ) self._server_thread: Optional[Thread] = None self._shutdown_event = Event() self._runner: Optional[AppRunner] = None self._site: Optional[TCPSite] = None - + + @property + def url(self) -> str: + return self._url async def _start_server(self) -> None: try: @@ -48,10 +63,38 @@ async def _run_server(): except Exception as error: raise error - async def _stop_server(self) -> None: - pass + async def __aenter__(self): + if self._server_thread: + raise RuntimeError("ResponseClient is already running.") + + self._shutdown_event.clear() + self._server_thread = Thread(target=lambda: asyncio.run(self._start_server()), daemon=True) + self._server_thread.start() + + # Wait a moment to ensure the server starts + await asyncio.sleep(0.5) + + return self + + async def _stop_server(self): + if not self._server_thread: + raise RuntimeError("ResponseClient is not running.") - async def _shutdown(self, request: Request) -> Response: + try: + async with ClientSession() as session: + async with session.get(f"http://{self._host}:{self._port}/shutdown") as response: + pass # Just trigger the shutdown + except Exception: + pass # Ignore errors during shutdown request + + # Set shutdown event as fallback + self._shutdown_event.set() + + # Wait for the server thread to finish + self._server_thread.join(timeout=5.0) + self._server_thread = None + + async def _shutdown_route(self, request: Request) -> Response: """Handle shutdown request by setting the shutdown event""" self._shutdown_event.set() return Response(status=200, text="Shutdown initiated") @@ -59,9 +102,6 @@ async def _shutdown(self, request: Request) -> Response: async def __aexit__(self, exc_type, exc, tb): if not self._server_thread: raise RuntimeError("ResponseClient is not running.") - if self._prev_stdout is not None: - sys.stdout = self._prev_stdout - try: async with ClientSession() as session: async with session.get(f"http://{self._host}:{self._port}/shutdown") as response: diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py index 6e44c31e..c16a1e37 100644 --- a/dev/integration/src/core/client/response_client.py +++ b/dev/integration/src/core/client/response_client.py @@ -6,9 +6,7 @@ from threading import Lock, Thread, Event import asyncio -from aiohttp import ClientSession -from aiohttp.web import Application, Request, Response, run_app -from aiohttp.web_runner import AppRunner, TCPSite +from aiohttp.web import Application, Request, Response from microsoft_agents.activity import ( Activity, @@ -34,79 +32,35 @@ def __init__( self._service_endpoint = service_endpoint self._activities_list = [] self._activities_list_lock = Lock() - self._server_thread: Optional[Thread] = None - self._shutdown_event = Event() - self._runner: Optional[AppRunner] = None - self._site: Optional[TCPSite] = None self._app.router.add_post( "/v3/conversations/{path:.*}", self._handle_conversation ) - self._app_runner = AiohttpRunner(self._app) + self._app_runner = AiohttpRunner( + self._app, + host, + port + ) @property def service_endpoint(self) -> str: return self._service_endpoint - - def start(self) -> None: - """Start the server in the current thread""" - async def _run_server(): - self._runner = AppRunner(self._app) - await self._runner.setup() - self._site = TCPSite(self._runner, self._host, self._port) - await self._site.start() - - # Wait for shutdown signal - while not self._shutdown_event.is_set(): - await asyncio.sleep(0.1) - - # Cleanup - await self._site.stop() - await self._runner.cleanup() - - # Run the server - asyncio.run(_run_server()) - - async def _shutdown(self, request: Request) -> Response: - """Handle shutdown request by setting the shutdown event""" - self._shutdown_event.set() - return Response(status=200, text="Shutdown initiated") async def __aenter__(self) -> ResponseClient: - if self._server_thread: - raise RuntimeError("ResponseClient is already running.") - self._prev_stdout = sys.stdout sys.stdout = StringIO() - self._shutdown_event.clear() - self._server_thread = Thread(target=self.start) - self._server_thread.start() - - # Wait a bit for the server to start - await asyncio.sleep(0.2) - return self + + await self._app_runner.__aenter__() - async def _stop_server(self): - if not self._server_thread: - raise RuntimeError("ResponseClient is not running.") + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: if self._prev_stdout is not None: sys.stdout = self._prev_stdout - try: - async with ClientSession() as session: - async with session.get(f"http://{self._host}:{self._port}/shutdown") as response: - pass # Just trigger the shutdown - except Exception: - pass # Ignore errors during shutdown request - - # Set shutdown event as fallback - self._shutdown_event.set() - - # Wait for the server thread to finish - self._server_thread.join(timeout=5.0) - self._server_thread = None + await self._app_runner.__aexit__(exc_type, exc_val, exc_tb) async def _handle_conversation(self, request: Request) -> Response: try: From 87610a53b9582229912f8864813cd637312e9ee1 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 11:10:20 -0800 Subject: [PATCH 23/81] Fixing issues in refactor --- .../src/core/aiohttp/aiohttp_environment.py | 2 +- .../src/core/aiohttp/aiohttp_runner.py | 28 +++++++++---------- dev/integration/src/tests/core/_common.py | 4 +-- .../src/tests/core/test_application_runner.py | 6 ++-- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/dev/integration/src/core/aiohttp/aiohttp_environment.py b/dev/integration/src/core/aiohttp/aiohttp_environment.py index 5616235c..58e01ff1 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_environment.py +++ b/dev/integration/src/core/aiohttp/aiohttp_environment.py @@ -36,7 +36,7 @@ async def init_env(self, environ_config: dict) -> None: storage=self.storage, adapter=self.adapter, authorization=self.authorization, - **self.agents_sdk_config + **self.config ) def create_runner(self) -> ApplicationRunner: diff --git a/dev/integration/src/core/aiohttp/aiohttp_runner.py b/dev/integration/src/core/aiohttp/aiohttp_runner.py index f07e6c95..bc328fd0 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_runner.py +++ b/dev/integration/src/core/aiohttp/aiohttp_runner.py @@ -44,22 +44,20 @@ def url(self) -> str: async def _start_server(self) -> None: try: assert isinstance(self._app, Application) - async def _run_server(): - self._runner = AppRunner(self._app) - await self._runner.setup() - self._site = TCPSite(self._runner, self._host, self._port) - await self._site.start() - - # Wait for shutdown signal - while not self._shutdown_event.is_set(): - await asyncio.sleep(0.1) - - # Cleanup - await self._site.stop() - await self._runner.cleanup() + + self._runner = AppRunner(self._app) + await self._runner.setup() + self._site = TCPSite(self._runner, self._host, self._port) + await self._site.start() + + # Wait for shutdown signal + while not self._shutdown_event.is_set(): + await asyncio.sleep(0.1) + + # Cleanup + await self._site.stop() + await self._runner.cleanup() - # Run the server - asyncio.run(_run_server()) except Exception as error: raise error diff --git a/dev/integration/src/tests/core/_common.py b/dev/integration/src/tests/core/_common.py index 895caef2..ec729ca2 100644 --- a/dev/integration/src/tests/core/_common.py +++ b/dev/integration/src/tests/core/_common.py @@ -1,7 +1,7 @@ from src.core import ApplicationRunner class SimpleRunner(ApplicationRunner): - def _start_server(self) -> None: + async def _start_server(self) -> None: self._app["running"] = True @property @@ -9,5 +9,5 @@ def app(self): return self._app class OtherSimpleRunner(SimpleRunner): - def _stop_server(self) -> None: + async def _stop_server(self) -> None: self._app["running"] = False \ No newline at end of file diff --git a/dev/integration/src/tests/core/test_application_runner.py b/dev/integration/src/tests/core/test_application_runner.py index dd1d0c28..fa97c77f 100644 --- a/dev/integration/src/tests/core/test_application_runner.py +++ b/dev/integration/src/tests/core/test_application_runner.py @@ -10,9 +10,8 @@ async def test_simple_runner(self): app = {} runner = SimpleRunner(app) - async with runner as r: + async with runner: sleep(0.1) - assert runner is r assert app["running"] is True assert app["running"] is True @@ -22,9 +21,8 @@ async def test_other_simple_runner(self): app = {} runner = OtherSimpleRunner(app) - async with runner as r: + async with runner: sleep(0.1) - assert runner is r assert app["running"] is True assert app["running"] is False From ddfdd0bc9e83c6496a66e6327ba1b1168ba17141 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 12:10:32 -0800 Subject: [PATCH 24/81] Fixed TestIntegrationFromSample unit test --- dev/integration/src/core/integration.py | 120 ++---------------- .../src/core/integration_fixtures.py | 32 +++-- .../core/test_integration_from_sample.py | 4 +- 3 files changed, 33 insertions(+), 123 deletions(-) diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index df7483e8..7941bbb7 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -1,3 +1,4 @@ +import pytest from typing import Optional, TypeVar, Union, Callable, Any import aiohttp.web @@ -9,113 +10,12 @@ T = TypeVar("T", bound=type) AppT = TypeVar("AppT", bound=aiohttp.web.Application) # for future extension w/ Union - -class _Integration: - - @staticmethod - def _with_response_client(target_cls: T, host_response: bool) -> T: - """Wraps the target class to include a response client if needed.""" - - _prev_setup_method = getattr(target_cls, "setup_method", None) - _prev_teardown_method = getattr(target_cls, "teardown_method", None) - - async def setup_method(self): - if host_response: - self._response_client = ResponseClient() - await self._response_client.__aenter__() - if _prev_setup_method: - await _prev_setup_method(self) - - async def teardown_method(self): - if host_response: - await self._response_client.__aexit__(None, None, None) - if _prev_teardown_method: - await _prev_teardown_method(self) - - target_cls.setup_method = setup_method - target_cls.teardown_method = teardown_method - - return target_cls - - @staticmethod - def from_service_url(service_url: str, host_response: bool = False) -> Callable[[T], T]: - """Creates an Integration instance using a service URL.""" - - def decorator(target_cls: T) -> T: - - async def setup_method(self): - self._service_url = service_url - - async def teardown_method(self): - self._service_url = service_url - - target_cls.setup_method = setup_method - target_cls.teardown_method = teardown_method - - target_cls = _Integration._with_response_client(target_cls, host_response) - - return target_cls - - return decorator - - @staticmethod - def from_sample( - sample_cls: type[Sample], - environment_cls: type[Environment], - host_agent: bool = False, - host_response: bool = False - ) -> Callable[[T], T]: - """Creates an Integration instance using a sample and environment.""" - - def decorator(target_cls: T) -> T: - - async def setup_method(self): - self._environment = environment_cls(sample_cls.get_config()) - await self._environment.__aenter__() - - self._sample = sample_cls(self._environment) - await self._sample.__aenter__() - - async def teardown_method(self): - await self._sample.__aexit__(None, None, None) - await self._environment.__aexit__(None, None, None) - - target_cls = _Integration._with_response_client(target_cls, host_response) - - return target_cls - - return decorator - - # not supported yet - # @staticmethod - # def from_app(app: Any, host_response: bool = True) -> Callable[[T], T]: - # """Creates an Integration instance using an aiohttp application.""" - - # def decorator(target_cls: T) -> T: - - # async def setup_method(self): - - # self._app = app - - # self._runner = self._environment.create_runner() - # await self._runner.__aenter__() - - # async def teardown_method(self): - # await self._runner.__aexit__(None, None, None) - - # target_cls = _Integration._with_response_client(target_cls, host_response) - - # return target_cls - - # return decorator def integration( service_url: Optional[str] = None, sample: Optional[type[Sample]] = None, environment: Optional[type[Environment]] = None, app: Optional[AppT] = None, - host_agent: bool = False, - host_response: bool = True, ) -> Callable[[T], T]: """Factory function to create an Integration instance based on provided parameters. @@ -136,15 +36,15 @@ def integration( :return: An instance of the Integration class. """ - decorator: Callable[[T], T] + def decorator(target_cls: T) -> T: - if service_url: - decorator = _Integration.from_service_url(service_url, host_response=host_response) - elif sample and environment: - decorator = _Integration.from_sample(sample, environment, host_agent=host_agent, host_response=host_response) - # elif app: - # decorator = _Integration.from_app(app, host_response=host_response) - else: - raise ValueError("Insufficient parameters to create Integration instance.") + if service_url: + target_cls._service_url = service_url + elif sample and environment: + target_cls._sample_cls = sample + target_cls._environment_cls = environment + else: + raise ValueError("Insufficient parameters to create Integration instance.") + return target_cls return decorator \ No newline at end of file diff --git a/dev/integration/src/core/integration_fixtures.py b/dev/integration/src/core/integration_fixtures.py index e5a46321..297e5dcc 100644 --- a/dev/integration/src/core/integration_fixtures.py +++ b/dev/integration/src/core/integration_fixtures.py @@ -1,4 +1,4 @@ -from typing import TypeVar, Any, AsyncGenerator, Callable +from typing import TypeVar, Any, AsyncGenerator, Callable, Optional import pytest from .environment import Environment @@ -8,31 +8,41 @@ class IntegrationFixtures: """Provides integration test fixtures.""" + _sample_cls: Optional[type[Sample]] = None + _environment_cls: Optional[type[Environment]] = None + _environment: Environment _sample: Sample _agent_client: AgentClient _response_client: ResponseClient @pytest.fixture - def environment(self) -> Environment: + async def environment(self): """Provides the test environment instance.""" - return self._environment + assert self._environment_cls + assert self._sample_cls + environment = self._environment_cls() + await environment.init_env(await self._sample_cls.get_config()) + yield environment @pytest.fixture - def sample(self) -> Sample: + async def sample(self, environment): """Provides the sample instance.""" - return self._sample + assert environment + assert self._sample_cls + sample = self._sample_cls(environment) + await sample.init_app() + yield sample @pytest.fixture def agent_client(self) -> AgentClient: return self._agent_client + async def _create_response_client(self) -> ResponseClient: + return ResponseClient() + @pytest.fixture async def response_client(self) -> AsyncGenerator[ResponseClient, None]: """Provides the response client instance.""" - async with ResponseClient() as response_client: - yield response_client - - @pytest.fixture - def create_response_client(self) -> Callable[None, ResponseClient]: - return lambda: ResponseClient() \ No newline at end of file + async with await self._create_response_client() as response_client: + yield response_client \ No newline at end of file diff --git a/dev/integration/src/tests/core/test_integration_from_sample.py b/dev/integration/src/tests/core/test_integration_from_sample.py index 65a160aa..15b30bc5 100644 --- a/dev/integration/src/tests/core/test_integration_from_sample.py +++ b/dev/integration/src/tests/core/test_integration_from_sample.py @@ -10,7 +10,7 @@ Sample ) -from ._common import SimpleRunner, OtherSimpleRunner +from ._common import SimpleRunner class SimpleEnvironment(Environment): """A simple implementation of the Environment for testing.""" @@ -54,4 +54,4 @@ async def test_sample_integration(self, sample, environment): assert sample.other_data == 1 runner = environment.create_runner() - assert runner.app == {"running": True} \ No newline at end of file + assert runner.app == {"sample_key": "sample_value"} \ No newline at end of file From dd0a87d7d62c60247b9d38c95252d04ef7918590 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 12:14:34 -0800 Subject: [PATCH 25/81] Another commit --- .../src/core/integration_fixtures.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/dev/integration/src/core/integration_fixtures.py b/dev/integration/src/core/integration_fixtures.py index 297e5dcc..affa4533 100644 --- a/dev/integration/src/core/integration_fixtures.py +++ b/dev/integration/src/core/integration_fixtures.py @@ -1,4 +1,4 @@ -from typing import TypeVar, Any, AsyncGenerator, Callable, Optional +from typing import TypeVar, Any, AsyncGenerator, Callable, Optional, Generator import pytest from .environment import Environment @@ -34,9 +34,22 @@ async def sample(self, environment): await sample.init_app() yield sample + def create_agent_client(self) -> AgentClient: + agent_client = AgentClient( + messaging_endpoint="http://localhost:8000/api/messages", + cid="test-cid", + client_id="test-client-id", + tenant_id="test-tenant-id", + client_secret="test-client-secret", + service_url="http://localhost:8000/api/messages" + ) + return agent_client + @pytest.fixture - def agent_client(self) -> AgentClient: - return self._agent_client + async def agent_client(self) -> AsyncGenerator[AgentClient, None]: + agent_client = self.create_agent_client() + yield agent_client + await agent_client.close() async def _create_response_client(self) -> ResponseClient: return ResponseClient() From 6a4e8ef7631ad63592d48bfd65e6e7e2973255f9 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 12:25:21 -0800 Subject: [PATCH 26/81] Reorganizing files --- dev/integration/src/core/__init__.py | 3 +- dev/integration/src/core/integration.py | 78 ++++++++++++++++++- .../src/core/integration_fixtures.py | 61 --------------- 3 files changed, 78 insertions(+), 64 deletions(-) delete mode 100644 dev/integration/src/core/integration_fixtures.py diff --git a/dev/integration/src/core/__init__.py b/dev/integration/src/core/__init__.py index ff94868f..1edefe97 100644 --- a/dev/integration/src/core/__init__.py +++ b/dev/integration/src/core/__init__.py @@ -4,8 +4,7 @@ ResponseClient, ) from .environment import Environment -from .integration import integration -from .integration_fixtures import IntegrationFixtures +from .integration import integration, IntegrationFixtures from .sample import Sample diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index 7941bbb7..2736622c 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -1,5 +1,12 @@ import pytest -from typing import Optional, TypeVar, Union, Callable, Any +from typing import ( + Optional, + TypeVar, + Union, + Callable, + Any, + AsyncGenerator, +) import aiohttp.web @@ -11,11 +18,77 @@ T = TypeVar("T", bound=type) AppT = TypeVar("AppT", bound=aiohttp.web.Application) # for future extension w/ Union +class IntegrationFixtures: + """Provides integration test fixtures.""" + + _sample_cls: Optional[type[Sample]] = None + _environment_cls: Optional[type[Environment]] = None + + _config: dict[str, Any] = {} + + _service_url: Optional[str] = None + _messaging_endpoint: Optional[str] = None + _cid: Optional[str] = None + _client_id: Optional[str] = None + _tenant_id: Optional[str] = None + _client_secret: Optional[str] = None + + _environment: Environment + _sample: Sample + _agent_client: AgentClient + _response_client: ResponseClient + + @pytest.fixture + async def environment(self): + """Provides the test environment instance.""" + assert self._environment_cls + assert self._sample_cls + environment = self._environment_cls() + await environment.init_env(await self._sample_cls.get_config()) + yield environment + + @pytest.fixture + async def sample(self, environment): + """Provides the sample instance.""" + assert environment + assert self._sample_cls + sample = self._sample_cls(environment) + await sample.init_app() + yield sample + + def create_agent_client(self) -> AgentClient: + if not self._config: + self._config = {} + agent_client = AgentClient( + messaging_endpoint=self._messaging_endpoint or self._config.get("messaging_endpoint", ""), + cid=self._cid or self._config.get("cid", ""), + client_id=self._client_id or self._config.get("client_id", ""), + tenant_id=self._tenant_id or self._config.get("tenant_id", ""), + client_secret=self._client_secret or self._config.get("client_secret", ""), + ) + return agent_client + + @pytest.fixture + async def agent_client(self) -> AsyncGenerator[AgentClient, None]: + agent_client = self.create_agent_client() + yield agent_client + await agent_client.close() + + async def _create_response_client(self) -> ResponseClient: + return ResponseClient() + + @pytest.fixture + async def response_client(self) -> AsyncGenerator[ResponseClient, None]: + """Provides the response client instance.""" + async with await self._create_response_client() as response_client: + yield response_client + def integration( service_url: Optional[str] = None, sample: Optional[type[Sample]] = None, environment: Optional[type[Environment]] = None, app: Optional[AppT] = None, + **kwargs ) -> Callable[[T], T]: """Factory function to create an Integration instance based on provided parameters. @@ -45,6 +118,9 @@ def decorator(target_cls: T) -> T: target_cls._environment_cls = environment else: raise ValueError("Insufficient parameters to create Integration instance.") + + target_cls._config = kwargs + return target_cls return decorator \ No newline at end of file diff --git a/dev/integration/src/core/integration_fixtures.py b/dev/integration/src/core/integration_fixtures.py deleted file mode 100644 index affa4533..00000000 --- a/dev/integration/src/core/integration_fixtures.py +++ /dev/null @@ -1,61 +0,0 @@ -from typing import TypeVar, Any, AsyncGenerator, Callable, Optional, Generator -import pytest - -from .environment import Environment -from .sample import Sample -from .client import AgentClient, ResponseClient - -class IntegrationFixtures: - """Provides integration test fixtures.""" - - _sample_cls: Optional[type[Sample]] = None - _environment_cls: Optional[type[Environment]] = None - - _environment: Environment - _sample: Sample - _agent_client: AgentClient - _response_client: ResponseClient - - @pytest.fixture - async def environment(self): - """Provides the test environment instance.""" - assert self._environment_cls - assert self._sample_cls - environment = self._environment_cls() - await environment.init_env(await self._sample_cls.get_config()) - yield environment - - @pytest.fixture - async def sample(self, environment): - """Provides the sample instance.""" - assert environment - assert self._sample_cls - sample = self._sample_cls(environment) - await sample.init_app() - yield sample - - def create_agent_client(self) -> AgentClient: - agent_client = AgentClient( - messaging_endpoint="http://localhost:8000/api/messages", - cid="test-cid", - client_id="test-client-id", - tenant_id="test-tenant-id", - client_secret="test-client-secret", - service_url="http://localhost:8000/api/messages" - ) - return agent_client - - @pytest.fixture - async def agent_client(self) -> AsyncGenerator[AgentClient, None]: - agent_client = self.create_agent_client() - yield agent_client - await agent_client.close() - - async def _create_response_client(self) -> ResponseClient: - return ResponseClient() - - @pytest.fixture - async def response_client(self) -> AsyncGenerator[ResponseClient, None]: - """Provides the response client instance.""" - async with await self._create_response_client() as response_client: - yield response_client \ No newline at end of file From 7793790d59bfdb81a493090dab44dc3a69a209e3 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 12:53:49 -0800 Subject: [PATCH 27/81] Completed TestIntegrationFromServiceUrl unit tests --- .../src/core/client/agent_client.py | 15 ++++++---- dev/integration/src/core/integration.py | 21 ++++++++++---- dev/integration/src/core/utils.py | 9 ++++++ .../core/test_integration_from_service_url.py | 29 +++++++++++-------- 4 files changed, 52 insertions(+), 22 deletions(-) create mode 100644 dev/integration/src/core/utils.py diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py index 97084137..e08fb4d4 100644 --- a/dev/integration/src/core/client/agent_client.py +++ b/dev/integration/src/core/client/agent_client.py @@ -62,11 +62,16 @@ async def get_access_token(self) -> str: async def _init_client(self) -> None: if not self._client: - token = await self.get_access_token() - self._headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json" - } + if self._client_secret: + token = await self.get_access_token() + self._headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + else: + self._headers = { + "Content-Type": "application/json" + } self._client = ClientSession( base_url=self._messaging_endpoint, diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index 2736622c..2daa6407 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -14,6 +14,7 @@ from .environment import Environment from .client import AgentClient, ResponseClient from .sample import Sample +from .utils import get_host_and_port T = TypeVar("T", bound=type) AppT = TypeVar("AppT", bound=aiohttp.web.Application) # for future extension w/ Union @@ -38,6 +39,14 @@ class IntegrationFixtures: _agent_client: AgentClient _response_client: ResponseClient + @property + def service_url(self) -> str: + return self._service_url or self._config.get("service_url", "") + + @property + def messaging_endpoint(self) -> str: + return self._messaging_endpoint or self._config.get("messaging_endpoint", "") + @pytest.fixture async def environment(self): """Provides the test environment instance.""" @@ -60,7 +69,7 @@ def create_agent_client(self) -> AgentClient: if not self._config: self._config = {} agent_client = AgentClient( - messaging_endpoint=self._messaging_endpoint or self._config.get("messaging_endpoint", ""), + messaging_endpoint=self.messaging_endpoint, cid=self._cid or self._config.get("cid", ""), client_id=self._client_id or self._config.get("client_id", ""), tenant_id=self._tenant_id or self._config.get("tenant_id", ""), @@ -75,7 +84,9 @@ async def agent_client(self) -> AsyncGenerator[AgentClient, None]: await agent_client.close() async def _create_response_client(self) -> ResponseClient: - return ResponseClient() + host, port = get_host_and_port(self.service_url) + assert host and port + return ResponseClient(host=host, port=port) @pytest.fixture async def response_client(self) -> AsyncGenerator[ResponseClient, None]: @@ -84,7 +95,7 @@ async def response_client(self) -> AsyncGenerator[ResponseClient, None]: yield response_client def integration( - service_url: Optional[str] = None, + messaging_endpoint: Optional[str] = None, sample: Optional[type[Sample]] = None, environment: Optional[type[Environment]] = None, app: Optional[AppT] = None, @@ -111,8 +122,8 @@ def integration( def decorator(target_cls: T) -> T: - if service_url: - target_cls._service_url = service_url + if messaging_endpoint: + target_cls._messaging_endpoint = messaging_endpoint elif sample and environment: target_cls._sample_cls = sample target_cls._environment_cls = environment diff --git a/dev/integration/src/core/utils.py b/dev/integration/src/core/utils.py new file mode 100644 index 00000000..1d5798ae --- /dev/null +++ b/dev/integration/src/core/utils.py @@ -0,0 +1,9 @@ +from urllib.parse import urlparse + +def get_host_and_port(url): + """Extract host and port from a URL.""" + + parsed_url = urlparse(url) + host = parsed_url.hostname or "localhost" + port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) + return host, port \ No newline at end of file diff --git a/dev/integration/src/tests/core/test_integration_from_service_url.py b/dev/integration/src/tests/core/test_integration_from_service_url.py index 773f5b76..9eac4e62 100644 --- a/dev/integration/src/tests/core/test_integration_from_service_url.py +++ b/dev/integration/src/tests/core/test_integration_from_service_url.py @@ -1,14 +1,15 @@ import pytest import asyncio +import requests from copy import copy -from aioresponses import aioresponses +from aioresponses import aioresponses, CallbackResult from src.core import ( integration, IntegrationFixtures ) -@integration(service_url="http://localhost:8000/api/messages") +@integration(messaging_endpoint="http://localhost:8000/api/messages/", service_url="http://localhost:8001/") class TestIntegrationFromServiceURL(IntegrationFixtures): @pytest.mark.asyncio @@ -17,11 +18,10 @@ async def test_service_url_integration(self, agent_client): with aioresponses() as mocked: - mocked.post("http://localhost:8000/api/messages", status=200, body="Service response") + mocked.post(self.messaging_endpoint, status=200, body="Service response") - response = await agent_client.send_activity("Hello, service!") - assert response.status_code == 200 - assert "service" in response.text.lower() + res = await agent_client.send_activity("Hello, service!") + assert res == "Service response" @pytest.mark.asyncio async def test_service_url_integration_with_response_side_effect(self, agent_client, response_client): @@ -29,13 +29,18 @@ async def test_service_url_integration_with_response_side_effect(self, agent_cli with aioresponses() as mocked: - mocked.post("http://localhost:8000/api/messages", status=200, body="Service response") + def callback(url, **kwargs): + a = requests.post(f"{self.service_url}/v3/conversations/test-conv", json=kwargs.get("json")) + return CallbackResult(status=200, body="Service response") - response = await agent_client.send_activity("Hello, service!") - assert response.status_code == 200 - assert "service" in response.text.lower() + mocked.post(self.messaging_endpoint, callback=callback) + + res = await agent_client.send_activity("Hello, service!") + assert res == "Service response" await asyncio.sleep(1) - res = await response_client.pop() - assert len(res) == 1 \ No newline at end of file + activities = await response_client.pop() + assert len(activities) == 1 + assert activities[0].type == "message" + assert activities[0].text == "Hello, service!" \ No newline at end of file From 3ebbd4463be132bd641b24f615001bfa2bfceb74 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 12:54:21 -0800 Subject: [PATCH 28/81] Reformatting with black --- .../integration_tests/test_quickstart.py | 9 +-- dev/integration/src/core/__init__.py | 2 +- dev/integration/src/core/aiohttp/__init__.py | 5 +- .../src/core/aiohttp/aiohttp_environment.py | 20 +++---- .../src/core/aiohttp/aiohttp_runner.py | 56 +++++++++--------- .../src/core/application_runner.py | 15 +++-- dev/integration/src/core/client/__init__.py | 2 +- .../src/core/client/agent_client.py | 58 +++++++++---------- .../src/core/client/auto_client.py | 6 +- .../src/core/client/response_client.py | 28 ++++----- dev/integration/src/core/environment.py | 3 +- dev/integration/src/core/integration.py | 18 +++--- dev/integration/src/core/sample.py | 2 +- dev/integration/src/core/utils.py | 3 +- dev/integration/src/samples/__init__.py | 4 +- .../src/samples/quickstart_sample.py | 12 +--- dev/integration/src/tests/core/_common.py | 4 +- .../src/tests/core/client/_common.py | 2 +- .../tests/core/client/test_agent_client.py | 46 +++++++++++---- .../tests/core/client/test_response_client.py | 11 ++-- .../src/tests/core/test_application_runner.py | 7 ++- .../core/test_integration_from_sample.py | 21 +++---- .../core/test_integration_from_service_url.py | 22 ++++--- 23 files changed, 191 insertions(+), 165 deletions(-) diff --git a/dev/integration/integration_tests/test_quickstart.py b/dev/integration/integration_tests/test_quickstart.py index 04a28946..3f07cbdc 100644 --- a/dev/integration/integration_tests/test_quickstart.py +++ b/dev/integration/integration_tests/test_quickstart.py @@ -14,8 +14,9 @@ async def test_quickstart_functionality(self, agent_client, response_client): response = (await response_client.pop())[0] assert "hello" in response.text.lower() + # class TestQuickstart(TestSuiteBase): - + # @pytest.mark.asyncio # async def test_quickstart_functionality(self): # pass @@ -33,11 +34,11 @@ async def test_quickstart_functionality(self, agent_client, response_client): # @integration(app=None) # (endpoint="alternative") # class TestQuickstartAlternative: - + # @pytest.mark.asyncio # async def test_hello(self, agent_client, response_client): - + # agent_client.send("hi") # await asyncio.sleep(10) -# assert receiver.has_activity("hello") \ No newline at end of file +# assert receiver.has_activity("hello") diff --git a/dev/integration/src/core/__init__.py b/dev/integration/src/core/__init__.py index 1edefe97..1736ceeb 100644 --- a/dev/integration/src/core/__init__.py +++ b/dev/integration/src/core/__init__.py @@ -16,4 +16,4 @@ "integration", "IntegrationFixtures", "Sample", -] \ No newline at end of file +] diff --git a/dev/integration/src/core/aiohttp/__init__.py b/dev/integration/src/core/aiohttp/__init__.py index 236bc136..82d2d1d0 100644 --- a/dev/integration/src/core/aiohttp/__init__.py +++ b/dev/integration/src/core/aiohttp/__init__.py @@ -1,7 +1,4 @@ from .aiohttp_environment import AiohttpEnvironment from .aiohttp_runner import AiohttpRunner -__all__ = [ - "AiohttpEnvironment", - "AiohttpRunner" -] \ No newline at end of file +__all__ = ["AiohttpEnvironment", "AiohttpRunner"] diff --git a/dev/integration/src/core/aiohttp/aiohttp_environment.py b/dev/integration/src/core/aiohttp/aiohttp_environment.py index 58e01ff1..fcbb90ed 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_environment.py +++ b/dev/integration/src/core/aiohttp/aiohttp_environment.py @@ -1,11 +1,10 @@ - from tkinter import E from aiohttp.web import Request, Response, Application, run_app from microsoft_agents.hosting.aiohttp import ( CloudAdapter, jwt_authorization_middleware, - start_agent_process + start_agent_process, ) from microsoft_agents.hosting.core import ( Authorization, @@ -19,6 +18,7 @@ from ..application_runner import ApplicationRunner from ..environment import Environment + class AiohttpEnvironment(Environment): """An environment for aiohttp-hosted agents.""" @@ -30,7 +30,9 @@ async def init_env(self, environ_config: dict) -> None: self.storage = MemoryStorage() self.connection_manager = MsalConnectionManager(**self.config) self.adapter = CloudAdapter(connection_manager=self.connection_manager) - self.authorization = Authorization(self.storage, self.connection_manager, **self.config) + self.authorization = Authorization( + self.storage, self.connection_manager, **self.config + ) self.agent_application = AgentApplication[TurnState]( storage=self.storage, @@ -38,17 +40,13 @@ async def init_env(self, environ_config: dict) -> None: authorization=self.authorization, **self.config ) - + def create_runner(self) -> ApplicationRunner: - + async def entry_point(req: Request) -> Response: agent: AgentApplication = req.app["agent_app"] adapter: CloudAdapter = req.app["adapter"] - return await start_agent_process( - req, - agent, - adapter - ) + return await start_agent_process(req, agent, adapter) APP = Application(middlewares=[jwt_authorization_middleware]) APP.router.add_post("/api/messages", entry_point) @@ -56,4 +54,4 @@ async def entry_point(req: Request) -> Response: APP["agent_app"] = self.agent_application APP["adapter"] = self.adapter - return AiohttpRunner(APP) \ No newline at end of file + return AiohttpRunner(APP) diff --git a/dev/integration/src/core/aiohttp/aiohttp_runner.py b/dev/integration/src/core/aiohttp/aiohttp_runner.py index bc328fd0..3b6780d4 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_runner.py +++ b/dev/integration/src/core/aiohttp/aiohttp_runner.py @@ -9,14 +9,11 @@ from ..application_runner import ApplicationRunner + class AiohttpRunner(ApplicationRunner): """A runner for aiohttp applications.""" - - def __init__(self, - app: Application, - host: str = "localhost", - port: int = 8000 - ): + + def __init__(self, app: Application, host: str = "localhost", port: int = 8000): assert isinstance(app, Application) super().__init__(app) @@ -27,16 +24,13 @@ def __init__(self, url = f"http://{url}" self._url = url - self._app.router.add_get( - "/shutdown", - self._shutdown_route - ) + self._app.router.add_get("/shutdown", self._shutdown_route) self._server_thread: Optional[Thread] = None self._shutdown_event = Event() self._runner: Optional[AppRunner] = None self._site: Optional[TCPSite] = None - + @property def url(self) -> str: return self._url @@ -49,45 +43,49 @@ async def _start_server(self) -> None: await self._runner.setup() self._site = TCPSite(self._runner, self._host, self._port) await self._site.start() - + # Wait for shutdown signal while not self._shutdown_event.is_set(): await asyncio.sleep(0.1) - + # Cleanup await self._site.stop() await self._runner.cleanup() - + except Exception as error: raise error - + async def __aenter__(self): if self._server_thread: raise RuntimeError("ResponseClient is already running.") - + self._shutdown_event.clear() - self._server_thread = Thread(target=lambda: asyncio.run(self._start_server()), daemon=True) + self._server_thread = Thread( + target=lambda: asyncio.run(self._start_server()), daemon=True + ) self._server_thread.start() - + # Wait a moment to ensure the server starts await asyncio.sleep(0.5) - + return self - + async def _stop_server(self): if not self._server_thread: raise RuntimeError("ResponseClient is not running.") try: async with ClientSession() as session: - async with session.get(f"http://{self._host}:{self._port}/shutdown") as response: + async with session.get( + f"http://{self._host}:{self._port}/shutdown" + ) as response: pass # Just trigger the shutdown except Exception: pass # Ignore errors during shutdown request - + # Set shutdown event as fallback self._shutdown_event.set() - + # Wait for the server thread to finish self._server_thread.join(timeout=5.0) self._server_thread = None @@ -96,20 +94,22 @@ async def _shutdown_route(self, request: Request) -> Response: """Handle shutdown request by setting the shutdown event""" self._shutdown_event.set() return Response(status=200, text="Shutdown initiated") - + async def __aexit__(self, exc_type, exc, tb): if not self._server_thread: raise RuntimeError("ResponseClient is not running.") try: async with ClientSession() as session: - async with session.get(f"http://{self._host}:{self._port}/shutdown") as response: + async with session.get( + f"http://{self._host}:{self._port}/shutdown" + ) as response: pass # Just trigger the shutdown except Exception: pass # Ignore errors during shutdown request - + # Set shutdown event as fallback self._shutdown_event.set() - + # Wait for the server thread to finish self._server_thread.join(timeout=5.0) - self._server_thread = None \ No newline at end of file + self._server_thread = None diff --git a/dev/integration/src/core/application_runner.py b/dev/integration/src/core/application_runner.py index cea6157c..ebbc56f9 100644 --- a/dev/integration/src/core/application_runner.py +++ b/dev/integration/src/core/application_runner.py @@ -3,17 +3,20 @@ from typing import Any, Optional from threading import Thread + class ApplicationRunner(ABC): """Base class for application runners.""" - + def __init__(self, app: Any): self._app = app self._thread: Optional[Thread] = None @abstractmethod async def _start_server(self) -> None: - raise NotImplementedError("Start server method must be implemented by subclasses") - + raise NotImplementedError( + "Start server method must be implemented by subclasses" + ) + async def _stop_server(self) -> None: pass @@ -21,10 +24,10 @@ async def __aenter__(self) -> None: if self._thread: raise RuntimeError("Server is already running") - + def target(): asyncio.run(self._start_server()) - + self._thread = Thread(target=target, daemon=True) self._thread.start() @@ -36,4 +39,4 @@ async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: self._thread.join() self._thread = None else: - raise RuntimeError("Server is not running") \ No newline at end of file + raise RuntimeError("Server is not running") diff --git a/dev/integration/src/core/client/__init__.py b/dev/integration/src/core/client/__init__.py index 01c71621..1d59411e 100644 --- a/dev/integration/src/core/client/__init__.py +++ b/dev/integration/src/core/client/__init__.py @@ -4,4 +4,4 @@ __all__ = [ "AgentClient", "ResponseClient", -] \ No newline at end of file +] diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py index e08fb4d4..9aac76be 100644 --- a/dev/integration/src/core/client/agent_client.py +++ b/dev/integration/src/core/client/agent_client.py @@ -5,26 +5,23 @@ from aiohttp import ClientSession -from microsoft_agents.activity import ( - Activity, - ActivityTypes, - DeliveryModes -) +from microsoft_agents.activity import Activity, ActivityTypes, DeliveryModes from msal import ConfidentialClientApplication + class AgentClient: def __init__( - self, - messaging_endpoint: str, - cid: str, - client_id: str, - tenant_id: str, - client_secret: str, - service_url: Optional[str] = None, - default_timeout: float = 5.0 - ): + self, + messaging_endpoint: str, + cid: str, + client_id: str, + tenant_id: str, + client_secret: str, + service_url: Optional[str] = None, + default_timeout: float = 5.0, + ): self._messaging_endpoint = messaging_endpoint self._cid = cid self._client_id = client_id @@ -49,12 +46,10 @@ async def get_access_token(self) -> str: msal_app = ConfidentialClientApplication( client_id=self._client_id, client_credential=self._client_secret, - authority=f"https://login.microsoftonline.com/{self._tenant_id}" + authority=f"https://login.microsoftonline.com/{self._tenant_id}", ) - res = msal_app.acquire_token_for_client( - scopes=[f"{self._client_id}/.default"] - ) + res = msal_app.acquire_token_for_client(scopes=[f"{self._client_id}/.default"]) token = res.get("access_token") if res else None if not token: raise Exception("Could not obtain access token") @@ -66,16 +61,13 @@ async def _init_client(self) -> None: token = await self.get_access_token() self._headers = { "Authorization": f"Bearer {token}", - "Content-Type": "application/json" + "Content-Type": "application/json", } else: - self._headers = { - "Content-Type": "application/json" - } + self._headers = {"Content-Type": "application/json"} self._client = ClientSession( - base_url=self._messaging_endpoint, - headers=self._headers + base_url=self._messaging_endpoint, headers=self._headers ) async def send_request(self, activity: Activity) -> str: @@ -92,13 +84,13 @@ async def send_request(self, activity: Activity) -> str: async with self._client.post( self._messaging_endpoint, headers=self._headers, - json=activity.model_dump(by_alias=True, exclude_none=True) + json=activity.model_dump(by_alias=True, exclude_none=True), ) as response: if not response.ok: raise Exception(f"Failed to send activity: {response.status}") content = await response.text() return content - + def _to_activity(self, activity_or_text: Activity | str) -> Activity: if isinstance(activity_or_text, str): activity = Activity( @@ -109,13 +101,17 @@ def _to_activity(self, activity_or_text: Activity | str) -> Activity: else: return cast(Activity, activity_or_text) - async def send_activity(self, activity_or_text: Activity | str, timeout: Optional[float] = None) -> str: + async def send_activity( + self, activity_or_text: Activity | str, timeout: Optional[float] = None + ) -> str: timeout = timeout or self._default_timeout activity = self._to_activity(activity_or_text) content = await self.send_request(activity) return content - - async def send_expect_replies(self, activity_or_text: Activity | str, timeout: Optional[float] = None) -> list[Activity]: + + async def send_expect_replies( + self, activity_or_text: Activity | str, timeout: Optional[float] = None + ) -> list[Activity]: timeout = timeout or self._default_timeout activity = self._to_activity(activity_or_text) activity.delivery_mode = DeliveryModes.expect_replies @@ -125,8 +121,8 @@ async def send_expect_replies(self, activity_or_text: Activity | str, timeout: O activities_data = json.loads(content).get("activities", []) activities = [Activity.model_validate(act) for act in activities_data] return activities - + async def close(self) -> None: if self._client: await self._client.close() - self._client = None \ No newline at end of file + self._client = None diff --git a/dev/integration/src/core/client/auto_client.py b/dev/integration/src/core/client/auto_client.py index 721994a6..dcea531b 100644 --- a/dev/integration/src/core/client/auto_client.py +++ b/dev/integration/src/core/client/auto_client.py @@ -3,7 +3,7 @@ # from ..agent_client import AgentClient # class AutoClient: - + # def __init__(self, agent_client: AgentClient): # self._agent_client = agent_client @@ -11,8 +11,8 @@ # pass # async def run(self, max_turns: int = 10, time_between_turns: float = 2.0) -> None: - + # for i in range(max_turns): # await self._agent_client.send_activity( # Activity(type="message", text=self.generate_message()) -# ) \ No newline at end of file +# ) diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py index c16a1e37..d93bfb80 100644 --- a/dev/integration/src/core/client/response_client.py +++ b/dev/integration/src/core/client/response_client.py @@ -15,8 +15,9 @@ from ..aiohttp import AiohttpRunner + class ResponseClient: - + def __init__( self, host: str = "localhost", @@ -34,15 +35,10 @@ def __init__( self._activities_list_lock = Lock() self._app.router.add_post( - "/v3/conversations/{path:.*}", - self._handle_conversation + "/v3/conversations/{path:.*}", self._handle_conversation ) - self._app_runner = AiohttpRunner( - self._app, - host, - port - ) + self._app_runner = AiohttpRunner(self._app, host, port) @property def service_endpoint(self) -> str: @@ -51,11 +47,11 @@ def service_endpoint(self) -> str: async def __aenter__(self) -> ResponseClient: self._prev_stdout = sys.stdout sys.stdout = StringIO() - + await self._app_runner.__aenter__() return self - + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: if self._prev_stdout is not None: sys.stdout = self._prev_stdout @@ -67,7 +63,9 @@ async def _handle_conversation(self, request: Request) -> Response: data = await request.json() activity = Activity.model_validate(data) - conversation_id = activity.conversation.id if activity.conversation else None + conversation_id = ( + activity.conversation.id if activity.conversation else None + ) with self._activities_list_lock: self._activities_list.append(activity) @@ -82,11 +80,13 @@ async def _handle_conversation(self, request: Request) -> Response: except Exception as e: return Response(status=500, text=str(e)) - async def _handle_streamed_activity(self, activity: Activity, *args, **kwargs) -> bool: + async def _handle_streamed_activity( + self, activity: Activity, *args, **kwargs + ) -> bool: raise NotImplementedError("_handle_streamed_activity is not implemented yet.") - + async def pop(self) -> list[Activity]: with self._activities_list_lock: activities = self._activities_list[:] self._activities_list.clear() - return activities \ No newline at end of file + return activities diff --git a/dev/integration/src/core/environment.py b/dev/integration/src/core/environment.py index 628460a6..2c9b1ae1 100644 --- a/dev/integration/src/core/environment.py +++ b/dev/integration/src/core/environment.py @@ -12,6 +12,7 @@ from .application_runner import ApplicationRunner + class Environment(ABC): """A sample data object for integration tests.""" @@ -33,4 +34,4 @@ async def init_env(self, environ_config: dict) -> None: @abstractmethod def create_runner(self) -> ApplicationRunner: """Create an application runner for the environment.""" - raise NotImplementedError() \ No newline at end of file + raise NotImplementedError() diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index 2daa6407..5a6ee5af 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -17,8 +17,9 @@ from .utils import get_host_and_port T = TypeVar("T", bound=type) -AppT = TypeVar("AppT", bound=aiohttp.web.Application) # for future extension w/ Union - +AppT = TypeVar("AppT", bound=aiohttp.web.Application) # for future extension w/ Union + + class IntegrationFixtures: """Provides integration test fixtures.""" @@ -55,7 +56,7 @@ async def environment(self): environment = self._environment_cls() await environment.init_env(await self._sample_cls.get_config()) yield environment - + @pytest.fixture async def sample(self, environment): """Provides the sample instance.""" @@ -64,7 +65,7 @@ async def sample(self, environment): sample = self._sample_cls(environment) await sample.init_app() yield sample - + def create_agent_client(self) -> AgentClient: if not self._config: self._config = {} @@ -93,7 +94,8 @@ async def response_client(self) -> AsyncGenerator[ResponseClient, None]: """Provides the response client instance.""" async with await self._create_response_client() as response_client: yield response_client - + + def integration( messaging_endpoint: Optional[str] = None, sample: Optional[type[Sample]] = None, @@ -110,7 +112,7 @@ def integration( If a service URL is provided, it creates the Integration using that. If both sample and environment are provided, it creates the Integration using them. If an aiohttp application is provided, it creates the Integration using that. - + :param cls: The Integration class type. :param service_url: Optional service URL to connect to. :param sample: Optional Sample instance. @@ -133,5 +135,5 @@ def decorator(target_cls: T) -> T: target_cls._config = kwargs return target_cls - - return decorator \ No newline at end of file + + return decorator diff --git a/dev/integration/src/core/sample.py b/dev/integration/src/core/sample.py index 31148791..6dde3668 100644 --- a/dev/integration/src/core/sample.py +++ b/dev/integration/src/core/sample.py @@ -16,4 +16,4 @@ async def get_config(cls) -> dict: @abstractmethod async def init_app(self): - """Initialize the application for the sample.""" \ No newline at end of file + """Initialize the application for the sample.""" diff --git a/dev/integration/src/core/utils.py b/dev/integration/src/core/utils.py index 1d5798ae..82d4c528 100644 --- a/dev/integration/src/core/utils.py +++ b/dev/integration/src/core/utils.py @@ -1,9 +1,10 @@ from urllib.parse import urlparse + def get_host_and_port(url): """Extract host and port from a URL.""" parsed_url = urlparse(url) host = parsed_url.hostname or "localhost" port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) - return host, port \ No newline at end of file + return host, port diff --git a/dev/integration/src/samples/__init__.py b/dev/integration/src/samples/__init__.py index 4963a4c7..a77ee72e 100644 --- a/dev/integration/src/samples/__init__.py +++ b/dev/integration/src/samples/__init__.py @@ -1,5 +1,3 @@ from .quickstart_sample import QuickstartSample -__all__ = [ - "QuickstartSample" -] \ No newline at end of file +__all__ = ["QuickstartSample"] diff --git a/dev/integration/src/samples/quickstart_sample.py b/dev/integration/src/samples/quickstart_sample.py index 05a5ebc9..e3c78cc3 100644 --- a/dev/integration/src/samples/quickstart_sample.py +++ b/dev/integration/src/samples/quickstart_sample.py @@ -4,14 +4,11 @@ from microsoft_agents.activity import ConversationUpdateTypes -from microsoft_agents.hosting.core import ( - AgentApplication, - TurnContext, - TurnState -) +from microsoft_agents.hosting.core import AgentApplication, TurnContext, TurnState from ..core.sample import Sample - + + class QuickstartSample(Sample): """A quickstart sample implementation.""" @@ -27,17 +24,14 @@ async def on_members_added(context: TurnContext, state: TurnState) -> None: "This agent is designed to be a starting point for your own agent development." ) - @app.message(re.compile(r"^hello$")) async def on_hello(context: TurnContext, state: TurnState) -> None: await context.send_activity("Hello!") - @app.activity("message") async def on_message(context: TurnContext, state: TurnState) -> None: await context.send_activity(f"you said: {context.activity.text}") - @app.error async def on_error(context: TurnContext, error: Exception): # This check writes out errors to console log .vs. app insights. diff --git a/dev/integration/src/tests/core/_common.py b/dev/integration/src/tests/core/_common.py index ec729ca2..c8dd0098 100644 --- a/dev/integration/src/tests/core/_common.py +++ b/dev/integration/src/tests/core/_common.py @@ -1,5 +1,6 @@ from src.core import ApplicationRunner + class SimpleRunner(ApplicationRunner): async def _start_server(self) -> None: self._app["running"] = True @@ -8,6 +9,7 @@ async def _start_server(self) -> None: def app(self): return self._app + class OtherSimpleRunner(SimpleRunner): async def _stop_server(self) -> None: - self._app["running"] = False \ No newline at end of file + self._app["running"] = False diff --git a/dev/integration/src/tests/core/client/_common.py b/dev/integration/src/tests/core/client/_common.py index e6df31e4..d6bdf917 100644 --- a/dev/integration/src/tests/core/client/_common.py +++ b/dev/integration/src/tests/core/client/_common.py @@ -7,4 +7,4 @@ class DEFAULTS: cid = "test-cid" client_id = "test-client-id" tenant_id = "test-tenant-id" - client_secret = "test-client-secret" \ No newline at end of file + client_secret = "test-client-secret" diff --git a/dev/integration/src/tests/core/client/test_agent_client.py b/dev/integration/src/tests/core/client/test_agent_client.py index a5a0ea7e..89ea467d 100644 --- a/dev/integration/src/tests/core/client/test_agent_client.py +++ b/dev/integration/src/tests/core/client/test_agent_client.py @@ -12,6 +12,7 @@ from ._common import DEFAULTS + class TestAgentClient: @pytest.fixture @@ -22,7 +23,7 @@ async def agent_client(self): client_id=DEFAULTS.client_id, tenant_id=DEFAULTS.tenant_id, client_secret=DEFAULTS.client_secret, - service_url=DEFAULTS.service_url + service_url=DEFAULTS.service_url, ) yield client await client.close() @@ -34,32 +35,53 @@ def aioresponses_mock(self): @pytest.mark.asyncio async def test_send_activity(self, mocker, agent_client, aioresponses_mock): - mocker.patch.object(AgentClient, 'get_access_token', return_value="mocked_token") - mocker.patch.object(ConfidentialClientApplication, "__new__", return_value=mocker.Mock(spec=ConfidentialClientApplication)) + mocker.patch.object( + AgentClient, "get_access_token", return_value="mocked_token" + ) + mocker.patch.object( + ConfidentialClientApplication, + "__new__", + return_value=mocker.Mock(spec=ConfidentialClientApplication), + ) assert agent_client.messaging_endpoint - aioresponses_mock.post(agent_client.messaging_endpoint, payload={"response": "Response from service"}) - + aioresponses_mock.post( + agent_client.messaging_endpoint, + payload={"response": "Response from service"}, + ) + response = await agent_client.send_activity("Hello, World!") data = json.loads(response) assert data == {"response": "Response from service"} @pytest.mark.asyncio async def test_send_expect_replies(self, mocker, agent_client, aioresponses_mock): - mocker.patch.object(AgentClient, 'get_access_token', return_value="mocked_token") - mocker.patch.object(ConfidentialClientApplication, "__new__", return_value=mocker.Mock(spec=ConfidentialClientApplication)) + mocker.patch.object( + AgentClient, "get_access_token", return_value="mocked_token" + ) + mocker.patch.object( + ConfidentialClientApplication, + "__new__", + return_value=mocker.Mock(spec=ConfidentialClientApplication), + ) assert agent_client.messaging_endpoint activities = [ Activity(type="message", text="Response from service"), Activity(type="message", text="Another response"), ] - aioresponses_mock.post(agent_client.messaging_endpoint, payload={ - "activities": [activity.model_dump(by_alias=True, exclude_none=True) for activity in activities], - }) - + aioresponses_mock.post( + agent_client.messaging_endpoint, + payload={ + "activities": [ + activity.model_dump(by_alias=True, exclude_none=True) + for activity in activities + ], + }, + ) + replies = await agent_client.send_expect_replies("Hello, World!") assert len(replies) == 2 assert replies[0].text == "Response from service" assert replies[1].text == "Another response" - assert replies[0].type == replies[1].type == "message" \ No newline at end of file + assert replies[0].type == replies[1].type == "message" diff --git a/dev/integration/src/tests/core/client/test_response_client.py b/dev/integration/src/tests/core/client/test_response_client.py index 73f4a105..986c4172 100644 --- a/dev/integration/src/tests/core/client/test_response_client.py +++ b/dev/integration/src/tests/core/client/test_response_client.py @@ -7,11 +7,14 @@ from src.core import ResponseClient from ._common import DEFAULTS + class TestResponseClient: - + @pytest.fixture async def response_client(self): - async with ResponseClient(host=DEFAULTS.host, port=DEFAULTS.response_port) as client: + async with ResponseClient( + host=DEFAULTS.host, port=DEFAULTS.response_port + ) as client: yield client @pytest.mark.asyncio @@ -26,7 +29,7 @@ async def test_endpoint(self, response_client): async with ClientSession() as session: async with session.post( f"{response_client.service_endpoint}/v3/conversations/test-conv", - json=activity.model_dump(by_alias=True, exclude_none=True) + json=activity.model_dump(by_alias=True, exclude_none=True), ) as resp: assert resp.status == 200 text = await resp.text() @@ -39,4 +42,4 @@ async def test_endpoint(self, response_client): assert activities[0].type == "message" assert activities[0].text == "Hello, World!" - assert (await response_client.pop()) == [] \ No newline at end of file + assert (await response_client.pop()) == [] diff --git a/dev/integration/src/tests/core/test_application_runner.py b/dev/integration/src/tests/core/test_application_runner.py index fa97c77f..719203b7 100644 --- a/dev/integration/src/tests/core/test_application_runner.py +++ b/dev/integration/src/tests/core/test_application_runner.py @@ -3,6 +3,7 @@ from ._common import SimpleRunner, OtherSimpleRunner + class TestApplicationRunner: @pytest.mark.asyncio @@ -13,7 +14,7 @@ async def test_simple_runner(self): async with runner: sleep(0.1) assert app["running"] is True - + assert app["running"] is True @pytest.mark.asyncio @@ -24,7 +25,7 @@ async def test_other_simple_runner(self): async with runner: sleep(0.1) assert app["running"] is True - + assert app["running"] is False @pytest.mark.asyncio @@ -36,4 +37,4 @@ async def test_double_start(self): sleep(0.1) with pytest.raises(RuntimeError): async with runner: - pass \ No newline at end of file + pass diff --git a/dev/integration/src/tests/core/test_integration_from_sample.py b/dev/integration/src/tests/core/test_integration_from_sample.py index 15b30bc5..53d4629d 100644 --- a/dev/integration/src/tests/core/test_integration_from_sample.py +++ b/dev/integration/src/tests/core/test_integration_from_sample.py @@ -7,14 +7,15 @@ Environment, integration, IntegrationFixtures, - Sample + Sample, ) from ._common import SimpleRunner + class SimpleEnvironment(Environment): """A simple implementation of the Environment for testing.""" - + async def init_env(self, environ_config: dict) -> None: self.config = environ_config # Initialize other components as needed @@ -22,6 +23,7 @@ async def init_env(self, environ_config: dict) -> None: def create_runner(self) -> ApplicationRunner: return SimpleRunner(copy(self.config)) + class SimpleSample(Sample): """A simple implementation of the Sample for testing.""" @@ -29,29 +31,28 @@ def __init__(self, environment: Environment, **kwargs): super().__init__(environment, **kwargs) self.data = kwargs.get("data", "default_data") self.other_data = None - + @classmethod async def get_config(cls) -> dict: return {"sample_key": "sample_value"} - + async def init_app(self): await asyncio.sleep(0.1) # Simulate some initialization delay self.other_data = len(self.env.config) - + + @integration(sample=SimpleSample, environment=SimpleEnvironment) class TestIntegrationFromSample(IntegrationFixtures): - + @pytest.mark.asyncio async def test_sample_integration(self, sample, environment): """Test the integration of SimpleSample with SimpleEnvironment.""" - assert environment.config == { - "sample_key": "sample_value" - } + assert environment.config == {"sample_key": "sample_value"} assert sample.env is environment assert sample.data == "default_data" assert sample.other_data == 1 runner = environment.create_runner() - assert runner.app == {"sample_key": "sample_value"} \ No newline at end of file + assert runner.app == {"sample_key": "sample_value"} diff --git a/dev/integration/src/tests/core/test_integration_from_service_url.py b/dev/integration/src/tests/core/test_integration_from_service_url.py index 9eac4e62..99d19f6c 100644 --- a/dev/integration/src/tests/core/test_integration_from_service_url.py +++ b/dev/integration/src/tests/core/test_integration_from_service_url.py @@ -4,12 +4,13 @@ from copy import copy from aioresponses import aioresponses, CallbackResult -from src.core import ( - integration, - IntegrationFixtures -) +from src.core import integration, IntegrationFixtures + -@integration(messaging_endpoint="http://localhost:8000/api/messages/", service_url="http://localhost:8001/") +@integration( + messaging_endpoint="http://localhost:8000/api/messages/", + service_url="http://localhost:8001/", +) class TestIntegrationFromServiceURL(IntegrationFixtures): @pytest.mark.asyncio @@ -24,13 +25,18 @@ async def test_service_url_integration(self, agent_client): assert res == "Service response" @pytest.mark.asyncio - async def test_service_url_integration_with_response_side_effect(self, agent_client, response_client): + async def test_service_url_integration_with_response_side_effect( + self, agent_client, response_client + ): """Test the integration using a service URL.""" with aioresponses() as mocked: def callback(url, **kwargs): - a = requests.post(f"{self.service_url}/v3/conversations/test-conv", json=kwargs.get("json")) + a = requests.post( + f"{self.service_url}/v3/conversations/test-conv", + json=kwargs.get("json"), + ) return CallbackResult(status=200, body="Service response") mocked.post(self.messaging_endpoint, callback=callback) @@ -43,4 +49,4 @@ def callback(url, **kwargs): activities = await response_client.pop() assert len(activities) == 1 assert activities[0].type == "message" - assert activities[0].text == "Hello, service!" \ No newline at end of file + assert activities[0].text == "Hello, service!" From a6681bf6774763c22811bef374de7b8d9a1514a9 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 12:59:24 -0800 Subject: [PATCH 29/81] Quickstart integration test beginning --- dev/integration/src/core/__init__.py | 2 ++ .../src/core/aiohttp/aiohttp_environment.py | 1 + .../src/{tests => framework_tests}/__init__.py | 0 .../src/{tests => framework_tests}/core/__init__.py | 0 .../src/{tests => framework_tests}/core/_common.py | 0 .../core/client/__init__.py | 0 .../{tests => framework_tests}/core/client/_common.py | 0 .../core/client/test_agent_client.py | 0 .../core/client/test_response_client.py | 0 .../core/test_application_runner.py | 0 .../core/test_integration_from_sample.py | 0 .../core/test_integration_from_service_url.py | 0 dev/integration/src/integration_tests/__init__.py | 0 .../src/integration_tests/test_quickstart.py | 11 +++++++++++ 14 files changed, 14 insertions(+) rename dev/integration/src/{tests => framework_tests}/__init__.py (100%) rename dev/integration/src/{tests => framework_tests}/core/__init__.py (100%) rename dev/integration/src/{tests => framework_tests}/core/_common.py (100%) rename dev/integration/src/{tests => framework_tests}/core/client/__init__.py (100%) rename dev/integration/src/{tests => framework_tests}/core/client/_common.py (100%) rename dev/integration/src/{tests => framework_tests}/core/client/test_agent_client.py (100%) rename dev/integration/src/{tests => framework_tests}/core/client/test_response_client.py (100%) rename dev/integration/src/{tests => framework_tests}/core/test_application_runner.py (100%) rename dev/integration/src/{tests => framework_tests}/core/test_integration_from_sample.py (100%) rename dev/integration/src/{tests => framework_tests}/core/test_integration_from_service_url.py (100%) create mode 100644 dev/integration/src/integration_tests/__init__.py create mode 100644 dev/integration/src/integration_tests/test_quickstart.py diff --git a/dev/integration/src/core/__init__.py b/dev/integration/src/core/__init__.py index 1736ceeb..08dee638 100644 --- a/dev/integration/src/core/__init__.py +++ b/dev/integration/src/core/__init__.py @@ -1,4 +1,5 @@ from .application_runner import ApplicationRunner +from aiohttp import AiohttpEnvironment from .client import ( AgentClient, ResponseClient, @@ -11,6 +12,7 @@ __all__ = [ "AgentClient", "ApplicationRunner", + "AiohttpEnvironment", "ResponseClient", "Environment", "integration", diff --git a/dev/integration/src/core/aiohttp/aiohttp_environment.py b/dev/integration/src/core/aiohttp/aiohttp_environment.py index fcbb90ed..99daab59 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_environment.py +++ b/dev/integration/src/core/aiohttp/aiohttp_environment.py @@ -17,6 +17,7 @@ from ..application_runner import ApplicationRunner from ..environment import Environment +from .aiohttp_runner import AiohttpRunner class AiohttpEnvironment(Environment): diff --git a/dev/integration/src/tests/__init__.py b/dev/integration/src/framework_tests/__init__.py similarity index 100% rename from dev/integration/src/tests/__init__.py rename to dev/integration/src/framework_tests/__init__.py diff --git a/dev/integration/src/tests/core/__init__.py b/dev/integration/src/framework_tests/core/__init__.py similarity index 100% rename from dev/integration/src/tests/core/__init__.py rename to dev/integration/src/framework_tests/core/__init__.py diff --git a/dev/integration/src/tests/core/_common.py b/dev/integration/src/framework_tests/core/_common.py similarity index 100% rename from dev/integration/src/tests/core/_common.py rename to dev/integration/src/framework_tests/core/_common.py diff --git a/dev/integration/src/tests/core/client/__init__.py b/dev/integration/src/framework_tests/core/client/__init__.py similarity index 100% rename from dev/integration/src/tests/core/client/__init__.py rename to dev/integration/src/framework_tests/core/client/__init__.py diff --git a/dev/integration/src/tests/core/client/_common.py b/dev/integration/src/framework_tests/core/client/_common.py similarity index 100% rename from dev/integration/src/tests/core/client/_common.py rename to dev/integration/src/framework_tests/core/client/_common.py diff --git a/dev/integration/src/tests/core/client/test_agent_client.py b/dev/integration/src/framework_tests/core/client/test_agent_client.py similarity index 100% rename from dev/integration/src/tests/core/client/test_agent_client.py rename to dev/integration/src/framework_tests/core/client/test_agent_client.py diff --git a/dev/integration/src/tests/core/client/test_response_client.py b/dev/integration/src/framework_tests/core/client/test_response_client.py similarity index 100% rename from dev/integration/src/tests/core/client/test_response_client.py rename to dev/integration/src/framework_tests/core/client/test_response_client.py diff --git a/dev/integration/src/tests/core/test_application_runner.py b/dev/integration/src/framework_tests/core/test_application_runner.py similarity index 100% rename from dev/integration/src/tests/core/test_application_runner.py rename to dev/integration/src/framework_tests/core/test_application_runner.py diff --git a/dev/integration/src/tests/core/test_integration_from_sample.py b/dev/integration/src/framework_tests/core/test_integration_from_sample.py similarity index 100% rename from dev/integration/src/tests/core/test_integration_from_sample.py rename to dev/integration/src/framework_tests/core/test_integration_from_sample.py diff --git a/dev/integration/src/tests/core/test_integration_from_service_url.py b/dev/integration/src/framework_tests/core/test_integration_from_service_url.py similarity index 100% rename from dev/integration/src/tests/core/test_integration_from_service_url.py rename to dev/integration/src/framework_tests/core/test_integration_from_service_url.py diff --git a/dev/integration/src/integration_tests/__init__.py b/dev/integration/src/integration_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/integration_tests/test_quickstart.py b/dev/integration/src/integration_tests/test_quickstart.py new file mode 100644 index 00000000..ca979066 --- /dev/null +++ b/dev/integration/src/integration_tests/test_quickstart.py @@ -0,0 +1,11 @@ +import pytest +from src.core import integration, IntegrationFixtures, AiohttpEnvironment +from src.samples import QuickstartSample + +@integration(sample=QuickstartSample, environment=AiohttpEnvironment) +class TestQuickstartSample(IntegrationFixtures): + + @pytest.mark.asyncio + async def test_welcome_message(self, sample, environment, agent_client, response_client): + response = await self.client.send_message("", members_added=["user1"]) + assert "Welcome to the empty agent!" in response.activities[0].text \ No newline at end of file From 1a7264bf1d8ae65401b82c61af3611d3dab56034 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 13:23:36 -0800 Subject: [PATCH 30/81] Draft of quickstart sample setup --- dev/integration/src/core/__init__.py | 2 +- .../src/core/aiohttp/aiohttp_environment.py | 4 +-- .../src/core/client/agent_client.py | 12 ++++--- dev/integration/src/core/integration.py | 36 +++++++++++-------- .../src/samples/quickstart_sample.py | 15 ++++++++ .../{framework_tests => tests}/__init__.py | 0 dev/integration/src/tests/env.TEMPLATE | 3 ++ .../core => tests/test_framework}/__init__.py | 0 .../test_framework/core}/__init__.py | 0 .../test_framework}/core/_common.py | 0 .../test_framework/core/client}/__init__.py | 0 .../test_framework}/core/client/_common.py | 0 .../core/client/test_agent_client.py | 0 .../core/client/test_response_client.py | 0 .../core/test_application_runner.py | 0 .../core/test_integration_from_sample.py | 0 .../core/test_integration_from_service_url.py | 0 .../test_quickstart.py | 7 ++-- 18 files changed, 53 insertions(+), 26 deletions(-) rename dev/integration/src/{framework_tests => tests}/__init__.py (100%) create mode 100644 dev/integration/src/tests/env.TEMPLATE rename dev/integration/src/{framework_tests/core => tests/test_framework}/__init__.py (100%) rename dev/integration/src/{framework_tests/core/client => tests/test_framework/core}/__init__.py (100%) rename dev/integration/src/{framework_tests => tests/test_framework}/core/_common.py (100%) rename dev/integration/src/{integration_tests => tests/test_framework/core/client}/__init__.py (100%) rename dev/integration/src/{framework_tests => tests/test_framework}/core/client/_common.py (100%) rename dev/integration/src/{framework_tests => tests/test_framework}/core/client/test_agent_client.py (100%) rename dev/integration/src/{framework_tests => tests/test_framework}/core/client/test_response_client.py (100%) rename dev/integration/src/{framework_tests => tests/test_framework}/core/test_application_runner.py (100%) rename dev/integration/src/{framework_tests => tests/test_framework}/core/test_integration_from_sample.py (100%) rename dev/integration/src/{framework_tests => tests/test_framework}/core/test_integration_from_service_url.py (100%) rename dev/integration/src/{integration_tests => tests}/test_quickstart.py (52%) diff --git a/dev/integration/src/core/__init__.py b/dev/integration/src/core/__init__.py index 08dee638..a8f5206c 100644 --- a/dev/integration/src/core/__init__.py +++ b/dev/integration/src/core/__init__.py @@ -1,5 +1,5 @@ from .application_runner import ApplicationRunner -from aiohttp import AiohttpEnvironment +from .aiohttp import AiohttpEnvironment from .client import ( AgentClient, ResponseClient, diff --git a/dev/integration/src/core/aiohttp/aiohttp_environment.py b/dev/integration/src/core/aiohttp/aiohttp_environment.py index 99daab59..5f6ed250 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_environment.py +++ b/dev/integration/src/core/aiohttp/aiohttp_environment.py @@ -42,7 +42,7 @@ async def init_env(self, environ_config: dict) -> None: **self.config ) - def create_runner(self) -> ApplicationRunner: + def create_runner(self, host: str, port: int) -> ApplicationRunner: async def entry_point(req: Request) -> Response: agent: AgentApplication = req.app["agent_app"] @@ -55,4 +55,4 @@ async def entry_point(req: Request) -> Response: APP["agent_app"] = self.agent_application APP["adapter"] = self.adapter - return AiohttpRunner(APP) + return AiohttpRunner(APP, host, port) diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py index 9aac76be..4375681e 100644 --- a/dev/integration/src/core/client/agent_client.py +++ b/dev/integration/src/core/client/agent_client.py @@ -70,7 +70,7 @@ async def _init_client(self) -> None: base_url=self._messaging_endpoint, headers=self._headers ) - async def send_request(self, activity: Activity) -> str: + async def send_request(self, activity: Activity, sleep: float = 0) -> str: await self._init_client() assert self._client @@ -89,6 +89,7 @@ async def send_request(self, activity: Activity) -> str: if not response.ok: raise Exception(f"Failed to send activity: {response.status}") content = await response.text() + await asyncio.sleep(sleep) return content def _to_activity(self, activity_or_text: Activity | str) -> Activity: @@ -102,24 +103,25 @@ def _to_activity(self, activity_or_text: Activity | str) -> Activity: return cast(Activity, activity_or_text) async def send_activity( - self, activity_or_text: Activity | str, timeout: Optional[float] = None + self, activity_or_text: Activity | str, sleep: float = 0, timeout: Optional[float] = None ) -> str: timeout = timeout or self._default_timeout activity = self._to_activity(activity_or_text) - content = await self.send_request(activity) + content = await self.send_request(activity, sleep=sleep) return content async def send_expect_replies( - self, activity_or_text: Activity | str, timeout: Optional[float] = None + self, activity_or_text: Activity | str, sleep: float = 0, timeout: Optional[float] = None ) -> list[Activity]: timeout = timeout or self._default_timeout activity = self._to_activity(activity_or_text) activity.delivery_mode = DeliveryModes.expect_replies - content = await self.send_request(activity) + content = await self.send_request(activity, sleep=sleep) activities_data = json.loads(content).get("activities", []) activities = [Activity.model_validate(act) for act in activities_data] + return activities async def close(self) -> None: diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index 5a6ee5af..6a380611 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -1,4 +1,5 @@ import pytest +import asyncio from typing import ( Optional, TypeVar, @@ -51,20 +52,27 @@ def messaging_endpoint(self) -> str: @pytest.fixture async def environment(self): """Provides the test environment instance.""" - assert self._environment_cls - assert self._sample_cls - environment = self._environment_cls() - await environment.init_env(await self._sample_cls.get_config()) - yield environment + if self._environment_cls: + assert self._sample_cls + environment = self._environment_cls() + await environment.init_env(await self._sample_cls.get_config()) + yield environment + else: + yield None @pytest.fixture async def sample(self, environment): """Provides the sample instance.""" - assert environment - assert self._sample_cls - sample = self._sample_cls(environment) - await sample.init_app() - yield sample + if environment: + assert self._sample_cls + sample = self._sample_cls(environment) + await sample.init_app() + host, port = get_host_and_port(self.messaging_endpoint) + async with environment.create_runner(host, port): + await asyncio.sleep(1) # Give time for the app to start + yield sample + else: + yield None def create_agent_client(self) -> AgentClient: if not self._config: @@ -79,7 +87,7 @@ def create_agent_client(self) -> AgentClient: return agent_client @pytest.fixture - async def agent_client(self) -> AsyncGenerator[AgentClient, None]: + async def agent_client(self, sample, environment) -> AsyncGenerator[AgentClient, None]: agent_client = self.create_agent_client() yield agent_client await agent_client.close() @@ -97,7 +105,7 @@ async def response_client(self) -> AsyncGenerator[ResponseClient, None]: def integration( - messaging_endpoint: Optional[str] = None, + messaging_endpoint: Optional[str] = "http://localhost:3978/api/messages/", sample: Optional[type[Sample]] = None, environment: Optional[type[Environment]] = None, app: Optional[AppT] = None, @@ -126,11 +134,9 @@ def decorator(target_cls: T) -> T: if messaging_endpoint: target_cls._messaging_endpoint = messaging_endpoint - elif sample and environment: + if sample and environment: target_cls._sample_cls = sample target_cls._environment_cls = environment - else: - raise ValueError("Insufficient parameters to create Integration instance.") target_cls._config = kwargs diff --git a/dev/integration/src/samples/quickstart_sample.py b/dev/integration/src/samples/quickstart_sample.py index e3c78cc3..64377860 100644 --- a/dev/integration/src/samples/quickstart_sample.py +++ b/dev/integration/src/samples/quickstart_sample.py @@ -1,7 +1,10 @@ import re +import os import sys import traceback +from dotenv import load_dotenv + from microsoft_agents.activity import ConversationUpdateTypes from microsoft_agents.hosting.core import AgentApplication, TurnContext, TurnState @@ -12,6 +15,18 @@ class QuickstartSample(Sample): """A quickstart sample implementation.""" + @classmethod + async def get_config(cls) -> dict: + """Retrieve the configuration for the sample.""" + + load_dotenv() + + return { + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID"), + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET"), + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID"), + } + async def init_app(self): """Initialize the application for the quickstart sample.""" diff --git a/dev/integration/src/framework_tests/__init__.py b/dev/integration/src/tests/__init__.py similarity index 100% rename from dev/integration/src/framework_tests/__init__.py rename to dev/integration/src/tests/__init__.py diff --git a/dev/integration/src/tests/env.TEMPLATE b/dev/integration/src/tests/env.TEMPLATE new file mode 100644 index 00000000..01dccc7c --- /dev/null +++ b/dev/integration/src/tests/env.TEMPLATE @@ -0,0 +1,3 @@ +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID=client-id +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET=client-secret +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID=tenant-id \ No newline at end of file diff --git a/dev/integration/src/framework_tests/core/__init__.py b/dev/integration/src/tests/test_framework/__init__.py similarity index 100% rename from dev/integration/src/framework_tests/core/__init__.py rename to dev/integration/src/tests/test_framework/__init__.py diff --git a/dev/integration/src/framework_tests/core/client/__init__.py b/dev/integration/src/tests/test_framework/core/__init__.py similarity index 100% rename from dev/integration/src/framework_tests/core/client/__init__.py rename to dev/integration/src/tests/test_framework/core/__init__.py diff --git a/dev/integration/src/framework_tests/core/_common.py b/dev/integration/src/tests/test_framework/core/_common.py similarity index 100% rename from dev/integration/src/framework_tests/core/_common.py rename to dev/integration/src/tests/test_framework/core/_common.py diff --git a/dev/integration/src/integration_tests/__init__.py b/dev/integration/src/tests/test_framework/core/client/__init__.py similarity index 100% rename from dev/integration/src/integration_tests/__init__.py rename to dev/integration/src/tests/test_framework/core/client/__init__.py diff --git a/dev/integration/src/framework_tests/core/client/_common.py b/dev/integration/src/tests/test_framework/core/client/_common.py similarity index 100% rename from dev/integration/src/framework_tests/core/client/_common.py rename to dev/integration/src/tests/test_framework/core/client/_common.py diff --git a/dev/integration/src/framework_tests/core/client/test_agent_client.py b/dev/integration/src/tests/test_framework/core/client/test_agent_client.py similarity index 100% rename from dev/integration/src/framework_tests/core/client/test_agent_client.py rename to dev/integration/src/tests/test_framework/core/client/test_agent_client.py diff --git a/dev/integration/src/framework_tests/core/client/test_response_client.py b/dev/integration/src/tests/test_framework/core/client/test_response_client.py similarity index 100% rename from dev/integration/src/framework_tests/core/client/test_response_client.py rename to dev/integration/src/tests/test_framework/core/client/test_response_client.py diff --git a/dev/integration/src/framework_tests/core/test_application_runner.py b/dev/integration/src/tests/test_framework/core/test_application_runner.py similarity index 100% rename from dev/integration/src/framework_tests/core/test_application_runner.py rename to dev/integration/src/tests/test_framework/core/test_application_runner.py diff --git a/dev/integration/src/framework_tests/core/test_integration_from_sample.py b/dev/integration/src/tests/test_framework/core/test_integration_from_sample.py similarity index 100% rename from dev/integration/src/framework_tests/core/test_integration_from_sample.py rename to dev/integration/src/tests/test_framework/core/test_integration_from_sample.py diff --git a/dev/integration/src/framework_tests/core/test_integration_from_service_url.py b/dev/integration/src/tests/test_framework/core/test_integration_from_service_url.py similarity index 100% rename from dev/integration/src/framework_tests/core/test_integration_from_service_url.py rename to dev/integration/src/tests/test_framework/core/test_integration_from_service_url.py diff --git a/dev/integration/src/integration_tests/test_quickstart.py b/dev/integration/src/tests/test_quickstart.py similarity index 52% rename from dev/integration/src/integration_tests/test_quickstart.py rename to dev/integration/src/tests/test_quickstart.py index ca979066..53bb3871 100644 --- a/dev/integration/src/integration_tests/test_quickstart.py +++ b/dev/integration/src/tests/test_quickstart.py @@ -1,4 +1,6 @@ import pytest +import asyncio + from src.core import integration, IntegrationFixtures, AiohttpEnvironment from src.samples import QuickstartSample @@ -6,6 +8,5 @@ class TestQuickstartSample(IntegrationFixtures): @pytest.mark.asyncio - async def test_welcome_message(self, sample, environment, agent_client, response_client): - response = await self.client.send_message("", members_added=["user1"]) - assert "Welcome to the empty agent!" in response.activities[0].text \ No newline at end of file + async def test_welcome_message(self, agent_client, response_client): + await agent_client.send_expect_replies("hi") \ No newline at end of file From 9ba4a43992b1c87f26dd20f8a3a7f403a1938f0c Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Mon, 3 Nov 2025 13:44:49 -0800 Subject: [PATCH 31/81] Environment config connection --- dev/integration/env.TEMPLATE | 3 --- .../src/core/aiohttp/aiohttp_environment.py | 2 +- dev/integration/src/core/integration.py | 13 +++++++++++ .../src/samples/quickstart_sample.py | 3 ++- dev/integration/src/tests/manual_test.py | 22 +++++++++++++++++++ dev/integration/src/tests/test_quickstart.py | 2 +- 6 files changed, 39 insertions(+), 6 deletions(-) delete mode 100644 dev/integration/env.TEMPLATE create mode 100644 dev/integration/src/tests/manual_test.py diff --git a/dev/integration/env.TEMPLATE b/dev/integration/env.TEMPLATE deleted file mode 100644 index ca2114c8..00000000 --- a/dev/integration/env.TEMPLATE +++ /dev/null @@ -1,3 +0,0 @@ -aioresponses -microsoft-agents-activity -microsoft-agents-hosting-core \ No newline at end of file diff --git a/dev/integration/src/core/aiohttp/aiohttp_environment.py b/dev/integration/src/core/aiohttp/aiohttp_environment.py index 5f6ed250..c8256618 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_environment.py +++ b/dev/integration/src/core/aiohttp/aiohttp_environment.py @@ -51,7 +51,7 @@ async def entry_point(req: Request) -> Response: APP = Application(middlewares=[jwt_authorization_middleware]) APP.router.add_post("/api/messages", entry_point) - APP["agent_configuration"] = self.connection_manager.get_default_connection() + APP["agent_configuration"] = self.connection_manager.get_default_connection_configuration() APP["agent_app"] = self.agent_application APP["adapter"] = self.adapter diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index 6a380611..dbad9f14 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -1,5 +1,7 @@ import pytest import asyncio + +import os from typing import ( Optional, TypeVar, @@ -10,6 +12,7 @@ ) import aiohttp.web +from dotenv import load_dotenv from .application_runner import ApplicationRunner from .environment import Environment @@ -77,6 +80,16 @@ async def sample(self, environment): def create_agent_client(self) -> AgentClient: if not self._config: self._config = {} + + load_dotenv("./dev/integration/src/tests/.env") + self._config.update( + { + "client_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", ""), + "tenant_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID", ""), + "client_secret": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET", ""), + } + ) + agent_client = AgentClient( messaging_endpoint=self.messaging_endpoint, cid=self._cid or self._config.get("cid", ""), diff --git a/dev/integration/src/samples/quickstart_sample.py b/dev/integration/src/samples/quickstart_sample.py index 64377860..6f4e6b55 100644 --- a/dev/integration/src/samples/quickstart_sample.py +++ b/dev/integration/src/samples/quickstart_sample.py @@ -19,7 +19,8 @@ class QuickstartSample(Sample): async def get_config(cls) -> dict: """Retrieve the configuration for the sample.""" - load_dotenv() + load_dotenv("./dev/integration/src/tests/.env") + return { "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID"), diff --git a/dev/integration/src/tests/manual_test.py b/dev/integration/src/tests/manual_test.py new file mode 100644 index 00000000..fa911430 --- /dev/null +++ b/dev/integration/src/tests/manual_test.py @@ -0,0 +1,22 @@ +import pytest +import asyncio + +from src.core import integration, IntegrationFixtures, AiohttpEnvironment, AiohttpEnvironment +from src.samples import QuickstartSample + +async def main(): + + env = AiohttpEnvironment() + await env.init_env(await QuickstartSample.get_config()) + sample = QuickstartSample(env) + await sample.init_app() + + host, port = "localhost", 3978 + + async with env.create_runner(host, port): + print(f"Server running at http://{host}:{port}/api/messages") + while True: + await asyncio.sleep(1) + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/dev/integration/src/tests/test_quickstart.py b/dev/integration/src/tests/test_quickstart.py index 53bb3871..3d57a506 100644 --- a/dev/integration/src/tests/test_quickstart.py +++ b/dev/integration/src/tests/test_quickstart.py @@ -5,7 +5,7 @@ from src.samples import QuickstartSample @integration(sample=QuickstartSample, environment=AiohttpEnvironment) -class TestQuickstartSample(IntegrationFixtures): +class TestQuickstart(IntegrationFixtures): @pytest.mark.asyncio async def test_welcome_message(self, agent_client, response_client): From fab436839ffeb53040f53fdcaf2b29d65cbc04ac Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Tue, 4 Nov 2025 08:29:28 -0800 Subject: [PATCH 32/81] Renaming messaging endpoint to agent url --- dev/integration/src/core/client/agent_client.py | 12 ++++++------ dev/integration/src/core/integration.py | 16 ++++++++-------- .../tests/test_framework/core/client/_common.py | 2 +- .../core/client/test_agent_client.py | 10 +++++----- .../core/test_integration_from_sample.py | 2 +- .../core/test_integration_from_service_url.py | 8 ++++---- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py index 4375681e..24b3ac3c 100644 --- a/dev/integration/src/core/client/agent_client.py +++ b/dev/integration/src/core/client/agent_client.py @@ -14,7 +14,7 @@ class AgentClient: def __init__( self, - messaging_endpoint: str, + agent_url: str, cid: str, client_id: str, tenant_id: str, @@ -22,7 +22,7 @@ def __init__( service_url: Optional[str] = None, default_timeout: float = 5.0, ): - self._messaging_endpoint = messaging_endpoint + self._agent_url = agent_url self._cid = cid self._client_id = client_id self._tenant_id = tenant_id @@ -34,8 +34,8 @@ def __init__( self._client: Optional[ClientSession] = None @property - def messaging_endpoint(self) -> str: - return self._messaging_endpoint + def agent_url(self) -> str: + return self._agent_url @property def service_url(self) -> Optional[str]: @@ -67,7 +67,7 @@ async def _init_client(self) -> None: self._headers = {"Content-Type": "application/json"} self._client = ClientSession( - base_url=self._messaging_endpoint, headers=self._headers + base_url=self._agent_url, headers=self._headers ) async def send_request(self, activity: Activity, sleep: float = 0) -> str: @@ -82,7 +82,7 @@ async def send_request(self, activity: Activity, sleep: float = 0) -> str: activity.service_url = self.service_url async with self._client.post( - self._messaging_endpoint, + "api/messages", headers=self._headers, json=activity.model_dump(by_alias=True, exclude_none=True), ) as response: diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index dbad9f14..c17a8117 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -33,7 +33,7 @@ class IntegrationFixtures: _config: dict[str, Any] = {} _service_url: Optional[str] = None - _messaging_endpoint: Optional[str] = None + _agent_url: Optional[str] = None _cid: Optional[str] = None _client_id: Optional[str] = None _tenant_id: Optional[str] = None @@ -49,8 +49,8 @@ def service_url(self) -> str: return self._service_url or self._config.get("service_url", "") @property - def messaging_endpoint(self) -> str: - return self._messaging_endpoint or self._config.get("messaging_endpoint", "") + def agent_url(self) -> str: + return self._agent_url or self._config.get("agent_url", "") @pytest.fixture async def environment(self): @@ -70,7 +70,7 @@ async def sample(self, environment): assert self._sample_cls sample = self._sample_cls(environment) await sample.init_app() - host, port = get_host_and_port(self.messaging_endpoint) + host, port = get_host_and_port(self.agent_url) async with environment.create_runner(host, port): await asyncio.sleep(1) # Give time for the app to start yield sample @@ -91,7 +91,7 @@ def create_agent_client(self) -> AgentClient: ) agent_client = AgentClient( - messaging_endpoint=self.messaging_endpoint, + agent_url=self.agent_url, cid=self._cid or self._config.get("cid", ""), client_id=self._client_id or self._config.get("client_id", ""), tenant_id=self._tenant_id or self._config.get("tenant_id", ""), @@ -118,7 +118,7 @@ async def response_client(self) -> AsyncGenerator[ResponseClient, None]: def integration( - messaging_endpoint: Optional[str] = "http://localhost:3978/api/messages/", + agent_url: Optional[str] = "http://localhost:3978/", sample: Optional[type[Sample]] = None, environment: Optional[type[Environment]] = None, app: Optional[AppT] = None, @@ -145,8 +145,8 @@ def integration( def decorator(target_cls: T) -> T: - if messaging_endpoint: - target_cls._messaging_endpoint = messaging_endpoint + if agent_url: + target_cls._agent_url = agent_url if sample and environment: target_cls._sample_cls = sample target_cls._environment_cls = environment diff --git a/dev/integration/src/tests/test_framework/core/client/_common.py b/dev/integration/src/tests/test_framework/core/client/_common.py index d6bdf917..00b4291f 100644 --- a/dev/integration/src/tests/test_framework/core/client/_common.py +++ b/dev/integration/src/tests/test_framework/core/client/_common.py @@ -2,7 +2,7 @@ class DEFAULTS: host = "localhost" response_port = 9873 - messaging_endpoint = f"http://{host}:8000" + agent_url = f"http://{host}:8000/" service_url = f"http://{host}:{response_port}" cid = "test-cid" client_id = "test-client-id" diff --git a/dev/integration/src/tests/test_framework/core/client/test_agent_client.py b/dev/integration/src/tests/test_framework/core/client/test_agent_client.py index 89ea467d..5708ccce 100644 --- a/dev/integration/src/tests/test_framework/core/client/test_agent_client.py +++ b/dev/integration/src/tests/test_framework/core/client/test_agent_client.py @@ -18,7 +18,7 @@ class TestAgentClient: @pytest.fixture async def agent_client(self): client = AgentClient( - messaging_endpoint=DEFAULTS.messaging_endpoint, + agent_url=DEFAULTS.agent_url, cid=DEFAULTS.cid, client_id=DEFAULTS.client_id, tenant_id=DEFAULTS.tenant_id, @@ -44,9 +44,9 @@ async def test_send_activity(self, mocker, agent_client, aioresponses_mock): return_value=mocker.Mock(spec=ConfidentialClientApplication), ) - assert agent_client.messaging_endpoint + assert agent_client.agent_url aioresponses_mock.post( - agent_client.messaging_endpoint, + f"{agent_client.agent_url}api/messages", payload={"response": "Response from service"}, ) @@ -65,13 +65,13 @@ async def test_send_expect_replies(self, mocker, agent_client, aioresponses_mock return_value=mocker.Mock(spec=ConfidentialClientApplication), ) - assert agent_client.messaging_endpoint + assert agent_client.agent_url activities = [ Activity(type="message", text="Response from service"), Activity(type="message", text="Another response"), ] aioresponses_mock.post( - agent_client.messaging_endpoint, + agent_client.agent_url + "api/messages", payload={ "activities": [ activity.model_dump(by_alias=True, exclude_none=True) diff --git a/dev/integration/src/tests/test_framework/core/test_integration_from_sample.py b/dev/integration/src/tests/test_framework/core/test_integration_from_sample.py index 53d4629d..89800fda 100644 --- a/dev/integration/src/tests/test_framework/core/test_integration_from_sample.py +++ b/dev/integration/src/tests/test_framework/core/test_integration_from_sample.py @@ -20,7 +20,7 @@ async def init_env(self, environ_config: dict) -> None: self.config = environ_config # Initialize other components as needed - def create_runner(self) -> ApplicationRunner: + def create_runner(self, *args) -> ApplicationRunner: return SimpleRunner(copy(self.config)) diff --git a/dev/integration/src/tests/test_framework/core/test_integration_from_service_url.py b/dev/integration/src/tests/test_framework/core/test_integration_from_service_url.py index 99d19f6c..8254070b 100644 --- a/dev/integration/src/tests/test_framework/core/test_integration_from_service_url.py +++ b/dev/integration/src/tests/test_framework/core/test_integration_from_service_url.py @@ -8,10 +8,10 @@ @integration( - messaging_endpoint="http://localhost:8000/api/messages/", + agent_url="http://localhost:8000/", service_url="http://localhost:8001/", ) -class TestIntegrationFromServiceURL(IntegrationFixtures): +class TestIntegrationFromURL(IntegrationFixtures): @pytest.mark.asyncio async def test_service_url_integration(self, agent_client): @@ -19,7 +19,7 @@ async def test_service_url_integration(self, agent_client): with aioresponses() as mocked: - mocked.post(self.messaging_endpoint, status=200, body="Service response") + mocked.post(f"{self.agent_url}api/messages", status=200, body="Service response") res = await agent_client.send_activity("Hello, service!") assert res == "Service response" @@ -39,7 +39,7 @@ def callback(url, **kwargs): ) return CallbackResult(status=200, body="Service response") - mocked.post(self.messaging_endpoint, callback=callback) + mocked.post(f"{self.agent_url}api/messages", callback=callback) res = await agent_client.send_activity("Hello, service!") assert res == "Service response" From a111155e60e130869a617398e5ca6c3d498029eb Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Tue, 4 Nov 2025 10:39:49 -0800 Subject: [PATCH 33/81] Removing unnecessary files --- dev/integration/integration_tests/__init__.py | 0 .../integration_tests/test_quickstart.py | 44 ------------------- dev/integration/src/tests/manual_test.py | 25 ++++++++++- dev/integration/src/tests/test_quickstart.py | 5 ++- 4 files changed, 28 insertions(+), 46 deletions(-) delete mode 100644 dev/integration/integration_tests/__init__.py delete mode 100644 dev/integration/integration_tests/test_quickstart.py diff --git a/dev/integration/integration_tests/__init__.py b/dev/integration/integration_tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/integration/integration_tests/test_quickstart.py b/dev/integration/integration_tests/test_quickstart.py deleted file mode 100644 index 3f07cbdc..00000000 --- a/dev/integration/integration_tests/test_quickstart.py +++ /dev/null @@ -1,44 +0,0 @@ -import pytest -import asyncio - -from ..core import integration - - -@integration(service_url="http://localhost:3978/api/messages") -class TestQuickstart: - - @pytest.mark.asyncio - async def test_quickstart_functionality(self, agent_client, response_client): - await agent_client.send("hi") - await asyncio.sleep(2) - response = (await response_client.pop())[0] - assert "hello" in response.text.lower() - - -# class TestQuickstart(TestSuiteBase): - -# @pytest.mark.asyncio -# async def test_quickstart_functionality(self): -# pass - -# @integration(QuickstartSample, None) # env -# class TestQuickstart: - -# @pytest.mark.asyncio -# async def test_hello(self, agent_client, env): -# agent_client.send("hi") - -# await asyncio.sleep(1) - -# # assert env.auth... - -# @integration(app=None) # (endpoint="alternative") -# class TestQuickstartAlternative: - -# @pytest.mark.asyncio -# async def test_hello(self, agent_client, response_client): - -# agent_client.send("hi") -# await asyncio.sleep(10) - -# assert receiver.has_activity("hello") diff --git a/dev/integration/src/tests/manual_test.py b/dev/integration/src/tests/manual_test.py index fa911430..6ebc9896 100644 --- a/dev/integration/src/tests/manual_test.py +++ b/dev/integration/src/tests/manual_test.py @@ -1,9 +1,12 @@ +import os import pytest import asyncio -from src.core import integration, IntegrationFixtures, AiohttpEnvironment, AiohttpEnvironment +from src.core import integration, IntegrationFixtures, AiohttpEnvironment, AiohttpEnvironment, AgentClient from src.samples import QuickstartSample +from dotenv import load_dotenv + async def main(): env = AiohttpEnvironment() @@ -13,9 +16,29 @@ async def main(): host, port = "localhost", 3978 + load_dotenv("./src/tests/.env") + config = { + "client_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", ""), + "tenant_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID", ""), + "client_secret": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET", ""), + } + + client = AgentClient( + agent_url="http://localhost:3978/", + cid=config.get("cid", ""), + client_id=config.get("client_id", ""), + tenant_id=config.get("tenant_id", ""), + client_secret=config.get("client_secret", ""), + ) + async with env.create_runner(host, port): print(f"Server running at http://{host}:{port}/api/messages") while True: + + res = await client.send_expect_replies("Hello, Agent!") + + breakpoint() + await asyncio.sleep(1) if __name__ == "__main__": diff --git a/dev/integration/src/tests/test_quickstart.py b/dev/integration/src/tests/test_quickstart.py index 3d57a506..80f83e42 100644 --- a/dev/integration/src/tests/test_quickstart.py +++ b/dev/integration/src/tests/test_quickstart.py @@ -9,4 +9,7 @@ class TestQuickstart(IntegrationFixtures): @pytest.mark.asyncio async def test_welcome_message(self, agent_client, response_client): - await agent_client.send_expect_replies("hi") \ No newline at end of file + await agent_client.send_expect_replies("hi") + # await asyncio.sleep(1) # Wait for processing + # responses = await response_client.pop() + From a2a1092fc23f66d5228358c1d92c50408d13c6da Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Tue, 4 Nov 2025 11:41:31 -0800 Subject: [PATCH 34/81] Fixing agent client payload sending --- .../src/core/aiohttp/aiohttp_environment.py | 2 ++ dev/integration/src/core/client/agent_client.py | 10 +++++++--- dev/integration/src/core/integration.py | 2 +- dev/integration/src/samples/quickstart_sample.py | 2 +- dev/integration/src/tests/manual_test.py | 7 +++++-- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/dev/integration/src/core/aiohttp/aiohttp_environment.py b/dev/integration/src/core/aiohttp/aiohttp_environment.py index c8256618..83edbe9a 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_environment.py +++ b/dev/integration/src/core/aiohttp/aiohttp_environment.py @@ -45,6 +45,8 @@ async def init_env(self, environ_config: dict) -> None: def create_runner(self, host: str, port: int) -> ApplicationRunner: async def entry_point(req: Request) -> Response: + # text = await req.text() + # breakpoint() agent: AgentApplication = req.app["agent_app"] adapter: CloudAdapter = req.app["adapter"] return await start_agent_process(req, agent, adapter) diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py index 24b3ac3c..d301bf82 100644 --- a/dev/integration/src/core/client/agent_client.py +++ b/dev/integration/src/core/client/agent_client.py @@ -5,7 +5,7 @@ from aiohttp import ClientSession -from microsoft_agents.activity import Activity, ActivityTypes, DeliveryModes +from microsoft_agents.activity import Activity, ActivityTypes, DeliveryModes, ConversationAccount from msal import ConfidentialClientApplication @@ -77,6 +77,8 @@ async def send_request(self, activity: Activity, sleep: float = 0) -> str: if activity.conversation: activity.conversation.id = self._cid + else: + activity.conversation = ConversationAccount(id=self._cid or "") if self.service_url: activity.service_url = self.service_url @@ -84,11 +86,12 @@ async def send_request(self, activity: Activity, sleep: float = 0) -> str: async with self._client.post( "api/messages", headers=self._headers, - json=activity.model_dump(by_alias=True, exclude_none=True), + json=activity.model_dump(by_alias=True, exclude_unset=True, exclude_none=True, mode="json"), ) as response: + content = await response.text() + breakpoint() if not response.ok: raise Exception(f"Failed to send activity: {response.status}") - content = await response.text() await asyncio.sleep(sleep) return content @@ -116,6 +119,7 @@ async def send_expect_replies( timeout = timeout or self._default_timeout activity = self._to_activity(activity_or_text) activity.delivery_mode = DeliveryModes.expect_replies + activity.service_url = activity.service_url or "http://localhost" # temporary fix content = await self.send_request(activity, sleep=sleep) diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py index c17a8117..59747d4c 100644 --- a/dev/integration/src/core/integration.py +++ b/dev/integration/src/core/integration.py @@ -81,7 +81,7 @@ def create_agent_client(self) -> AgentClient: if not self._config: self._config = {} - load_dotenv("./dev/integration/src/tests/.env") + load_dotenv("./src/tests/.env") self._config.update( { "client_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", ""), diff --git a/dev/integration/src/samples/quickstart_sample.py b/dev/integration/src/samples/quickstart_sample.py index 6f4e6b55..a18d7553 100644 --- a/dev/integration/src/samples/quickstart_sample.py +++ b/dev/integration/src/samples/quickstart_sample.py @@ -19,7 +19,7 @@ class QuickstartSample(Sample): async def get_config(cls) -> dict: """Retrieve the configuration for the sample.""" - load_dotenv("./dev/integration/src/tests/.env") + load_dotenv("./src/tests/.env") return { diff --git a/dev/integration/src/tests/manual_test.py b/dev/integration/src/tests/manual_test.py index 6ebc9896..ea415580 100644 --- a/dev/integration/src/tests/manual_test.py +++ b/dev/integration/src/tests/manual_test.py @@ -7,6 +7,11 @@ from dotenv import load_dotenv +import logging +ms_agents_logger = logging.getLogger("microsoft_agents") +ms_agents_logger.addHandler(logging.StreamHandler()) +ms_agents_logger.setLevel(logging.DEBUG) + async def main(): env = AiohttpEnvironment() @@ -37,8 +42,6 @@ async def main(): res = await client.send_expect_replies("Hello, Agent!") - breakpoint() - await asyncio.sleep(1) if __name__ == "__main__": From ae8a2864ea7ede7353b2c270944521101c32a948 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Tue, 4 Nov 2025 13:34:01 -0800 Subject: [PATCH 35/81] First successful integration test --- .../src/core/aiohttp/aiohttp_environment.py | 2 -- .../src/core/client/agent_client.py | 8 +++-- dev/integration/src/core/utils.py | 36 ++++++++++++++++++- dev/integration/src/tests/manual_test.py | 9 +---- dev/integration/src/tests/test_quickstart.py | 11 ++++-- 5 files changed, 49 insertions(+), 17 deletions(-) diff --git a/dev/integration/src/core/aiohttp/aiohttp_environment.py b/dev/integration/src/core/aiohttp/aiohttp_environment.py index 83edbe9a..c8256618 100644 --- a/dev/integration/src/core/aiohttp/aiohttp_environment.py +++ b/dev/integration/src/core/aiohttp/aiohttp_environment.py @@ -45,8 +45,6 @@ async def init_env(self, environ_config: dict) -> None: def create_runner(self, host: str, port: int) -> ApplicationRunner: async def entry_point(req: Request) -> Response: - # text = await req.text() - # breakpoint() agent: AgentApplication = req.app["agent_app"] adapter: CloudAdapter = req.app["adapter"] return await start_agent_process(req, agent, adapter) diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py index d301bf82..bbf650b6 100644 --- a/dev/integration/src/core/client/agent_client.py +++ b/dev/integration/src/core/client/agent_client.py @@ -1,13 +1,14 @@ import os import json import asyncio -from typing import Any, Optional, cast +from typing import Optional, cast from aiohttp import ClientSession +from msal import ConfidentialClientApplication from microsoft_agents.activity import Activity, ActivityTypes, DeliveryModes, ConversationAccount -from msal import ConfidentialClientApplication +from ..utils import _populate_incoming_activity class AgentClient: @@ -83,13 +84,14 @@ async def send_request(self, activity: Activity, sleep: float = 0) -> str: if self.service_url: activity.service_url = self.service_url + activity = _populate_incoming_activity(activity) + async with self._client.post( "api/messages", headers=self._headers, json=activity.model_dump(by_alias=True, exclude_unset=True, exclude_none=True, mode="json"), ) as response: content = await response.text() - breakpoint() if not response.ok: raise Exception(f"Failed to send activity: {response.status}") await asyncio.sleep(sleep) diff --git a/dev/integration/src/core/utils.py b/dev/integration/src/core/utils.py index 82d4c528..2a183169 100644 --- a/dev/integration/src/core/utils.py +++ b/dev/integration/src/core/utils.py @@ -1,10 +1,44 @@ from urllib.parse import urlparse +from microsoft_agents.activity import ( + Activity, + ActivityTypes, + DeliveryModes, + ConversationAccount, + ChannelAccount, +) -def get_host_and_port(url): +def get_host_and_port(url: str) -> tuple[str, int]: """Extract host and port from a URL.""" parsed_url = urlparse(url) host = parsed_url.hostname or "localhost" port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) return host, port + +def _populate_incoming_activity(activity: Activity) -> Activity: + + activity = activity.model_copy() + + if not activity.locale: + activity.locale = "en-US" + + if not activity.channel_id: + activity.channel_id = "emulator" + + if not activity.delivery_mode: + activity.delivery_mode = DeliveryModes.normal + + if not activity.service_url: + activity.service_url = "http://localhost" + + if not activity.recipient: + activity.recipient = ChannelAccount(id="agent", name="Agent") + + if not activity.from_property: + activity.from_property = ChannelAccount(id="user", name="User") + + if not activity.conversation: + activity.conversation = ConversationAccount(id="conversation1") + + return activity \ No newline at end of file diff --git a/dev/integration/src/tests/manual_test.py b/dev/integration/src/tests/manual_test.py index ea415580..a24f9c34 100644 --- a/dev/integration/src/tests/manual_test.py +++ b/dev/integration/src/tests/manual_test.py @@ -7,11 +7,6 @@ from dotenv import load_dotenv -import logging -ms_agents_logger = logging.getLogger("microsoft_agents") -ms_agents_logger.addHandler(logging.StreamHandler()) -ms_agents_logger.setLevel(logging.DEBUG) - async def main(): env = AiohttpEnvironment() @@ -39,10 +34,8 @@ async def main(): async with env.create_runner(host, port): print(f"Server running at http://{host}:{port}/api/messages") while True: - - res = await client.send_expect_replies("Hello, Agent!") - await asyncio.sleep(1) + res = await client.send_expect_replies("Hello, Agent!") if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file diff --git a/dev/integration/src/tests/test_quickstart.py b/dev/integration/src/tests/test_quickstart.py index 80f83e42..db786ab5 100644 --- a/dev/integration/src/tests/test_quickstart.py +++ b/dev/integration/src/tests/test_quickstart.py @@ -9,7 +9,12 @@ class TestQuickstart(IntegrationFixtures): @pytest.mark.asyncio async def test_welcome_message(self, agent_client, response_client): - await agent_client.send_expect_replies("hi") - # await asyncio.sleep(1) # Wait for processing - # responses = await response_client.pop() + res = await agent_client.send_expect_replies("hi") + await asyncio.sleep(1) # Wait for processing + responses = await response_client.pop() + assert len(responses) == 0 + + first_non_typing = next((r for r in res if r.type != "typing"), None) + assert first_non_typing is not None + assert first_non_typing.text == "you said: hi" \ No newline at end of file From 9a5a6a8a8dad4a771a3a38c467e28273b497a794 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Tue, 4 Nov 2025 14:36:06 -0800 Subject: [PATCH 36/81] Beginning foundational test cases --- dev/integration/README.md | 2 + .../src/tests/integration/__init__.py | 0 .../integration/foundational/__init__.py | 0 .../tests/integration/foundational/_common.py | 10 +++ .../integration/foundational/test_suite.py | 64 +++++++++++++++++++ .../{ => integration}/test_quickstart.py | 0 6 files changed, 76 insertions(+) create mode 100644 dev/integration/README.md create mode 100644 dev/integration/src/tests/integration/__init__.py create mode 100644 dev/integration/src/tests/integration/foundational/__init__.py create mode 100644 dev/integration/src/tests/integration/foundational/_common.py create mode 100644 dev/integration/src/tests/integration/foundational/test_suite.py rename dev/integration/src/tests/{ => integration}/test_quickstart.py (100%) diff --git a/dev/integration/README.md b/dev/integration/README.md new file mode 100644 index 00000000..614462c9 --- /dev/null +++ b/dev/integration/README.md @@ -0,0 +1,2 @@ +# Microsoft 365 Agents SDK for Python Integration Testing Framework + diff --git a/dev/integration/src/tests/integration/__init__.py b/dev/integration/src/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/integration/foundational/__init__.py b/dev/integration/src/tests/integration/foundational/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/integration/foundational/_common.py b/dev/integration/src/tests/integration/foundational/_common.py new file mode 100644 index 00000000..91672ffd --- /dev/null +++ b/dev/integration/src/tests/integration/foundational/_common.py @@ -0,0 +1,10 @@ +import json + +from microsoft_agents.activity import Activity + +def load_activity(channel: str, name: str) -> Activity: + + with open("./dev/integration/src/tests/integration/foundational/activities/{}/{}.json".format(channel, name), "r") as f: + activity = json.load(f) + + return Activity.model_validate(activity) \ No newline at end of file diff --git a/dev/integration/src/tests/integration/foundational/test_suite.py b/dev/integration/src/tests/integration/foundational/test_suite.py new file mode 100644 index 00000000..42ce5693 --- /dev/null +++ b/dev/integration/src/tests/integration/foundational/test_suite.py @@ -0,0 +1,64 @@ +import json +import pytest +import asyncio + +from microsoft_agents.activity import ( + ActivityTypes, +) + +from src.core import integration, IntegrationFixtures, AiohttpEnvironment +from src.samples import QuickstartSample + +from ._common import load_activity + +DIRECTLINE = "directline" + +@integration() +class TestFoundation(IntegrationFixtures): + + def load_activity(self, activity_name) -> Activity: + return load_activity(DIRECTLINE, activity_name) + + @pytest.mark.asyncio + async def test__send_activity__sends_hello_world__returns_hello_world(self, agent_client): + activity = load_activity(DIRECTLINE, "hello_world.json") + result = await agent_client.send_activity(activity) + assert result is not None + last = result[-1] + assert last.type == ActivityTypes.message + assert last.text.lower() == "you said: {activity.text}".lower() + + @pytest.mark.asyncio + async def test__send_invoke__send_basic_invoke_activity__receive_invoke_response(self, agent_client): + activity = load_activity(DIRECTLINE, "basic_invoke.json") + result = await agent_client.send_activity(activity) + assert result + data = json.loads(result) + message = data.get("message", {}) + assert "Invoke received." in message + assert "data" in data + assert data["parameters"] and len(data["parameters"]) > 0 + assert "hi" in data["value"] + + @pytest.mark.asyncio + async def test__send_activity__sends_message_activity_to_ac_submit__return_valid_response(self, agent_client): + activity = load_activity(DIRECTLINE, "ac_submit.json") + result = await agent_client.send_activity(activity) + assert result is not None + last = result[-1] + assert last.type == ActivityTypes.message + assert "doStuff" in last.text + assert "Action.Submit" in last.text + assert "hello" in last.text + + @pytest.mark.asyncio + async def test__send_invoke_sends_invoke_activity_to_ac_execute__returns_valid_adaptive_card_invoke_response(self, agent_client): + activity = load_activity(DIRECTLINE, "ac_execute.json") + result = await agent_client.send_activity(activity) + assert result + data = json.loads(result) + message = data.get("message", {}) + assert "Adaptive Card Invoke received." in message + assert "data" in data + assert "doStuff" in data["parameters"] + assert "hello" in data["value"] \ No newline at end of file diff --git a/dev/integration/src/tests/test_quickstart.py b/dev/integration/src/tests/integration/test_quickstart.py similarity index 100% rename from dev/integration/src/tests/test_quickstart.py rename to dev/integration/src/tests/integration/test_quickstart.py From aad011aa9902ff9666d9d91f7694c7e8e3395f9d Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Tue, 4 Nov 2025 14:42:21 -0800 Subject: [PATCH 37/81] TypingIndicator test --- .../tests/integration/components/__init__.py | 0 .../components/test_typing_indicator.py | 36 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 dev/integration/src/tests/integration/components/__init__.py create mode 100644 dev/integration/src/tests/integration/components/test_typing_indicator.py diff --git a/dev/integration/src/tests/integration/components/__init__.py b/dev/integration/src/tests/integration/components/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/integration/components/test_typing_indicator.py b/dev/integration/src/tests/integration/components/test_typing_indicator.py new file mode 100644 index 00000000..a51613c8 --- /dev/null +++ b/dev/integration/src/tests/integration/components/test_typing_indicator.py @@ -0,0 +1,36 @@ +import pytest +import asyncio + +from microsoft_agents.activity import ( + Activity, + ActivityTypes, + ChannelAccount +) + +from src.core import integration, IntegrationFixtures, AiohttpEnvironment +from src.samples import QuickstartSample + +@integration(sample=QuickstartSample, environment=AiohttpEnvironment) +class TestTypingIndicator(IntegrationFixtures): + + @pytest.mark.asyncio + async def test_typing_indicator(self, agent_client, response_client): + + activity_base = Activity( + type=ActivityTypes.message, + from_property={"id": "user1", "name": "User 1"}, + recipient={"id": "agent", "name": "Agent"}, + conversation={"id": "conv1"}, + channel_id="test_channel" + ) + + activity_a = activity_base.model_copy() + activity_b = activity_base.model_copy() + + activity_a.from_property = ChannelAccount(id="user1", name="User 1") + activity_b.from_property = ChannelAccount(id="user2", name="User 2") + + await asyncio.gather( + agent_client.send_activity(activity_a), + agent_client.send_activity(activity_b) + ) \ No newline at end of file From be0032ad03aa475b2a31f67dd46ade9ebcf18bba Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Tue, 4 Nov 2025 14:55:12 -0800 Subject: [PATCH 38/81] Adding more test cases --- .../integration/foundational/test_suite.py | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/dev/integration/src/tests/integration/foundational/test_suite.py b/dev/integration/src/tests/integration/foundational/test_suite.py index 42ce5693..0e691097 100644 --- a/dev/integration/src/tests/integration/foundational/test_suite.py +++ b/dev/integration/src/tests/integration/foundational/test_suite.py @@ -54,11 +54,28 @@ async def test__send_activity__sends_message_activity_to_ac_submit__return_valid @pytest.mark.asyncio async def test__send_invoke_sends_invoke_activity_to_ac_execute__returns_valid_adaptive_card_invoke_response(self, agent_client): activity = load_activity(DIRECTLINE, "ac_execute.json") + result = await agent_client.send_invoke(activity) + + result = json.loads(result) + + assert result.status == 200 + assert result.value + + assert "application/vnd.microsoft.card.adaptive" in result.type + + activity_data = json.loads(activity.value) + assert activity_data.get("action") + user_text = activity_data.get("usertext") + assert user_text in result.value + + @pytest.mark.asyncio + async def test__send_activity_sends_text__returns_poem(self, agent_client): + activity = self.load_activity("poem_request.json") result = await agent_client.send_activity(activity) + assert result - data = json.loads(result) - message = data.get("message", {}) - assert "Adaptive Card Invoke received." in message - assert "data" in data - assert "doStuff" in data["parameters"] - assert "hello" in data["value"] \ No newline at end of file + assert result[0] + + index = 0 + if result[0].type == ActivityTypes.typing and not result[0].text: + index += 1 \ No newline at end of file From e3f43240515732f04e7da64198037f368f93accb Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Tue, 4 Nov 2025 15:18:48 -0800 Subject: [PATCH 39/81] More foundational integration test cases --- .../integration/foundational/test_suite.py | 72 +++++++++++++++++-- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/dev/integration/src/tests/integration/foundational/test_suite.py b/dev/integration/src/tests/integration/foundational/test_suite.py index 0e691097..a38edf0b 100644 --- a/dev/integration/src/tests/integration/foundational/test_suite.py +++ b/dev/integration/src/tests/integration/foundational/test_suite.py @@ -70,12 +70,72 @@ async def test__send_invoke_sends_invoke_activity_to_ac_execute__returns_valid_a @pytest.mark.asyncio async def test__send_activity_sends_text__returns_poem(self, agent_client): - activity = self.load_activity("poem_request.json") + pass + + @pytest.mark.asyncio + async def test__send_expected_replies_activity__sends_text__returns_poem(self, agent_client): + activity = self.load_activity("expected_replies.json") + result = await agent_client.send_expected_replies(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert "Apollo" in last.text + assert "\n" in last.text + + @pytest.mark.asyncio + async def test__send_invoke__query_link__returns_text(self, agent_client): + activity = self.load_activity("query_link.json") + result = await agent_client.send_invoke(activity) + pass # TODO + + @pytest.mark.asyncio + async def test__send_invoke__select_item__receive_item(self, agent_client): + activity = self.load_activity("select_item.json") + result = await agent_client.send_invoke(activity) + pass # TODO + + @pytest.mark.asyncio + async def test__send_activity__conversation_update__returns_welcome_message(self, agent_client): + activity = self.load_activity("conversation_update.json") result = await agent_client.send_activity(activity) + last = result[-1] + assert "Hello and Welcome!" in last.text - assert result - assert result[0] + @pytest.mark.asyncio + async def test__send_activity__send_heart_message_reaction__returns_message_reaction_heart(self, agent_client): + activity = self.load_activity("message_reaction_heart.json") + result = await agent_client.send_activity(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert "Message Reaction Added: heart" in last.text + + @pytest.mark.asyncio + async def test__send_activity__remove_heart_message_reaction__returns_message_reaction_heart(self, agent_client): + activity = self.load_activity + result = await agent_client.send_activity(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert "Message Reaction Removed: heart" in last.text - index = 0 - if result[0].type == ActivityTypes.typing and not result[0].text: - index += 1 \ No newline at end of file + @pytest.mark.asyncio + async def test__send_expected_replies_activity__send_seattle_today_weather__returns_weather(self, agent_client): + activity = self.load_activity("expected_replies_seattle_weather.json") + result = await agent_client.send_expected_replies(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert last.attachments and len(last.attachments) > 0 + + adaptive_card = last.attachments.first() + assert adaptive_card + assert "application/vnd.microsoft.card.adaptive" == adaptive_card.content_type + assert adaptive_card.content + + assert \ + "�" in adaptive_card.content or \ + "\\u00B0" in adaptive_card.content or \ + f"Missing temperature inside adaptive card: {adaptive_card.content}" in adaptive_card.content + + @pytest.mark.asyncio + async def test__send_activity__simulate_message_loop__expect_question_about_time_and_returns_weather(self, agent_client): + activities = self.load_activity("message_loop_1.json") + fresult = await agent_client.send_activity(activities[0]) + assert \ No newline at end of file From a077eed4eaf1d271ccc9ab0e7ee83ef9cecc8d9b Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Wed, 5 Nov 2025 10:49:24 -0800 Subject: [PATCH 40/81] Reorganizing testing tools into package --- dev/microsoft-agents-testing/README.md | 1 + .../microsoft_agents/testing}/__init__.py | 0 .../testing/_executors/__init__.py | 11 ++++ .../testing/_executors/coroutine_executor.py | 28 ++++++++++ .../testing/_executors/execution_result.py | 28 ++++++++++ .../testing/_executors/executor.py | 49 ++++++++++++++++++ .../testing/_executors/thread_executor.py | 37 ++++++++++++++ .../testing/benchmark}/__init__.py | 0 .../testing/benchmark/aggregated_results.py | 51 +++++++++++++++++++ .../testing/benchmark/benchmark.py | 49 ++++++++++++++++++ .../testing/benchmark/config.py | 23 +++++++++ .../microsoft_agents/testing/cli.py} | 0 .../testing/common/__init__.py | 7 +++ .../testing/common/_generate_token.py | 34 +++++++++++++ .../testing/common/_payload_sender.py | 32 ++++++++++++ .../testing/integration}/__init__.py | 0 .../testing/integration/core}/__init__.py | 0 .../integration/core/aiohttp}/__init__.py | 0 .../core/aiohttp/aiohttp_environment.py} | 0 .../core/aiohttp/aiohttp_runner.py | 0 .../integration/core/application_runner.py | 0 .../integration/core/client/__init__.py | 0 .../integration/core/client/agent_client.py | 0 .../integration/core/client/auto_client.py | 0 .../core/client/response_client.py | 0 .../testing/integration/core/environment.py | 0 .../testing/integration/core/integration.py | 0 .../testing/integration/core/sample.py | 0 .../testing/integration/core/utils.py | 0 .../testing/integration/samples/__init__.py | 0 .../integration/samples/quickstart_sample.py | 0 dev/microsoft-agents-testing/pyproject.toml | 25 +++++++++ dev/microsoft-agents-testing/setup.py | 18 +++++++ .../tests/__init__.py | 0 .../tests/env.TEMPLATE | 0 .../tests/integration/__init__.py | 0 .../tests/integration/components/__init__.py | 0 .../components/test_typing_indicator.py | 0 .../integration/foundational/__init__.py | 0 .../tests/integration/foundational/_common.py | 0 .../integration/foundational/test_suite.py | 0 .../tests/integration/test_quickstart.py | 0 .../tests/manual_test.py | 0 .../tests/test_framework/__init__.py | 0 .../tests/test_framework/core/__init__.py | 0 .../tests/test_framework/core/_common.py | 0 .../test_framework/core/client/__init__.py | 0 .../test_framework/core/client/_common.py | 0 .../core/client/test_agent_client.py | 0 .../core/client/test_response_client.py | 0 .../core/test_application_runner.py | 0 .../core/test_integration_from_sample.py | 0 .../core/test_integration_from_service_url.py | 0 53 files changed, 393 insertions(+) create mode 100644 dev/microsoft-agents-testing/README.md rename dev/{integration/src/tests => microsoft-agents-testing/microsoft_agents/testing}/__init__.py (100%) create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/_executors/__init__.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/_executors/coroutine_executor.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/_executors/execution_result.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/_executors/executor.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/_executors/thread_executor.py rename dev/{integration/src/tests/integration => microsoft-agents-testing/microsoft_agents/testing/benchmark}/__init__.py (100%) create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/aggregated_results.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/benchmark.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/config.py rename dev/{integration/src/tests/integration/components/__init__.py => microsoft-agents-testing/microsoft_agents/testing/cli.py} (100%) create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/common/__init__.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/common/_generate_token.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/common/_payload_sender.py rename dev/{integration/src/tests/integration/foundational => microsoft-agents-testing/microsoft_agents/testing/integration}/__init__.py (100%) rename dev/{integration/src/tests/test_framework => microsoft-agents-testing/microsoft_agents/testing/integration/core}/__init__.py (100%) rename dev/{integration/src/tests/test_framework/core => microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp}/__init__.py (100%) rename dev/{integration/src/tests/test_framework/core/client/__init__.py => microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py} (100%) create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/auto_client.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/utils.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/__init__.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/quickstart_sample.py create mode 100644 dev/microsoft-agents-testing/pyproject.toml create mode 100644 dev/microsoft-agents-testing/setup.py create mode 100644 dev/microsoft-agents-testing/tests/__init__.py rename dev/{integration/src => microsoft-agents-testing}/tests/env.TEMPLATE (100%) create mode 100644 dev/microsoft-agents-testing/tests/integration/__init__.py create mode 100644 dev/microsoft-agents-testing/tests/integration/components/__init__.py rename dev/{integration/src => microsoft-agents-testing}/tests/integration/components/test_typing_indicator.py (100%) create mode 100644 dev/microsoft-agents-testing/tests/integration/foundational/__init__.py rename dev/{integration/src => microsoft-agents-testing}/tests/integration/foundational/_common.py (100%) rename dev/{integration/src => microsoft-agents-testing}/tests/integration/foundational/test_suite.py (100%) rename dev/{integration/src => microsoft-agents-testing}/tests/integration/test_quickstart.py (100%) rename dev/{integration/src => microsoft-agents-testing}/tests/manual_test.py (100%) create mode 100644 dev/microsoft-agents-testing/tests/test_framework/__init__.py create mode 100644 dev/microsoft-agents-testing/tests/test_framework/core/__init__.py rename dev/{integration/src => microsoft-agents-testing}/tests/test_framework/core/_common.py (100%) create mode 100644 dev/microsoft-agents-testing/tests/test_framework/core/client/__init__.py rename dev/{integration/src => microsoft-agents-testing}/tests/test_framework/core/client/_common.py (100%) rename dev/{integration/src => microsoft-agents-testing}/tests/test_framework/core/client/test_agent_client.py (100%) rename dev/{integration/src => microsoft-agents-testing}/tests/test_framework/core/client/test_response_client.py (100%) rename dev/{integration/src => microsoft-agents-testing}/tests/test_framework/core/test_application_runner.py (100%) rename dev/{integration/src => microsoft-agents-testing}/tests/test_framework/core/test_integration_from_sample.py (100%) rename dev/{integration/src => microsoft-agents-testing}/tests/test_framework/core/test_integration_from_service_url.py (100%) diff --git a/dev/microsoft-agents-testing/README.md b/dev/microsoft-agents-testing/README.md new file mode 100644 index 00000000..7ed3dd57 --- /dev/null +++ b/dev/microsoft-agents-testing/README.md @@ -0,0 +1 @@ +# Microsoft 365 Agents SDK for Python - Testing Framework diff --git a/dev/integration/src/tests/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py similarity index 100% rename from dev/integration/src/tests/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/__init__.py new file mode 100644 index 00000000..b01cfb1c --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/__init__.py @@ -0,0 +1,11 @@ +from .coroutine_executor import CoroutineExecutor +from .execution_result import ExecutionResult +from .executor import Executor +from .thread_executor import ThreadExecutor + +__all__ = [ + "CoroutineExecutor", + "ExecutionResult", + "Executor", + "ThreadExecutor", +] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/coroutine_executor.py b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/coroutine_executor.py new file mode 100644 index 00000000..5d03ff19 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/coroutine_executor.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +from typing import Callable, Awaitable, Any + +from .executor import Executor +from .execution_result import ExecutionResult + + +class CoroutineExecutor(Executor): + """An executor that runs asynchronous functions using asyncio.""" + + def run( + self, func: Callable[[], Awaitable[Any]], num_workers: int = 1 + ) -> list[ExecutionResult]: + """Run the given asynchronous function using the specified number of coroutines. + + :param func: An asynchronous function to be executed. + :param num_workers: The number of coroutines to use. + """ + + async def gather(): + return await asyncio.gather( + *[self.run_func(i, func) for i in range(num_workers)] + ) + + return asyncio.run(gather()) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/execution_result.py b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/execution_result.py new file mode 100644 index 00000000..ae72cabb --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/execution_result.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Any, Optional +from dataclasses import dataclass + + +@dataclass +class ExecutionResult: + """Class to represent the result of an execution.""" + + exe_id: int + + start_time: float + end_time: float + + result: Any = None + error: Optional[Exception] = None + + @property + def success(self) -> bool: + """Indicate whether the execution was successful.""" + return self.error is None + + @property + def duration(self) -> float: + """Calculate the duration of the execution, in seconds.""" + return self.end_time - self.start_time diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/executor.py b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/executor.py new file mode 100644 index 00000000..688c1cfb --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/executor.py @@ -0,0 +1,49 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from datetime import datetime, timezone +from abc import ABC, abstractmethod +from typing import Callable, Awaitable, Any + +from .execution_result import ExecutionResult + + +class Executor(ABC): + """Protocol for executing asynchronous functions concurrently.""" + + async def run_func( + self, exe_id: int, func: Callable[[], Awaitable[Any]] + ) -> ExecutionResult: + """Run the given asynchronous function. + + :param exe_id: An identifier for the execution instance. + :param func: An asynchronous function to be executed. + """ + + start_time = datetime.now(timezone.utc).timestamp() + try: + result = await func() + return ExecutionResult( + exe_id=exe_id, + result=result, + start_time=start_time, + end_time=datetime.now(timezone.utc).timestamp(), + ) + except Exception as e: # pylint: disable=broad-except + return ExecutionResult( + exe_id=exe_id, + error=e, + start_time=start_time, + end_time=datetime.now(timezone.utc).timestamp(), + ) + + @abstractmethod + def run( + self, func: Callable[[], Awaitable[Any]], num_workers: int = 1 + ) -> list[ExecutionResult]: + """Run the given asynchronous function using the specified number of workers. + + :param func: An asynchronous function to be executed. + :param num_workers: The number of concurrent workers to use. + """ + raise NotImplementedError("This method should be implemented by subclasses.") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/thread_executor.py b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/thread_executor.py new file mode 100644 index 00000000..ee3ce532 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/thread_executor.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import logging +import asyncio +from typing import Callable, Awaitable, Any +from concurrent.futures import ThreadPoolExecutor + +from .executor import Executor +from .execution_result import ExecutionResult + +logger = logging.getLogger(__name__) + + +class ThreadExecutor(Executor): + """An executor that runs asynchronous functions using multiple threads.""" + + def run( + self, func: Callable[[], Awaitable[Any]], num_workers: int = 1 + ) -> list[ExecutionResult]: + """Run the given asynchronous function using the specified number of threads. + + :param func: An asynchronous function to be executed. + :param num_workers: The number of concurrent threads to use. + """ + + def _func(exe_id: int) -> ExecutionResult: + return asyncio.run(self.run_func(exe_id, func)) + + results: list[ExecutionResult] = [] + + with ThreadPoolExecutor(max_workers=num_workers) as executor: + futures = [executor.submit(_func, i) for i in range(num_workers)] + for future in futures: + results.append(future.result()) + + return results diff --git a/dev/integration/src/tests/integration/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/__init__.py similarity index 100% rename from dev/integration/src/tests/integration/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/__init__.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/aggregated_results.py b/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/aggregated_results.py new file mode 100644 index 00000000..b1edaa5e --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/aggregated_results.py @@ -0,0 +1,51 @@ +from .executor import ExecutionResult + + +class AggregatedResults: + """Class to analyze execution time results.""" + + def __init__(self, results: list[ExecutionResult]): + self._results = results + + self.average = sum(r.duration for r in results) / len(results) if results else 0 + self.min = min((r.duration for r in results), default=0) + self.max = max((r.duration for r in results), default=0) + self.success_count = sum(1 for r in results if r.success) + self.failure_count = len(results) - self.success_count + self.total_time = sum(r.duration for r in results) + + def display(self, start_time: float, end_time: float): + """Display aggregated results.""" + print() + print("---- Aggregated Results ----") + print() + print(f"Average Time: {self.average:.4f} seconds") + print(f"Min Time: {self.min:.4f} seconds") + print(f"Max Time: {self.max:.4f} seconds") + print() + print(f"Success Rate: {self.success_count} / {len(self._results)}") + print() + print(f"Total Time: {end_time - start_time} seconds") + print("----------------------------") + print() + + def display_timeline(self): + """Display timeline of individual execution results.""" + print() + print("---- Execution Timeline ----") + print( + "Each '.' represents 1 second of successful execution. So a line like '...' is a success that took 3 seconds (rounded up), 'x' represents a failure." + ) + print() + for result in sorted(self._results, key=lambda r: r.exe_id): + c = "." if result.success else "x" + if c == ".": + duration = int(round(result.duration)) + for _ in range(1 + duration): + print(c, end="") + print() + else: + print(c) + + print("----------------------------") + print() diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/benchmark.py b/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/benchmark.py new file mode 100644 index 00000000..af9e177e --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/benchmark.py @@ -0,0 +1,49 @@ +import json +import logging +from datetime import datetime, timezone + +import click + +from .payload_sender import create_payload_sender +from .executor import Executor, CoroutineExecutor, ThreadExecutor +from .aggregated_results import AggregatedResults +from .config import BenchmarkConfig + +LOG_FORMAT = "%(asctime)s: %(message)s" +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO, datefmt="%H:%M:%S") + +BenchmarkConfig.load_from_env() + + +@click.command() +@click.option( + "--payload_path", "-p", default="./payload.json", help="Path to the payload file." +) +@click.option("--num_workers", "-n", default=1, help="Number of workers to use.") +@click.option( + "--async_mode", + "-a", + is_flag=True, + help="Run coroutine workers rather than thread workers.", +) +def main(payload_path: str, num_workers: int, async_mode: bool): + """Main function to run the benchmark.""" + + with open(payload_path, "r", encoding="utf-8") as f: + payload = json.load(f) + + func = create_payload_sender(payload) + + executor: Executor = CoroutineExecutor() if async_mode else ThreadExecutor() + + start_time = datetime.now(timezone.utc).timestamp() + results = executor.run(func, num_workers=num_workers) + end_time = datetime.now(timezone.utc).timestamp() + + agg = AggregatedResults(results) + agg.display(start_time, end_time) + agg.display_timeline() + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/config.py b/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/config.py new file mode 100644 index 00000000..403fbafc --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/config.py @@ -0,0 +1,23 @@ +import os +from dotenv import load_dotenv + +load_dotenv() + + +class BenchmarkConfig: + """Configuration class for benchmark settings.""" + + TENANT_ID: str = "" + APP_ID: str = "" + APP_SECRET: str = "" + AGENT_API_URL: str = "" + + @classmethod + def load_from_env(cls) -> None: + """Loads configuration values from environment variables.""" + cls.TENANT_ID = os.environ.get("TENANT_ID", "") + cls.APP_ID = os.environ.get("APP_ID", "") + cls.APP_SECRET = os.environ.get("APP_SECRET", "") + cls.AGENT_URL = os.environ.get( + "AGENT_API_URL", "http://localhost:3978/api/messages" + ) diff --git a/dev/integration/src/tests/integration/components/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli.py similarity index 100% rename from dev/integration/src/tests/integration/components/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/cli.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/common/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/common/__init__.py new file mode 100644 index 00000000..24aefe1f --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/common/__init__.py @@ -0,0 +1,7 @@ +from ._generate_token import generate_token +from ._payload_sender import create_payload_sender + +__all__ = [ + "generate_token", + "create_payload_sender", +] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/common/_generate_token.py b/dev/microsoft-agents-testing/microsoft_agents/testing/common/_generate_token.py new file mode 100644 index 00000000..19c0e93e --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/common/_generate_token.py @@ -0,0 +1,34 @@ +import requests +from .config import BenchmarkConfig + +URL = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" + + +def generate_token(app_id: str, app_secret: str) -> str: + """Generate a token using the provided app credentials.""" + + url = URL.format(tenant_id=BenchmarkConfig.TENANT_ID) + + res = requests.post( + url, + headers={ + "Content-Type": "application/x-www-form-urlencoded", + }, + data={ + "grant_type": "client_credentials", + "client_id": app_id, + "client_secret": app_secret, + "scope": f"{app_id}/.default", + }, + timeout=10, + ) + return res.json().get("access_token") + + +def generate_token_from_env() -> str: + """Generates a token using environment variables.""" + app_id = BenchmarkConfig.APP_ID + app_secret = BenchmarkConfig.APP_SECRET + if not app_id or not app_secret: + raise ValueError("APP_ID and APP_SECRET must be set in the BenchmarkConfig.") + return generate_token(app_id, app_secret) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/common/_payload_sender.py b/dev/microsoft-agents-testing/microsoft_agents/testing/common/_payload_sender.py new file mode 100644 index 00000000..a27f87c0 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/common/_payload_sender.py @@ -0,0 +1,32 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +import requests +from typing import Callable, Awaitable, Any + +from .config import BenchmarkConfig +from .generate_token import generate_token_from_env + + +def create_payload_sender( + payload: dict[str, Any], timeout: int = 60 +) -> Callable[..., Awaitable[Any]]: + """Create a payload sender function that sends the given payload to the configured endpoint. + + :param payload: The payload to be sent. + :param timeout: The timeout for the request in seconds. + :return: A callable that sends the payload when invoked. + """ + + token = generate_token_from_env() + endpoint = BenchmarkConfig.AGENT_URL + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + + async def payload_sender() -> Any: + response = await asyncio.to_thread( + requests.post, endpoint, headers=headers, json=payload, timeout=timeout + ) + return response.content + + return payload_sender diff --git a/dev/integration/src/tests/integration/foundational/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py similarity index 100% rename from dev/integration/src/tests/integration/foundational/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py diff --git a/dev/integration/src/tests/test_framework/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py similarity index 100% rename from dev/integration/src/tests/test_framework/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py diff --git a/dev/integration/src/tests/test_framework/core/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py similarity index 100% rename from dev/integration/src/tests/test_framework/core/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py diff --git a/dev/integration/src/tests/test_framework/core/client/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py similarity index 100% rename from dev/integration/src/tests/test_framework/core/client/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/auto_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/auto_client.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/utils.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/utils.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/quickstart_sample.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/quickstart_sample.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/pyproject.toml b/dev/microsoft-agents-testing/pyproject.toml new file mode 100644 index 00000000..c231347b --- /dev/null +++ b/dev/microsoft-agents-testing/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "microsoft-agents-hosting-core" +dynamic = ["version", "dependencies"] +description = "Core library for Microsoft Agents" +readme = {file = "readme.md", content-type = "text/markdown"} +authors = [{name = "Microsoft Corporation"}] +license = "MIT" +license-files = ["LICENSE"] +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://github.com/microsoft/Agents" diff --git a/dev/microsoft-agents-testing/setup.py b/dev/microsoft-agents-testing/setup.py new file mode 100644 index 00000000..02fb3e84 --- /dev/null +++ b/dev/microsoft-agents-testing/setup.py @@ -0,0 +1,18 @@ +from os import environ +from setuptools import setup + +package_version = environ.get("PackageVersion", "0.0.0") + +setup( + version=package_version, + install_requires=[ + "microsoft-agents-activity", + "microsoft-agents-hosting-core", + "microsoft-agents-authentication-msal", + "microsoft-agents-hosting-aiohttp", + "pyjwt>=2.10.1", + "isodate>=0.6.1", + "azure-core>=1.30.0", + "python-dotenv>=1.1.1", + ], +) diff --git a/dev/microsoft-agents-testing/tests/__init__.py b/dev/microsoft-agents-testing/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/env.TEMPLATE b/dev/microsoft-agents-testing/tests/env.TEMPLATE similarity index 100% rename from dev/integration/src/tests/env.TEMPLATE rename to dev/microsoft-agents-testing/tests/env.TEMPLATE diff --git a/dev/microsoft-agents-testing/tests/integration/__init__.py b/dev/microsoft-agents-testing/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/integration/components/__init__.py b/dev/microsoft-agents-testing/tests/integration/components/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/integration/components/test_typing_indicator.py b/dev/microsoft-agents-testing/tests/integration/components/test_typing_indicator.py similarity index 100% rename from dev/integration/src/tests/integration/components/test_typing_indicator.py rename to dev/microsoft-agents-testing/tests/integration/components/test_typing_indicator.py diff --git a/dev/microsoft-agents-testing/tests/integration/foundational/__init__.py b/dev/microsoft-agents-testing/tests/integration/foundational/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/integration/foundational/_common.py b/dev/microsoft-agents-testing/tests/integration/foundational/_common.py similarity index 100% rename from dev/integration/src/tests/integration/foundational/_common.py rename to dev/microsoft-agents-testing/tests/integration/foundational/_common.py diff --git a/dev/integration/src/tests/integration/foundational/test_suite.py b/dev/microsoft-agents-testing/tests/integration/foundational/test_suite.py similarity index 100% rename from dev/integration/src/tests/integration/foundational/test_suite.py rename to dev/microsoft-agents-testing/tests/integration/foundational/test_suite.py diff --git a/dev/integration/src/tests/integration/test_quickstart.py b/dev/microsoft-agents-testing/tests/integration/test_quickstart.py similarity index 100% rename from dev/integration/src/tests/integration/test_quickstart.py rename to dev/microsoft-agents-testing/tests/integration/test_quickstart.py diff --git a/dev/integration/src/tests/manual_test.py b/dev/microsoft-agents-testing/tests/manual_test.py similarity index 100% rename from dev/integration/src/tests/manual_test.py rename to dev/microsoft-agents-testing/tests/manual_test.py diff --git a/dev/microsoft-agents-testing/tests/test_framework/__init__.py b/dev/microsoft-agents-testing/tests/test_framework/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/__init__.py b/dev/microsoft-agents-testing/tests/test_framework/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/test_framework/core/_common.py b/dev/microsoft-agents-testing/tests/test_framework/core/_common.py similarity index 100% rename from dev/integration/src/tests/test_framework/core/_common.py rename to dev/microsoft-agents-testing/tests/test_framework/core/_common.py diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/client/__init__.py b/dev/microsoft-agents-testing/tests/test_framework/core/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/src/tests/test_framework/core/client/_common.py b/dev/microsoft-agents-testing/tests/test_framework/core/client/_common.py similarity index 100% rename from dev/integration/src/tests/test_framework/core/client/_common.py rename to dev/microsoft-agents-testing/tests/test_framework/core/client/_common.py diff --git a/dev/integration/src/tests/test_framework/core/client/test_agent_client.py b/dev/microsoft-agents-testing/tests/test_framework/core/client/test_agent_client.py similarity index 100% rename from dev/integration/src/tests/test_framework/core/client/test_agent_client.py rename to dev/microsoft-agents-testing/tests/test_framework/core/client/test_agent_client.py diff --git a/dev/integration/src/tests/test_framework/core/client/test_response_client.py b/dev/microsoft-agents-testing/tests/test_framework/core/client/test_response_client.py similarity index 100% rename from dev/integration/src/tests/test_framework/core/client/test_response_client.py rename to dev/microsoft-agents-testing/tests/test_framework/core/client/test_response_client.py diff --git a/dev/integration/src/tests/test_framework/core/test_application_runner.py b/dev/microsoft-agents-testing/tests/test_framework/core/test_application_runner.py similarity index 100% rename from dev/integration/src/tests/test_framework/core/test_application_runner.py rename to dev/microsoft-agents-testing/tests/test_framework/core/test_application_runner.py diff --git a/dev/integration/src/tests/test_framework/core/test_integration_from_sample.py b/dev/microsoft-agents-testing/tests/test_framework/core/test_integration_from_sample.py similarity index 100% rename from dev/integration/src/tests/test_framework/core/test_integration_from_sample.py rename to dev/microsoft-agents-testing/tests/test_framework/core/test_integration_from_sample.py diff --git a/dev/integration/src/tests/test_framework/core/test_integration_from_service_url.py b/dev/microsoft-agents-testing/tests/test_framework/core/test_integration_from_service_url.py similarity index 100% rename from dev/integration/src/tests/test_framework/core/test_integration_from_service_url.py rename to dev/microsoft-agents-testing/tests/test_framework/core/test_integration_from_service_url.py From 49161c357f922c8c1eb7e506c125f212359f4d0f Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 09:36:33 -0800 Subject: [PATCH 41/81] Polished the testing framework package --- .../microsoft_agents/testing/__init__.py | 36 +++++ .../testing/_executors/__init__.py | 11 -- .../testing/_executors/coroutine_executor.py | 28 ---- .../testing/_executors/execution_result.py | 28 ---- .../testing/_executors/executor.py | 49 ------ .../testing/_executors/thread_executor.py | 37 ----- .../microsoft_agents/testing/auth/__init__.py | 6 + .../testing/auth/generate_token.py | 47 ++++++ .../testing/benchmark/aggregated_results.py | 51 ------ .../testing/benchmark/benchmark.py | 49 ------ .../testing/benchmark/config.py | 23 --- .../microsoft_agents/testing/cli.py | 0 .../testing/common/__init__.py | 7 - .../testing/common/_generate_token.py | 34 ---- .../testing/common/_payload_sender.py | 32 ---- .../testing/integration/__init__.py | 19 +++ .../testing/integration/core/__init__.py | 20 +++ .../integration/core/aiohttp/__init__.py | 4 + .../core/aiohttp/aiohttp_environment.py | 58 +++++++ .../core/aiohttp/aiohttp_runner.py | 115 +++++++++++++ .../integration/core/application_runner.py | 42 +++++ .../integration/core/client/__init__.py | 7 + .../integration/core/client/agent_client.py | 152 ++++++++++++++++++ .../integration/core/client/auto_client.py | 18 +++ .../core/client/response_client.py | 92 +++++++++++ .../testing/integration/core/environment.py | 37 +++++ .../testing/integration/core/integration.py | 113 +++++++++++++ .../testing/integration/core/sample.py | 19 +++ .../testing/integration/core/utils.py | 0 .../integration/samples/quickstart_sample.py | 0 .../microsoft_agents/testing/sdk_config.py | 39 +++++ .../testing/utils/__init__.py | 7 + .../testing/utils/populate_activity.py | 45 ++++++ .../microsoft_agents/testing/utils/urls.py | 9 ++ .../components/test_typing_indicator.py | 36 ----- .../integration/core}/__init__.py | 0 .../core/_common.py | 2 +- .../integration/core/client}/__init__.py | 0 .../core/client/_common.py | 0 .../core/client/test_agent_client.py | 3 +- .../core/client/test_response_client.py | 2 +- .../core/test_application_runner.py | 1 - .../core/test_integration_from_sample.py | 15 +- .../core/test_integration_from_service_url.py | 11 +- .../integration/foundational/__init__.py | 0 .../tests/integration/foundational/_common.py | 10 -- .../integration/foundational/test_suite.py | 141 ---------------- .../tests/integration/test_quickstart.py | 20 --- .../components => manual_test}/__init__.py | 0 .../tests/{ => manual_test}/env.TEMPLATE | 0 .../{manual_test.py => manual_test/main.py} | 17 +- .../tests/samples/__init__.py | 3 + .../tests/samples/quickstart_sample.py | 57 +++++++ .../tests/test_framework/__init__.py | 0 .../tests/test_framework/core/__init__.py | 0 .../test_framework/core/client/__init__.py | 0 56 files changed, 972 insertions(+), 580 deletions(-) delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/_executors/__init__.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/_executors/coroutine_executor.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/_executors/execution_result.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/_executors/executor.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/_executors/thread_executor.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/aggregated_results.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/benchmark.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/config.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/cli.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/common/__init__.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/common/_generate_token.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/common/_payload_sender.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/utils.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/quickstart_sample.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py delete mode 100644 dev/microsoft-agents-testing/tests/integration/components/test_typing_indicator.py rename dev/microsoft-agents-testing/{microsoft_agents/testing/benchmark => tests/integration/core}/__init__.py (100%) rename dev/microsoft-agents-testing/tests/{test_framework => integration}/core/_common.py (84%) rename dev/microsoft-agents-testing/{microsoft_agents/testing/integration/samples => tests/integration/core/client}/__init__.py (100%) rename dev/microsoft-agents-testing/tests/{test_framework => integration}/core/client/_common.py (100%) rename dev/microsoft-agents-testing/tests/{test_framework => integration}/core/client/test_agent_client.py (98%) rename dev/microsoft-agents-testing/tests/{test_framework => integration}/core/client/test_response_client.py (96%) rename dev/microsoft-agents-testing/tests/{test_framework => integration}/core/test_application_runner.py (99%) rename dev/microsoft-agents-testing/tests/{test_framework => integration}/core/test_integration_from_sample.py (85%) rename dev/microsoft-agents-testing/tests/{test_framework => integration}/core/test_integration_from_service_url.py (87%) delete mode 100644 dev/microsoft-agents-testing/tests/integration/foundational/__init__.py delete mode 100644 dev/microsoft-agents-testing/tests/integration/foundational/_common.py delete mode 100644 dev/microsoft-agents-testing/tests/integration/foundational/test_suite.py delete mode 100644 dev/microsoft-agents-testing/tests/integration/test_quickstart.py rename dev/microsoft-agents-testing/tests/{integration/components => manual_test}/__init__.py (100%) rename dev/microsoft-agents-testing/tests/{ => manual_test}/env.TEMPLATE (100%) rename dev/microsoft-agents-testing/tests/{manual_test.py => manual_test/main.py} (77%) create mode 100644 dev/microsoft-agents-testing/tests/samples/__init__.py create mode 100644 dev/microsoft-agents-testing/tests/samples/quickstart_sample.py delete mode 100644 dev/microsoft-agents-testing/tests/test_framework/__init__.py delete mode 100644 dev/microsoft-agents-testing/tests/test_framework/core/__init__.py delete mode 100644 dev/microsoft-agents-testing/tests/test_framework/core/client/__init__.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py index e69de29b..0de64c77 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py @@ -0,0 +1,36 @@ +from .sdk_config import SDKConfig + +from .auth import ( + generate_token, + generate_token_from_config +) + +from .utils import ( + populate_activity, + get_host_and_port +) + +from .integration import ( + Sample, + Environment, + ApplicationRunner, + AgentClient, + ResponseClient, + AiohttpEnvironment, + Integration +) + +__all__ = [ + "SDKConfig", + "generate_token", + "generate_token_from_config", + "Sample", + "Environment", + "ApplicationRunner", + "AgentClient", + "ResponseClient", + "AiohttpEnvironment", + "Integration", + "populate_activity", + "get_host_and_port" +] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/__init__.py deleted file mode 100644 index b01cfb1c..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from .coroutine_executor import CoroutineExecutor -from .execution_result import ExecutionResult -from .executor import Executor -from .thread_executor import ThreadExecutor - -__all__ = [ - "CoroutineExecutor", - "ExecutionResult", - "Executor", - "ThreadExecutor", -] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/coroutine_executor.py b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/coroutine_executor.py deleted file mode 100644 index 5d03ff19..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/coroutine_executor.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import asyncio -from typing import Callable, Awaitable, Any - -from .executor import Executor -from .execution_result import ExecutionResult - - -class CoroutineExecutor(Executor): - """An executor that runs asynchronous functions using asyncio.""" - - def run( - self, func: Callable[[], Awaitable[Any]], num_workers: int = 1 - ) -> list[ExecutionResult]: - """Run the given asynchronous function using the specified number of coroutines. - - :param func: An asynchronous function to be executed. - :param num_workers: The number of coroutines to use. - """ - - async def gather(): - return await asyncio.gather( - *[self.run_func(i, func) for i in range(num_workers)] - ) - - return asyncio.run(gather()) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/execution_result.py b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/execution_result.py deleted file mode 100644 index ae72cabb..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/execution_result.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from typing import Any, Optional -from dataclasses import dataclass - - -@dataclass -class ExecutionResult: - """Class to represent the result of an execution.""" - - exe_id: int - - start_time: float - end_time: float - - result: Any = None - error: Optional[Exception] = None - - @property - def success(self) -> bool: - """Indicate whether the execution was successful.""" - return self.error is None - - @property - def duration(self) -> float: - """Calculate the duration of the execution, in seconds.""" - return self.end_time - self.start_time diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/executor.py b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/executor.py deleted file mode 100644 index 688c1cfb..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/executor.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from datetime import datetime, timezone -from abc import ABC, abstractmethod -from typing import Callable, Awaitable, Any - -from .execution_result import ExecutionResult - - -class Executor(ABC): - """Protocol for executing asynchronous functions concurrently.""" - - async def run_func( - self, exe_id: int, func: Callable[[], Awaitable[Any]] - ) -> ExecutionResult: - """Run the given asynchronous function. - - :param exe_id: An identifier for the execution instance. - :param func: An asynchronous function to be executed. - """ - - start_time = datetime.now(timezone.utc).timestamp() - try: - result = await func() - return ExecutionResult( - exe_id=exe_id, - result=result, - start_time=start_time, - end_time=datetime.now(timezone.utc).timestamp(), - ) - except Exception as e: # pylint: disable=broad-except - return ExecutionResult( - exe_id=exe_id, - error=e, - start_time=start_time, - end_time=datetime.now(timezone.utc).timestamp(), - ) - - @abstractmethod - def run( - self, func: Callable[[], Awaitable[Any]], num_workers: int = 1 - ) -> list[ExecutionResult]: - """Run the given asynchronous function using the specified number of workers. - - :param func: An asynchronous function to be executed. - :param num_workers: The number of concurrent workers to use. - """ - raise NotImplementedError("This method should be implemented by subclasses.") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/thread_executor.py b/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/thread_executor.py deleted file mode 100644 index ee3ce532..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/_executors/thread_executor.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import logging -import asyncio -from typing import Callable, Awaitable, Any -from concurrent.futures import ThreadPoolExecutor - -from .executor import Executor -from .execution_result import ExecutionResult - -logger = logging.getLogger(__name__) - - -class ThreadExecutor(Executor): - """An executor that runs asynchronous functions using multiple threads.""" - - def run( - self, func: Callable[[], Awaitable[Any]], num_workers: int = 1 - ) -> list[ExecutionResult]: - """Run the given asynchronous function using the specified number of threads. - - :param func: An asynchronous function to be executed. - :param num_workers: The number of concurrent threads to use. - """ - - def _func(exe_id: int) -> ExecutionResult: - return asyncio.run(self.run_func(exe_id, func)) - - results: list[ExecutionResult] = [] - - with ThreadPoolExecutor(max_workers=num_workers) as executor: - futures = [executor.submit(_func, i) for i in range(num_workers)] - for future in futures: - results.append(future.result()) - - return results diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py new file mode 100644 index 00000000..a34ef21f --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py @@ -0,0 +1,6 @@ +from .generate_token import generate_token, generate_token_from_config + +__all__ = [ + "generate_token", + "generate_token_from_config" +] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py new file mode 100644 index 00000000..029b05ab --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py @@ -0,0 +1,47 @@ +import requests + +from microsoft_agents.hosting.core import AgentAuthConfiguration +from microsoft_agents.testing.sdk_config import SDKConfig + +def generate_token(app_id: str, app_secret: str, tenant_id: str) -> str: + """Generate a token using the provided app credentials. + + :param app_id: Application (client) ID. + :param app_secret: Application client secret. + :param tenant_id: Directory (tenant) ID. + :return: Generated access token as a string. + """ + + authority_endpoint = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" + + res = requests.post( + authority_endpoint, + headers={ + "Content-Type": "application/x-www-form-urlencoded", + }, + data={ + "grant_type": "client_credentials", + "client_id": app_id, + "client_secret": app_secret, + "scope": f"{app_id}/.default", + }, + timeout=10, + ) + return res.json().get("access_token") + +def generate_token_from_config(sdk_config: SDKConfig) -> str: + """Generates a token using a provided config object. + + :param config: Configuration dictionary containing connection settings. + :return: Generated access token as a string. + """ + + settings: AgentAuthConfiguration = sdk_config.get_connection() + + app_id = settings.CLIENT_ID + app_secret = settings.CLIENT_SECRET + tenant_id = settings.TENANT_ID + + if not app_id or not app_secret or not tenant_id: + raise ValueError("Incorrect configuration provided for token generation.") + return generate_token(app_id, app_secret, tenant_id) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/aggregated_results.py b/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/aggregated_results.py deleted file mode 100644 index b1edaa5e..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/aggregated_results.py +++ /dev/null @@ -1,51 +0,0 @@ -from .executor import ExecutionResult - - -class AggregatedResults: - """Class to analyze execution time results.""" - - def __init__(self, results: list[ExecutionResult]): - self._results = results - - self.average = sum(r.duration for r in results) / len(results) if results else 0 - self.min = min((r.duration for r in results), default=0) - self.max = max((r.duration for r in results), default=0) - self.success_count = sum(1 for r in results if r.success) - self.failure_count = len(results) - self.success_count - self.total_time = sum(r.duration for r in results) - - def display(self, start_time: float, end_time: float): - """Display aggregated results.""" - print() - print("---- Aggregated Results ----") - print() - print(f"Average Time: {self.average:.4f} seconds") - print(f"Min Time: {self.min:.4f} seconds") - print(f"Max Time: {self.max:.4f} seconds") - print() - print(f"Success Rate: {self.success_count} / {len(self._results)}") - print() - print(f"Total Time: {end_time - start_time} seconds") - print("----------------------------") - print() - - def display_timeline(self): - """Display timeline of individual execution results.""" - print() - print("---- Execution Timeline ----") - print( - "Each '.' represents 1 second of successful execution. So a line like '...' is a success that took 3 seconds (rounded up), 'x' represents a failure." - ) - print() - for result in sorted(self._results, key=lambda r: r.exe_id): - c = "." if result.success else "x" - if c == ".": - duration = int(round(result.duration)) - for _ in range(1 + duration): - print(c, end="") - print() - else: - print(c) - - print("----------------------------") - print() diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/benchmark.py b/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/benchmark.py deleted file mode 100644 index af9e177e..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/benchmark.py +++ /dev/null @@ -1,49 +0,0 @@ -import json -import logging -from datetime import datetime, timezone - -import click - -from .payload_sender import create_payload_sender -from .executor import Executor, CoroutineExecutor, ThreadExecutor -from .aggregated_results import AggregatedResults -from .config import BenchmarkConfig - -LOG_FORMAT = "%(asctime)s: %(message)s" -logging.basicConfig(format=LOG_FORMAT, level=logging.INFO, datefmt="%H:%M:%S") - -BenchmarkConfig.load_from_env() - - -@click.command() -@click.option( - "--payload_path", "-p", default="./payload.json", help="Path to the payload file." -) -@click.option("--num_workers", "-n", default=1, help="Number of workers to use.") -@click.option( - "--async_mode", - "-a", - is_flag=True, - help="Run coroutine workers rather than thread workers.", -) -def main(payload_path: str, num_workers: int, async_mode: bool): - """Main function to run the benchmark.""" - - with open(payload_path, "r", encoding="utf-8") as f: - payload = json.load(f) - - func = create_payload_sender(payload) - - executor: Executor = CoroutineExecutor() if async_mode else ThreadExecutor() - - start_time = datetime.now(timezone.utc).timestamp() - results = executor.run(func, num_workers=num_workers) - end_time = datetime.now(timezone.utc).timestamp() - - agg = AggregatedResults(results) - agg.display(start_time, end_time) - agg.display_timeline() - - -if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/config.py b/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/config.py deleted file mode 100644 index 403fbafc..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/config.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -from dotenv import load_dotenv - -load_dotenv() - - -class BenchmarkConfig: - """Configuration class for benchmark settings.""" - - TENANT_ID: str = "" - APP_ID: str = "" - APP_SECRET: str = "" - AGENT_API_URL: str = "" - - @classmethod - def load_from_env(cls) -> None: - """Loads configuration values from environment variables.""" - cls.TENANT_ID = os.environ.get("TENANT_ID", "") - cls.APP_ID = os.environ.get("APP_ID", "") - cls.APP_SECRET = os.environ.get("APP_SECRET", "") - cls.AGENT_URL = os.environ.get( - "AGENT_API_URL", "http://localhost:3978/api/messages" - ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/cli.py b/dev/microsoft-agents-testing/microsoft_agents/testing/cli.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/common/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/common/__init__.py deleted file mode 100644 index 24aefe1f..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/common/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from ._generate_token import generate_token -from ._payload_sender import create_payload_sender - -__all__ = [ - "generate_token", - "create_payload_sender", -] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/common/_generate_token.py b/dev/microsoft-agents-testing/microsoft_agents/testing/common/_generate_token.py deleted file mode 100644 index 19c0e93e..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/common/_generate_token.py +++ /dev/null @@ -1,34 +0,0 @@ -import requests -from .config import BenchmarkConfig - -URL = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" - - -def generate_token(app_id: str, app_secret: str) -> str: - """Generate a token using the provided app credentials.""" - - url = URL.format(tenant_id=BenchmarkConfig.TENANT_ID) - - res = requests.post( - url, - headers={ - "Content-Type": "application/x-www-form-urlencoded", - }, - data={ - "grant_type": "client_credentials", - "client_id": app_id, - "client_secret": app_secret, - "scope": f"{app_id}/.default", - }, - timeout=10, - ) - return res.json().get("access_token") - - -def generate_token_from_env() -> str: - """Generates a token using environment variables.""" - app_id = BenchmarkConfig.APP_ID - app_secret = BenchmarkConfig.APP_SECRET - if not app_id or not app_secret: - raise ValueError("APP_ID and APP_SECRET must be set in the BenchmarkConfig.") - return generate_token(app_id, app_secret) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/common/_payload_sender.py b/dev/microsoft-agents-testing/microsoft_agents/testing/common/_payload_sender.py deleted file mode 100644 index a27f87c0..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/common/_payload_sender.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import asyncio -import requests -from typing import Callable, Awaitable, Any - -from .config import BenchmarkConfig -from .generate_token import generate_token_from_env - - -def create_payload_sender( - payload: dict[str, Any], timeout: int = 60 -) -> Callable[..., Awaitable[Any]]: - """Create a payload sender function that sends the given payload to the configured endpoint. - - :param payload: The payload to be sent. - :param timeout: The timeout for the request in seconds. - :return: A callable that sends the payload when invoked. - """ - - token = generate_token_from_env() - endpoint = BenchmarkConfig.AGENT_URL - headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} - - async def payload_sender() -> Any: - response = await asyncio.to_thread( - requests.post, endpoint, headers=headers, json=payload, timeout=timeout - ) - return response.content - - return payload_sender diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py index e69de29b..a283062f 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py @@ -0,0 +1,19 @@ +from .core import ( + AgentClient, + ApplicationRunner, + AiohttpEnvironment, + ResponseClient, + Environment, + Integration, + Sample, +) + +__all__ = [ + "AgentClient", + "ApplicationRunner", + "AiohttpEnvironment", + "ResponseClient", + "Environment", + "Integration", + "Sample", +] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py index e69de29b..9c69a2ae 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py @@ -0,0 +1,20 @@ +from .application_runner import ApplicationRunner +from .aiohttp import AiohttpEnvironment +from .client import ( + AgentClient, + ResponseClient, +) +from .environment import Environment +from .integration import Integration +from .sample import Sample + + +__all__ = [ + "AgentClient", + "ApplicationRunner", + "AiohttpEnvironment", + "ResponseClient", + "Environment", + "Integration", + "Sample", +] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py index e69de29b..82d2d1d0 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py @@ -0,0 +1,4 @@ +from .aiohttp_environment import AiohttpEnvironment +from .aiohttp_runner import AiohttpRunner + +__all__ = ["AiohttpEnvironment", "AiohttpRunner"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py index e69de29b..c8256618 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py @@ -0,0 +1,58 @@ +from tkinter import E +from aiohttp.web import Request, Response, Application, run_app + +from microsoft_agents.hosting.aiohttp import ( + CloudAdapter, + jwt_authorization_middleware, + start_agent_process, +) +from microsoft_agents.hosting.core import ( + Authorization, + AgentApplication, + TurnState, + MemoryStorage, +) +from microsoft_agents.authentication.msal import MsalConnectionManager +from microsoft_agents.activity import load_configuration_from_env + +from ..application_runner import ApplicationRunner +from ..environment import Environment +from .aiohttp_runner import AiohttpRunner + + +class AiohttpEnvironment(Environment): + """An environment for aiohttp-hosted agents.""" + + async def init_env(self, environ_config: dict) -> None: + environ_config = environ_config or {} + + self.config = load_configuration_from_env(environ_config) + + self.storage = MemoryStorage() + self.connection_manager = MsalConnectionManager(**self.config) + self.adapter = CloudAdapter(connection_manager=self.connection_manager) + self.authorization = Authorization( + self.storage, self.connection_manager, **self.config + ) + + self.agent_application = AgentApplication[TurnState]( + storage=self.storage, + adapter=self.adapter, + authorization=self.authorization, + **self.config + ) + + def create_runner(self, host: str, port: int) -> ApplicationRunner: + + async def entry_point(req: Request) -> Response: + agent: AgentApplication = req.app["agent_app"] + adapter: CloudAdapter = req.app["adapter"] + return await start_agent_process(req, agent, adapter) + + APP = Application(middlewares=[jwt_authorization_middleware]) + APP.router.add_post("/api/messages", entry_point) + APP["agent_configuration"] = self.connection_manager.get_default_connection_configuration() + APP["agent_app"] = self.agent_application + APP["adapter"] = self.adapter + + return AiohttpRunner(APP, host, port) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py index e69de29b..3b6780d4 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py @@ -0,0 +1,115 @@ +from typing import Optional +from typing import Optional +from threading import Thread, Event +import asyncio + +from aiohttp import ClientSession +from aiohttp.web import Application, Request, Response +from aiohttp.web_runner import AppRunner, TCPSite + +from ..application_runner import ApplicationRunner + + +class AiohttpRunner(ApplicationRunner): + """A runner for aiohttp applications.""" + + def __init__(self, app: Application, host: str = "localhost", port: int = 8000): + assert isinstance(app, Application) + super().__init__(app) + + url = f"{host}:{port}" + self._host = host + self._port = port + if "http" not in url: + url = f"http://{url}" + self._url = url + + self._app.router.add_get("/shutdown", self._shutdown_route) + + self._server_thread: Optional[Thread] = None + self._shutdown_event = Event() + self._runner: Optional[AppRunner] = None + self._site: Optional[TCPSite] = None + + @property + def url(self) -> str: + return self._url + + async def _start_server(self) -> None: + try: + assert isinstance(self._app, Application) + + self._runner = AppRunner(self._app) + await self._runner.setup() + self._site = TCPSite(self._runner, self._host, self._port) + await self._site.start() + + # Wait for shutdown signal + while not self._shutdown_event.is_set(): + await asyncio.sleep(0.1) + + # Cleanup + await self._site.stop() + await self._runner.cleanup() + + except Exception as error: + raise error + + async def __aenter__(self): + if self._server_thread: + raise RuntimeError("ResponseClient is already running.") + + self._shutdown_event.clear() + self._server_thread = Thread( + target=lambda: asyncio.run(self._start_server()), daemon=True + ) + self._server_thread.start() + + # Wait a moment to ensure the server starts + await asyncio.sleep(0.5) + + return self + + async def _stop_server(self): + if not self._server_thread: + raise RuntimeError("ResponseClient is not running.") + + try: + async with ClientSession() as session: + async with session.get( + f"http://{self._host}:{self._port}/shutdown" + ) as response: + pass # Just trigger the shutdown + except Exception: + pass # Ignore errors during shutdown request + + # Set shutdown event as fallback + self._shutdown_event.set() + + # Wait for the server thread to finish + self._server_thread.join(timeout=5.0) + self._server_thread = None + + async def _shutdown_route(self, request: Request) -> Response: + """Handle shutdown request by setting the shutdown event""" + self._shutdown_event.set() + return Response(status=200, text="Shutdown initiated") + + async def __aexit__(self, exc_type, exc, tb): + if not self._server_thread: + raise RuntimeError("ResponseClient is not running.") + try: + async with ClientSession() as session: + async with session.get( + f"http://{self._host}:{self._port}/shutdown" + ) as response: + pass # Just trigger the shutdown + except Exception: + pass # Ignore errors during shutdown request + + # Set shutdown event as fallback + self._shutdown_event.set() + + # Wait for the server thread to finish + self._server_thread.join(timeout=5.0) + self._server_thread = None diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py index e69de29b..ebbc56f9 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py @@ -0,0 +1,42 @@ +import asyncio +from abc import ABC, abstractmethod +from typing import Any, Optional +from threading import Thread + + +class ApplicationRunner(ABC): + """Base class for application runners.""" + + def __init__(self, app: Any): + self._app = app + self._thread: Optional[Thread] = None + + @abstractmethod + async def _start_server(self) -> None: + raise NotImplementedError( + "Start server method must be implemented by subclasses" + ) + + async def _stop_server(self) -> None: + pass + + async def __aenter__(self) -> None: + + if self._thread: + raise RuntimeError("Server is already running") + + def target(): + asyncio.run(self._start_server()) + + self._thread = Thread(target=target, daemon=True) + self._thread.start() + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + + if self._thread: + await self._stop_server() + + self._thread.join() + self._thread = None + else: + raise RuntimeError("Server is not running") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py index e69de29b..1d59411e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py @@ -0,0 +1,7 @@ +from .agent_client import AgentClient +from .response_client import ResponseClient + +__all__ = [ + "AgentClient", + "ResponseClient", +] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py index e69de29b..7330bc98 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py @@ -0,0 +1,152 @@ +from email.policy import default +import os +import json +import asyncio +from typing import Optional, cast + +from aiohttp import ClientSession +from msal import ConfidentialClientApplication + +from microsoft_agents.activity import ( + Activity, + ActivityTypes, + DeliveryModes, + ChannelAccount, + ConversationAccount +) +from microsoft_agents.testing.utils import populate_activity + +_DEFAULT_ACTIVITY_VALUES = { + "service_url": "http://localhost", + "channel_id": "test_channel", + "from_property": ChannelAccount(id="sender"), + "recipient": ChannelAccount(id="recipient"), + "locale": "en-US", +} + +class AgentClient: + + def __init__( + self, + agent_url: str, + cid: str, + client_id: str, + tenant_id: str, + client_secret: str, + service_url: Optional[str] = None, + default_timeout: float = 5.0, + default_activity_data: Optional[Activity | dict] = None, + ): + self._agent_url = agent_url + self._cid = cid + self._client_id = client_id + self._tenant_id = tenant_id + self._client_secret = client_secret + self._service_url = service_url + self._headers = None + self._default_timeout = default_timeout + + self._client: Optional[ClientSession] = None + + self._default_activity_data: Activity | dict = default_activity_data or _DEFAULT_ACTIVITY_VALUES + + @property + def agent_url(self) -> str: + return self._agent_url + + @property + def service_url(self) -> Optional[str]: + return self._service_url + + async def get_access_token(self) -> str: + + msal_app = ConfidentialClientApplication( + client_id=self._client_id, + client_credential=self._client_secret, + authority=f"https://login.microsoftonline.com/{self._tenant_id}", + ) + + res = msal_app.acquire_token_for_client(scopes=[f"{self._client_id}/.default"]) + token = res.get("access_token") if res else None + if not token: + raise Exception("Could not obtain access token") + return token + + async def _init_client(self) -> None: + if not self._client: + if self._client_secret: + token = await self.get_access_token() + self._headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + else: + self._headers = {"Content-Type": "application/json"} + + self._client = ClientSession( + base_url=self._agent_url, headers=self._headers + ) + + async def send_request(self, activity: Activity, sleep: float = 0) -> str: + + await self._init_client() + assert self._client + + if activity.conversation: + activity.conversation.id = self._cid + else: + activity.conversation = ConversationAccount(id=self._cid or "") + + if self.service_url: + activity.service_url = self.service_url + + activity = populate_activity(activity, self._default_activity_data) + + async with self._client.post( + "api/messages", + headers=self._headers, + json=activity.model_dump(by_alias=True, exclude_unset=True, exclude_none=True, mode="json"), + ) as response: + content = await response.text() + if not response.ok: + raise Exception(f"Failed to send activity: {response.status}") + await asyncio.sleep(sleep) + return content + + def _to_activity(self, activity_or_text: Activity | str) -> Activity: + if isinstance(activity_or_text, str): + activity = Activity( + type=ActivityTypes.message, + text=activity_or_text, + ) + return activity + else: + return cast(Activity, activity_or_text) + + async def send_activity( + self, activity_or_text: Activity | str, sleep: float = 0, timeout: Optional[float] = None + ) -> str: + timeout = timeout or self._default_timeout + activity = self._to_activity(activity_or_text) + content = await self.send_request(activity, sleep=sleep) + return content + + async def send_expect_replies( + self, activity_or_text: Activity | str, sleep: float = 0, timeout: Optional[float] = None + ) -> list[Activity]: + timeout = timeout or self._default_timeout + activity = self._to_activity(activity_or_text) + activity.delivery_mode = DeliveryModes.expect_replies + activity.service_url = activity.service_url or "http://localhost" # temporary fix + + content = await self.send_request(activity, sleep=sleep) + + activities_data = json.loads(content).get("activities", []) + activities = [Activity.model_validate(act) for act in activities_data] + + return activities + + async def close(self) -> None: + if self._client: + await self._client.close() + self._client = None diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/auto_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/auto_client.py index e69de29b..dcea531b 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/auto_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/auto_client.py @@ -0,0 +1,18 @@ +# from microsoft_agents.activity import Activity + +# from ..agent_client import AgentClient + +# class AutoClient: + +# def __init__(self, agent_client: AgentClient): +# self._agent_client = agent_client + +# async def generate_message(self) -> str: +# pass + +# async def run(self, max_turns: int = 10, time_between_turns: float = 2.0) -> None: + +# for i in range(max_turns): +# await self._agent_client.send_activity( +# Activity(type="message", text=self.generate_message()) +# ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py index e69de29b..d93bfb80 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import sys +from io import StringIO +from typing import Optional +from threading import Lock, Thread, Event +import asyncio + +from aiohttp.web import Application, Request, Response + +from microsoft_agents.activity import ( + Activity, + ActivityTypes, +) + +from ..aiohttp import AiohttpRunner + + +class ResponseClient: + + def __init__( + self, + host: str = "localhost", + port: int = 9873, + ): + self._app: Application = Application() + self._prev_stdout = None + service_endpoint = f"{host}:{port}" + self._host = host + self._port = port + if "http" not in service_endpoint: + service_endpoint = f"http://{service_endpoint}" + self._service_endpoint = service_endpoint + self._activities_list = [] + self._activities_list_lock = Lock() + + self._app.router.add_post( + "/v3/conversations/{path:.*}", self._handle_conversation + ) + + self._app_runner = AiohttpRunner(self._app, host, port) + + @property + def service_endpoint(self) -> str: + return self._service_endpoint + + async def __aenter__(self) -> ResponseClient: + self._prev_stdout = sys.stdout + sys.stdout = StringIO() + + await self._app_runner.__aenter__() + + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + if self._prev_stdout is not None: + sys.stdout = self._prev_stdout + + await self._app_runner.__aexit__(exc_type, exc_val, exc_tb) + + async def _handle_conversation(self, request: Request) -> Response: + try: + data = await request.json() + activity = Activity.model_validate(data) + + conversation_id = ( + activity.conversation.id if activity.conversation else None + ) + + with self._activities_list_lock: + self._activities_list.append(activity) + + if any(map(lambda x: x.type == "streaminfo", activity.entities or [])): + await self._handle_streamed_activity(activity) + return Response(status=200, text="Stream info handled") + else: + if activity.type != ActivityTypes.typing: + await asyncio.sleep(0.1) # Simulate processing delay + return Response(status=200, text="Activity received") + except Exception as e: + return Response(status=500, text=str(e)) + + async def _handle_streamed_activity( + self, activity: Activity, *args, **kwargs + ) -> bool: + raise NotImplementedError("_handle_streamed_activity is not implemented yet.") + + async def pop(self) -> list[Activity]: + with self._activities_list_lock: + activities = self._activities_list[:] + self._activities_list.clear() + return activities diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py index e69de29b..2c9b1ae1 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py @@ -0,0 +1,37 @@ +from abc import ABC, abstractmethod +from typing import Awaitable, Callable + +from microsoft_agents.hosting.core import ( + AgentApplication, + ChannelAdapter, + Connections, + Authorization, + Storage, + TurnState, +) + +from .application_runner import ApplicationRunner + + +class Environment(ABC): + """A sample data object for integration tests.""" + + agent_application: AgentApplication[TurnState] + storage: Storage + adapter: ChannelAdapter + connection_manager: Connections + authorization: Authorization + + config: dict + + driver: Callable[[], Awaitable[None]] + + @abstractmethod + async def init_env(self, environ_config: dict) -> None: + """Initialize the environment.""" + raise NotImplementedError() + + @abstractmethod + def create_runner(self) -> ApplicationRunner: + """Create an application runner for the environment.""" + raise NotImplementedError() diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py index e69de29b..c3105c74 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py @@ -0,0 +1,113 @@ +import pytest + +import os +from typing import ( + Optional, + TypeVar, + Any, + AsyncGenerator, +) + +import aiohttp.web +from dotenv import load_dotenv + +from microsoft_agents.testing.utils import get_host_and_port +from .environment import Environment +from .client import AgentClient, ResponseClient +from .sample import Sample + +T = TypeVar("T", bound=type) +AppT = TypeVar("AppT", bound=aiohttp.web.Application) # for future extension w/ Union + + +class Integration: + """Provides integration test fixtures.""" + + _sample_cls: Optional[type[Sample]] = None + _environment_cls: Optional[type[Environment]] = None + + _config: dict[str, Any] = {} + + _service_url: Optional[str] = None + _agent_url: Optional[str] = None + _cid: Optional[str] = None + _client_id: Optional[str] = None + _tenant_id: Optional[str] = None + _client_secret: Optional[str] = None + + _environment: Environment + _sample: Sample + _agent_client: AgentClient + _response_client: ResponseClient + + @property + def service_url(self) -> str: + return self._service_url or self._config.get("service_url", "") + + @property + def agent_url(self) -> str: + return self._agent_url or self._config.get("agent_url", "") + + @pytest.fixture + async def environment(self): + """Provides the test environment instance.""" + if self._environment_cls: + assert self._sample_cls + environment = self._environment_cls() + await environment.init_env(await self._sample_cls.get_config()) + yield environment + else: + yield None + + @pytest.fixture + async def sample(self, environment): + """Provides the sample instance.""" + if environment: + assert self._sample_cls + sample = self._sample_cls(environment) + await sample.init_app() + # host, port = get_host_and_port(self.agent_url) + app_runner = environment.create_runner(sample.app) + async with app_runner: + yield sample + else: + yield None + + def create_agent_client(self) -> AgentClient: + if not self._config: + self._config = {} + + load_dotenv("./src/tests/.env") + self._config.update( + { + "client_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", ""), + "tenant_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID", ""), + "client_secret": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET", ""), + } + ) + + agent_client = AgentClient( + agent_url=self.agent_url, + cid=self._cid or self._config.get("cid", ""), + client_id=self._client_id or self._config.get("client_id", ""), + tenant_id=self._tenant_id or self._config.get("tenant_id", ""), + client_secret=self._client_secret or self._config.get("client_secret", ""), + ) + return agent_client + + @pytest.fixture + async def agent_client(self, sample, environment) -> AsyncGenerator[AgentClient, None]: + agent_client = self.create_agent_client() + yield agent_client + await agent_client.close() + + async def _create_response_client(self) -> ResponseClient: + host, port = get_host_and_port(self.service_url) + assert host and port + return ResponseClient(host=host, port=port) + + @pytest.fixture + async def response_client(self) -> AsyncGenerator[ResponseClient, None]: + """Provides the response client instance.""" + async with await self._create_response_client() as response_client: + yield response_client \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py index e69de29b..6dde3668 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py @@ -0,0 +1,19 @@ +from abc import ABC, abstractmethod + +from .environment import Environment + + +class Sample(ABC): + """Base class for all samples.""" + + def __init__(self, environment: Environment, **kwargs): + self.env = environment + + @classmethod + async def get_config(cls) -> dict: + """Retrieve the configuration for the sample.""" + return {} + + @abstractmethod + async def init_app(self): + """Initialize the application for the sample.""" diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/utils.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/utils.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/quickstart_sample.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/quickstart_sample.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py new file mode 100644 index 00000000..6352b42c --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py @@ -0,0 +1,39 @@ +import os +from copy import deepcopy +from dotenv import load_dotenv, dotenv_values +from typing import Optional + +from microsoft_agents.activity import ( + load_configuration_from_env +) +from microsoft_agents.hosting.core import ( + AgentAuthConfiguration +) + +class SDKConfig: + """Loads and provides access to SDK configuration from a .env file or environment variables. + + Immutable access to the configuration dictionary is provided via the `config` property. + """ + + def __init__(self, env_path: Optional[str] = None, load_into_environment: bool = False): + """Initializes the SDKConfig by loading configuration from a .env file or environment variables. + + :param env_path: Optional path to the .env file. If None, defaults to '.env' in the current directory. + :param load_into_environment: If True, loads the .env file directly into the configuration dictionary. + """ + if load_into_environment: + self._config = load_configuration_from_env(dotenv_values(env_path)) # Load .env file + else: + load_dotenv(env_path) # Load .env file into environment variables + self._config = load_configuration_from_env(os.environ) # Load from environment variables + + @property + def config(self) -> dict: + """Returns the loaded configuration dictionary.""" + return deepcopy(self._config) + + def get_connection(self, connection_name: str = "SERVICE_CONNECTION") -> AgentAuthConfiguration: + """Creates an AgentAuthConfiguration from a provided config object.""" + data = self._config["CONNECTIONS"][connection_name]["SETTINGS"] + return AgentAuthConfiguration(**data) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py new file mode 100644 index 00000000..c8c26208 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -0,0 +1,7 @@ +from .populate_activity import populate_activity +from .urls import get_host_and_port + +__all__ = [ + "populate_activity", + "get_host_and_port", +] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py new file mode 100644 index 00000000..5ba73ec6 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py @@ -0,0 +1,45 @@ +from microsoft_agents.activity import Activity + +def populate_activity( + original: Activity, + defaults: Activity | dict +) -> Activity: + + if isinstance(defaults, Activity): + defaults = Activity.model_dump(exclude_unset=True) + + new_activity = original.model_copy() + + for key in defaults.keys(): + if getattr(new_activity, key) is None: + setattr(new_activity, key, defaults[key]) + + return new_activity + + +# def _populate_incoming_activity(activity: Activity) -> Activity: + +# activity = activity.model_copy() + +# if not activity.locale: +# activity.locale = "en-US" + +# if not activity.channel_id: +# activity.channel_id = "emulator" + +# if not activity.delivery_mode: +# activity.delivery_mode = DeliveryModes.normal + +# if not activity.service_url: +# activity.service_url = "http://localhost" + +# if not activity.recipient: +# activity.recipient = ChannelAccount(id="agent", name="Agent") + +# if not activity.from_property: +# activity.from_property = ChannelAccount(id="user", name="User") + +# if not activity.conversation: +# activity.conversation = ConversationAccount(id="conversation1") + +# return activity \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py new file mode 100644 index 00000000..df452416 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py @@ -0,0 +1,9 @@ +from urllib.parse import urlparse + +def get_host_and_port(url: str) -> tuple[str, int]: + """Extract host and port from a URL.""" + + parsed_url = urlparse(url) + host = parsed_url.hostname or "localhost" + port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) + return host, port \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/integration/components/test_typing_indicator.py b/dev/microsoft-agents-testing/tests/integration/components/test_typing_indicator.py deleted file mode 100644 index a51613c8..00000000 --- a/dev/microsoft-agents-testing/tests/integration/components/test_typing_indicator.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest -import asyncio - -from microsoft_agents.activity import ( - Activity, - ActivityTypes, - ChannelAccount -) - -from src.core import integration, IntegrationFixtures, AiohttpEnvironment -from src.samples import QuickstartSample - -@integration(sample=QuickstartSample, environment=AiohttpEnvironment) -class TestTypingIndicator(IntegrationFixtures): - - @pytest.mark.asyncio - async def test_typing_indicator(self, agent_client, response_client): - - activity_base = Activity( - type=ActivityTypes.message, - from_property={"id": "user1", "name": "User 1"}, - recipient={"id": "agent", "name": "Agent"}, - conversation={"id": "conv1"}, - channel_id="test_channel" - ) - - activity_a = activity_base.model_copy() - activity_b = activity_base.model_copy() - - activity_a.from_property = ChannelAccount(id="user1", name="User 1") - activity_b.from_property = ChannelAccount(id="user2", name="User 2") - - await asyncio.gather( - agent_client.send_activity(activity_a), - agent_client.send_activity(activity_b) - ) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/__init__.py b/dev/microsoft-agents-testing/tests/integration/core/__init__.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/benchmark/__init__.py rename to dev/microsoft-agents-testing/tests/integration/core/__init__.py diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/_common.py b/dev/microsoft-agents-testing/tests/integration/core/_common.py similarity index 84% rename from dev/microsoft-agents-testing/tests/test_framework/core/_common.py rename to dev/microsoft-agents-testing/tests/integration/core/_common.py index c8dd0098..cd22114a 100644 --- a/dev/microsoft-agents-testing/tests/test_framework/core/_common.py +++ b/dev/microsoft-agents-testing/tests/integration/core/_common.py @@ -1,4 +1,4 @@ -from src.core import ApplicationRunner +from microsoft_agents.testing import ApplicationRunner class SimpleRunner(ApplicationRunner): diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/__init__.py b/dev/microsoft-agents-testing/tests/integration/core/client/__init__.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/integration/samples/__init__.py rename to dev/microsoft-agents-testing/tests/integration/core/client/__init__.py diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/client/_common.py b/dev/microsoft-agents-testing/tests/integration/core/client/_common.py similarity index 100% rename from dev/microsoft-agents-testing/tests/test_framework/core/client/_common.py rename to dev/microsoft-agents-testing/tests/integration/core/client/_common.py diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/client/test_agent_client.py b/dev/microsoft-agents-testing/tests/integration/core/client/test_agent_client.py similarity index 98% rename from dev/microsoft-agents-testing/tests/test_framework/core/client/test_agent_client.py rename to dev/microsoft-agents-testing/tests/integration/core/client/test_agent_client.py index 5708ccce..1a8055dc 100644 --- a/dev/microsoft-agents-testing/tests/test_framework/core/client/test_agent_client.py +++ b/dev/microsoft-agents-testing/tests/integration/core/client/test_agent_client.py @@ -7,8 +7,7 @@ from msal import ConfidentialClientApplication from microsoft_agents.activity import Activity - -from src.core import AgentClient +from microsoft_agents.testing import AgentClient from ._common import DEFAULTS diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/client/test_response_client.py b/dev/microsoft-agents-testing/tests/integration/core/client/test_response_client.py similarity index 96% rename from dev/microsoft-agents-testing/tests/test_framework/core/client/test_response_client.py rename to dev/microsoft-agents-testing/tests/integration/core/client/test_response_client.py index 986c4172..f5d1ed6d 100644 --- a/dev/microsoft-agents-testing/tests/test_framework/core/client/test_response_client.py +++ b/dev/microsoft-agents-testing/tests/integration/core/client/test_response_client.py @@ -3,8 +3,8 @@ from aiohttp import ClientSession from microsoft_agents.activity import Activity +from microsoft_agents.testing import ResponseClient -from src.core import ResponseClient from ._common import DEFAULTS diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/test_application_runner.py b/dev/microsoft-agents-testing/tests/integration/core/test_application_runner.py similarity index 99% rename from dev/microsoft-agents-testing/tests/test_framework/core/test_application_runner.py rename to dev/microsoft-agents-testing/tests/integration/core/test_application_runner.py index 719203b7..65090594 100644 --- a/dev/microsoft-agents-testing/tests/test_framework/core/test_application_runner.py +++ b/dev/microsoft-agents-testing/tests/integration/core/test_application_runner.py @@ -3,7 +3,6 @@ from ._common import SimpleRunner, OtherSimpleRunner - class TestApplicationRunner: @pytest.mark.asyncio diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/test_integration_from_sample.py b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_sample.py similarity index 85% rename from dev/microsoft-agents-testing/tests/test_framework/core/test_integration_from_sample.py rename to dev/microsoft-agents-testing/tests/integration/core/test_integration_from_sample.py index 89800fda..b7019573 100644 --- a/dev/microsoft-agents-testing/tests/test_framework/core/test_integration_from_sample.py +++ b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_sample.py @@ -2,12 +2,11 @@ import asyncio from copy import copy -from src.core import ( +from microsoft_agents.testing import ( ApplicationRunner, Environment, - integration, - IntegrationFixtures, - Sample, + Integration, + Sample ) from ._common import SimpleRunner @@ -40,9 +39,13 @@ async def init_app(self): await asyncio.sleep(0.1) # Simulate some initialization delay self.other_data = len(self.env.config) + @property + def app(self) -> None: + return None -@integration(sample=SimpleSample, environment=SimpleEnvironment) -class TestIntegrationFromSample(IntegrationFixtures): +class TestIntegrationFromSample(Integration): + _sample_cls = SimpleSample + _environment_cls = SimpleEnvironment @pytest.mark.asyncio async def test_sample_integration(self, sample, environment): diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/test_integration_from_service_url.py b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py similarity index 87% rename from dev/microsoft-agents-testing/tests/test_framework/core/test_integration_from_service_url.py rename to dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py index 8254070b..4ff707cc 100644 --- a/dev/microsoft-agents-testing/tests/test_framework/core/test_integration_from_service_url.py +++ b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py @@ -4,14 +4,11 @@ from copy import copy from aioresponses import aioresponses, CallbackResult -from src.core import integration, IntegrationFixtures +from microsoft_agents.testing import Integration - -@integration( - agent_url="http://localhost:8000/", - service_url="http://localhost:8001/", -) -class TestIntegrationFromURL(IntegrationFixtures): +class TestIntegrationFromURL(Integration): + _agent_url = "http://localhost:8000/" + _service_url = "http://localhost:8001/" @pytest.mark.asyncio async def test_service_url_integration(self, agent_client): diff --git a/dev/microsoft-agents-testing/tests/integration/foundational/__init__.py b/dev/microsoft-agents-testing/tests/integration/foundational/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/microsoft-agents-testing/tests/integration/foundational/_common.py b/dev/microsoft-agents-testing/tests/integration/foundational/_common.py deleted file mode 100644 index 91672ffd..00000000 --- a/dev/microsoft-agents-testing/tests/integration/foundational/_common.py +++ /dev/null @@ -1,10 +0,0 @@ -import json - -from microsoft_agents.activity import Activity - -def load_activity(channel: str, name: str) -> Activity: - - with open("./dev/integration/src/tests/integration/foundational/activities/{}/{}.json".format(channel, name), "r") as f: - activity = json.load(f) - - return Activity.model_validate(activity) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/integration/foundational/test_suite.py b/dev/microsoft-agents-testing/tests/integration/foundational/test_suite.py deleted file mode 100644 index a38edf0b..00000000 --- a/dev/microsoft-agents-testing/tests/integration/foundational/test_suite.py +++ /dev/null @@ -1,141 +0,0 @@ -import json -import pytest -import asyncio - -from microsoft_agents.activity import ( - ActivityTypes, -) - -from src.core import integration, IntegrationFixtures, AiohttpEnvironment -from src.samples import QuickstartSample - -from ._common import load_activity - -DIRECTLINE = "directline" - -@integration() -class TestFoundation(IntegrationFixtures): - - def load_activity(self, activity_name) -> Activity: - return load_activity(DIRECTLINE, activity_name) - - @pytest.mark.asyncio - async def test__send_activity__sends_hello_world__returns_hello_world(self, agent_client): - activity = load_activity(DIRECTLINE, "hello_world.json") - result = await agent_client.send_activity(activity) - assert result is not None - last = result[-1] - assert last.type == ActivityTypes.message - assert last.text.lower() == "you said: {activity.text}".lower() - - @pytest.mark.asyncio - async def test__send_invoke__send_basic_invoke_activity__receive_invoke_response(self, agent_client): - activity = load_activity(DIRECTLINE, "basic_invoke.json") - result = await agent_client.send_activity(activity) - assert result - data = json.loads(result) - message = data.get("message", {}) - assert "Invoke received." in message - assert "data" in data - assert data["parameters"] and len(data["parameters"]) > 0 - assert "hi" in data["value"] - - @pytest.mark.asyncio - async def test__send_activity__sends_message_activity_to_ac_submit__return_valid_response(self, agent_client): - activity = load_activity(DIRECTLINE, "ac_submit.json") - result = await agent_client.send_activity(activity) - assert result is not None - last = result[-1] - assert last.type == ActivityTypes.message - assert "doStuff" in last.text - assert "Action.Submit" in last.text - assert "hello" in last.text - - @pytest.mark.asyncio - async def test__send_invoke_sends_invoke_activity_to_ac_execute__returns_valid_adaptive_card_invoke_response(self, agent_client): - activity = load_activity(DIRECTLINE, "ac_execute.json") - result = await agent_client.send_invoke(activity) - - result = json.loads(result) - - assert result.status == 200 - assert result.value - - assert "application/vnd.microsoft.card.adaptive" in result.type - - activity_data = json.loads(activity.value) - assert activity_data.get("action") - user_text = activity_data.get("usertext") - assert user_text in result.value - - @pytest.mark.asyncio - async def test__send_activity_sends_text__returns_poem(self, agent_client): - pass - - @pytest.mark.asyncio - async def test__send_expected_replies_activity__sends_text__returns_poem(self, agent_client): - activity = self.load_activity("expected_replies.json") - result = await agent_client.send_expected_replies(activity) - last = result[-1] - assert last.type == ActivityTypes.message - assert "Apollo" in last.text - assert "\n" in last.text - - @pytest.mark.asyncio - async def test__send_invoke__query_link__returns_text(self, agent_client): - activity = self.load_activity("query_link.json") - result = await agent_client.send_invoke(activity) - pass # TODO - - @pytest.mark.asyncio - async def test__send_invoke__select_item__receive_item(self, agent_client): - activity = self.load_activity("select_item.json") - result = await agent_client.send_invoke(activity) - pass # TODO - - @pytest.mark.asyncio - async def test__send_activity__conversation_update__returns_welcome_message(self, agent_client): - activity = self.load_activity("conversation_update.json") - result = await agent_client.send_activity(activity) - last = result[-1] - assert "Hello and Welcome!" in last.text - - @pytest.mark.asyncio - async def test__send_activity__send_heart_message_reaction__returns_message_reaction_heart(self, agent_client): - activity = self.load_activity("message_reaction_heart.json") - result = await agent_client.send_activity(activity) - last = result[-1] - assert last.type == ActivityTypes.message - assert "Message Reaction Added: heart" in last.text - - @pytest.mark.asyncio - async def test__send_activity__remove_heart_message_reaction__returns_message_reaction_heart(self, agent_client): - activity = self.load_activity - result = await agent_client.send_activity(activity) - last = result[-1] - assert last.type == ActivityTypes.message - assert "Message Reaction Removed: heart" in last.text - - @pytest.mark.asyncio - async def test__send_expected_replies_activity__send_seattle_today_weather__returns_weather(self, agent_client): - activity = self.load_activity("expected_replies_seattle_weather.json") - result = await agent_client.send_expected_replies(activity) - last = result[-1] - assert last.type == ActivityTypes.message - assert last.attachments and len(last.attachments) > 0 - - adaptive_card = last.attachments.first() - assert adaptive_card - assert "application/vnd.microsoft.card.adaptive" == adaptive_card.content_type - assert adaptive_card.content - - assert \ - "�" in adaptive_card.content or \ - "\\u00B0" in adaptive_card.content or \ - f"Missing temperature inside adaptive card: {adaptive_card.content}" in adaptive_card.content - - @pytest.mark.asyncio - async def test__send_activity__simulate_message_loop__expect_question_about_time_and_returns_weather(self, agent_client): - activities = self.load_activity("message_loop_1.json") - fresult = await agent_client.send_activity(activities[0]) - assert \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/integration/test_quickstart.py b/dev/microsoft-agents-testing/tests/integration/test_quickstart.py deleted file mode 100644 index db786ab5..00000000 --- a/dev/microsoft-agents-testing/tests/integration/test_quickstart.py +++ /dev/null @@ -1,20 +0,0 @@ -import pytest -import asyncio - -from src.core import integration, IntegrationFixtures, AiohttpEnvironment -from src.samples import QuickstartSample - -@integration(sample=QuickstartSample, environment=AiohttpEnvironment) -class TestQuickstart(IntegrationFixtures): - - @pytest.mark.asyncio - async def test_welcome_message(self, agent_client, response_client): - res = await agent_client.send_expect_replies("hi") - await asyncio.sleep(1) # Wait for processing - responses = await response_client.pop() - - assert len(responses) == 0 - - first_non_typing = next((r for r in res if r.type != "typing"), None) - assert first_non_typing is not None - assert first_non_typing.text == "you said: hi" \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/integration/components/__init__.py b/dev/microsoft-agents-testing/tests/manual_test/__init__.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/components/__init__.py rename to dev/microsoft-agents-testing/tests/manual_test/__init__.py diff --git a/dev/microsoft-agents-testing/tests/env.TEMPLATE b/dev/microsoft-agents-testing/tests/manual_test/env.TEMPLATE similarity index 100% rename from dev/microsoft-agents-testing/tests/env.TEMPLATE rename to dev/microsoft-agents-testing/tests/manual_test/env.TEMPLATE diff --git a/dev/microsoft-agents-testing/tests/manual_test.py b/dev/microsoft-agents-testing/tests/manual_test/main.py similarity index 77% rename from dev/microsoft-agents-testing/tests/manual_test.py rename to dev/microsoft-agents-testing/tests/manual_test/main.py index a24f9c34..3246c51d 100644 --- a/dev/microsoft-agents-testing/tests/manual_test.py +++ b/dev/microsoft-agents-testing/tests/manual_test/main.py @@ -1,9 +1,11 @@ import os -import pytest import asyncio -from src.core import integration, IntegrationFixtures, AiohttpEnvironment, AiohttpEnvironment, AgentClient -from src.samples import QuickstartSample +from microsoft_agents.testing import ( + AiohttpEnvironment, + AgentClient, +) +from ..samples import QuickstartSample from dotenv import load_dotenv @@ -33,9 +35,12 @@ async def main(): async with env.create_runner(host, port): print(f"Server running at http://{host}:{port}/api/messages") - while True: - await asyncio.sleep(1) - res = await client.send_expect_replies("Hello, Agent!") + await asyncio.sleep(1) + res = await client.send_expect_replies("Hello, Agent!") + print("\nReply from agent:") + print(res) + + await client.close() if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/samples/__init__.py b/dev/microsoft-agents-testing/tests/samples/__init__.py new file mode 100644 index 00000000..a77ee72e --- /dev/null +++ b/dev/microsoft-agents-testing/tests/samples/__init__.py @@ -0,0 +1,3 @@ +from .quickstart_sample import QuickstartSample + +__all__ = ["QuickstartSample"] diff --git a/dev/microsoft-agents-testing/tests/samples/quickstart_sample.py b/dev/microsoft-agents-testing/tests/samples/quickstart_sample.py new file mode 100644 index 00000000..04da32c1 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/samples/quickstart_sample.py @@ -0,0 +1,57 @@ +import re +import os +import sys +import traceback + +from dotenv import load_dotenv + +from microsoft_agents.activity import ConversationUpdateTypes +from microsoft_agents.hosting.core import AgentApplication, TurnContext, TurnState +from microsoft_agents.testing.integration.core import Sample + +class QuickstartSample(Sample): + """A quickstart sample implementation.""" + + @classmethod + async def get_config(cls) -> dict: + """Retrieve the configuration for the sample.""" + + load_dotenv("./src/tests/.env") + + + return { + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID"), + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET"), + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID"), + } + + async def init_app(self): + """Initialize the application for the quickstart sample.""" + + app: AgentApplication[TurnState] = self.env.agent_application + + @app.conversation_update(ConversationUpdateTypes.MEMBERS_ADDED) + async def on_members_added(context: TurnContext, state: TurnState) -> None: + await context.send_activity( + "Welcome to the empty agent! " + "This agent is designed to be a starting point for your own agent development." + ) + + @app.message(re.compile(r"^hello$")) + async def on_hello(context: TurnContext, state: TurnState) -> None: + await context.send_activity("Hello!") + + @app.activity("message") + async def on_message(context: TurnContext, state: TurnState) -> None: + await context.send_activity(f"you said: {context.activity.text}") + + @app.error + async def on_error(context: TurnContext, error: Exception): + # This check writes out errors to console log .vs. app insights. + # NOTE: In production environment, you should consider logging this to Azure + # application insights. + print(f"\n [on_turn_error] unhandled error: {error}", file=sys.stderr) + traceback.print_exc() + + # Send a message to the user + await context.send_activity("The bot encountered an error or bug.") diff --git a/dev/microsoft-agents-testing/tests/test_framework/__init__.py b/dev/microsoft-agents-testing/tests/test_framework/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/__init__.py b/dev/microsoft-agents-testing/tests/test_framework/core/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/microsoft-agents-testing/tests/test_framework/core/client/__init__.py b/dev/microsoft-agents-testing/tests/test_framework/core/client/__init__.py deleted file mode 100644 index e69de29b..00000000 From 69d68d49324b965c0610237928f96886b9502987 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 09:50:51 -0800 Subject: [PATCH 42/81] Adding verbose logging for results with benchmark tool --- dev/benchmark/__init__.py | 0 dev/benchmark/payload.json | 3 ++- dev/benchmark/src/main.py | 13 +++++++++---- dev/benchmark/src/output.py | 9 +++++++++ 4 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 dev/benchmark/__init__.py create mode 100644 dev/benchmark/src/output.py diff --git a/dev/benchmark/__init__.py b/dev/benchmark/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/benchmark/payload.json b/dev/benchmark/payload.json index 28ac2a8c..399023d6 100644 --- a/dev/benchmark/payload.json +++ b/dev/benchmark/payload.json @@ -15,5 +15,6 @@ "id": "user-id-0", "aadObjectId": "00000000-0000-0000-0000-0000000000020" }, - "type": "message" + "type": "message", + "text": "Hello, Bot!" } \ No newline at end of file diff --git a/dev/benchmark/src/main.py b/dev/benchmark/src/main.py index af9e177e..c27b49ca 100644 --- a/dev/benchmark/src/main.py +++ b/dev/benchmark/src/main.py @@ -1,6 +1,8 @@ -import json +import json, sys +from io import StringIO import logging from datetime import datetime, timezone +from contextlib import contextmanager import click @@ -8,25 +10,26 @@ from .executor import Executor, CoroutineExecutor, ThreadExecutor from .aggregated_results import AggregatedResults from .config import BenchmarkConfig +from .output import output_results LOG_FORMAT = "%(asctime)s: %(message)s" logging.basicConfig(format=LOG_FORMAT, level=logging.INFO, datefmt="%H:%M:%S") BenchmarkConfig.load_from_env() - @click.command() @click.option( "--payload_path", "-p", default="./payload.json", help="Path to the payload file." ) @click.option("--num_workers", "-n", default=1, help="Number of workers to use.") +@click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging.") @click.option( "--async_mode", "-a", is_flag=True, help="Run coroutine workers rather than thread workers.", ) -def main(payload_path: str, num_workers: int, async_mode: bool): +def main(payload_path: str, num_workers: int, verbose: bool, async_mode: bool): """Main function to run the benchmark.""" with open(payload_path, "r", encoding="utf-8") as f: @@ -35,10 +38,12 @@ def main(payload_path: str, num_workers: int, async_mode: bool): func = create_payload_sender(payload) executor: Executor = CoroutineExecutor() if async_mode else ThreadExecutor() - + start_time = datetime.now(timezone.utc).timestamp() results = executor.run(func, num_workers=num_workers) end_time = datetime.now(timezone.utc).timestamp() + if verbose: + output_results(results) agg = AggregatedResults(results) agg.display(start_time, end_time) diff --git a/dev/benchmark/src/output.py b/dev/benchmark/src/output.py new file mode 100644 index 00000000..b7efa12d --- /dev/null +++ b/dev/benchmark/src/output.py @@ -0,0 +1,9 @@ +from .executor import ExecutionResult + +def output_results(results: list[ExecutionResult]) -> None: + """Output the results of the benchmark to the console.""" + + for result in results: + status = "Success" if result.success else "Failure" + print(f"Execution ID: {result.exe_id}, Duration: {result.duration:.4f} seconds, Status: {status}") + print(result.result) \ No newline at end of file From 3dc3efa70858b993e334fc15760ba38891a2d5dc Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 09:53:34 -0800 Subject: [PATCH 43/81] General cleanup --- dev/README.md | 6 - dev/install.sh | 1 + dev/integration/README.md | 2 - dev/integration/requirements.txt | 1 - dev/integration/src/__init__.py | 0 dev/integration/src/core/__init__.py | 21 --- dev/integration/src/core/aiohttp/__init__.py | 4 - .../src/core/aiohttp/aiohttp_environment.py | 58 ------- .../src/core/aiohttp/aiohttp_runner.py | 115 ------------- .../src/core/application_runner.py | 42 ----- dev/integration/src/core/client/__init__.py | 7 - .../src/core/client/agent_client.py | 136 --------------- .../src/core/client/auto_client.py | 18 -- .../src/core/client/response_client.py | 92 ---------- dev/integration/src/core/environment.py | 37 ---- dev/integration/src/core/integration.py | 158 ------------------ dev/integration/src/core/sample.py | 19 --- dev/integration/src/core/utils.py | 44 ----- dev/integration/src/samples/__init__.py | 3 - .../src/samples/quickstart_sample.py | 60 ------- 20 files changed, 1 insertion(+), 823 deletions(-) create mode 100644 dev/install.sh delete mode 100644 dev/integration/README.md delete mode 100644 dev/integration/requirements.txt delete mode 100644 dev/integration/src/__init__.py delete mode 100644 dev/integration/src/core/__init__.py delete mode 100644 dev/integration/src/core/aiohttp/__init__.py delete mode 100644 dev/integration/src/core/aiohttp/aiohttp_environment.py delete mode 100644 dev/integration/src/core/aiohttp/aiohttp_runner.py delete mode 100644 dev/integration/src/core/application_runner.py delete mode 100644 dev/integration/src/core/client/__init__.py delete mode 100644 dev/integration/src/core/client/agent_client.py delete mode 100644 dev/integration/src/core/client/auto_client.py delete mode 100644 dev/integration/src/core/client/response_client.py delete mode 100644 dev/integration/src/core/environment.py delete mode 100644 dev/integration/src/core/integration.py delete mode 100644 dev/integration/src/core/sample.py delete mode 100644 dev/integration/src/core/utils.py delete mode 100644 dev/integration/src/samples/__init__.py delete mode 100644 dev/integration/src/samples/quickstart_sample.py diff --git a/dev/README.md b/dev/README.md index e8c10764..e69de29b 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,6 +0,0 @@ -This directory contains tools to aid the developers of the Microsoft 365 Agents SDK for Python. - -### `benchmark` - -This folder contains benchmarking utilities built in Python to send concurrent requests -to an agent. \ No newline at end of file diff --git a/dev/install.sh b/dev/install.sh new file mode 100644 index 00000000..512d1d88 --- /dev/null +++ b/dev/install.sh @@ -0,0 +1 @@ +pip install -e ./microsoft-agents-testing/ --config-settings editable_mode=compat \ No newline at end of file diff --git a/dev/integration/README.md b/dev/integration/README.md deleted file mode 100644 index 614462c9..00000000 --- a/dev/integration/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Microsoft 365 Agents SDK for Python Integration Testing Framework - diff --git a/dev/integration/requirements.txt b/dev/integration/requirements.txt deleted file mode 100644 index d524e63a..00000000 --- a/dev/integration/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -aioresponses \ No newline at end of file diff --git a/dev/integration/src/__init__.py b/dev/integration/src/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/integration/src/core/__init__.py b/dev/integration/src/core/__init__.py deleted file mode 100644 index a8f5206c..00000000 --- a/dev/integration/src/core/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -from .application_runner import ApplicationRunner -from .aiohttp import AiohttpEnvironment -from .client import ( - AgentClient, - ResponseClient, -) -from .environment import Environment -from .integration import integration, IntegrationFixtures -from .sample import Sample - - -__all__ = [ - "AgentClient", - "ApplicationRunner", - "AiohttpEnvironment", - "ResponseClient", - "Environment", - "integration", - "IntegrationFixtures", - "Sample", -] diff --git a/dev/integration/src/core/aiohttp/__init__.py b/dev/integration/src/core/aiohttp/__init__.py deleted file mode 100644 index 82d2d1d0..00000000 --- a/dev/integration/src/core/aiohttp/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .aiohttp_environment import AiohttpEnvironment -from .aiohttp_runner import AiohttpRunner - -__all__ = ["AiohttpEnvironment", "AiohttpRunner"] diff --git a/dev/integration/src/core/aiohttp/aiohttp_environment.py b/dev/integration/src/core/aiohttp/aiohttp_environment.py deleted file mode 100644 index c8256618..00000000 --- a/dev/integration/src/core/aiohttp/aiohttp_environment.py +++ /dev/null @@ -1,58 +0,0 @@ -from tkinter import E -from aiohttp.web import Request, Response, Application, run_app - -from microsoft_agents.hosting.aiohttp import ( - CloudAdapter, - jwt_authorization_middleware, - start_agent_process, -) -from microsoft_agents.hosting.core import ( - Authorization, - AgentApplication, - TurnState, - MemoryStorage, -) -from microsoft_agents.authentication.msal import MsalConnectionManager -from microsoft_agents.activity import load_configuration_from_env - -from ..application_runner import ApplicationRunner -from ..environment import Environment -from .aiohttp_runner import AiohttpRunner - - -class AiohttpEnvironment(Environment): - """An environment for aiohttp-hosted agents.""" - - async def init_env(self, environ_config: dict) -> None: - environ_config = environ_config or {} - - self.config = load_configuration_from_env(environ_config) - - self.storage = MemoryStorage() - self.connection_manager = MsalConnectionManager(**self.config) - self.adapter = CloudAdapter(connection_manager=self.connection_manager) - self.authorization = Authorization( - self.storage, self.connection_manager, **self.config - ) - - self.agent_application = AgentApplication[TurnState]( - storage=self.storage, - adapter=self.adapter, - authorization=self.authorization, - **self.config - ) - - def create_runner(self, host: str, port: int) -> ApplicationRunner: - - async def entry_point(req: Request) -> Response: - agent: AgentApplication = req.app["agent_app"] - adapter: CloudAdapter = req.app["adapter"] - return await start_agent_process(req, agent, adapter) - - APP = Application(middlewares=[jwt_authorization_middleware]) - APP.router.add_post("/api/messages", entry_point) - APP["agent_configuration"] = self.connection_manager.get_default_connection_configuration() - APP["agent_app"] = self.agent_application - APP["adapter"] = self.adapter - - return AiohttpRunner(APP, host, port) diff --git a/dev/integration/src/core/aiohttp/aiohttp_runner.py b/dev/integration/src/core/aiohttp/aiohttp_runner.py deleted file mode 100644 index 3b6780d4..00000000 --- a/dev/integration/src/core/aiohttp/aiohttp_runner.py +++ /dev/null @@ -1,115 +0,0 @@ -from typing import Optional -from typing import Optional -from threading import Thread, Event -import asyncio - -from aiohttp import ClientSession -from aiohttp.web import Application, Request, Response -from aiohttp.web_runner import AppRunner, TCPSite - -from ..application_runner import ApplicationRunner - - -class AiohttpRunner(ApplicationRunner): - """A runner for aiohttp applications.""" - - def __init__(self, app: Application, host: str = "localhost", port: int = 8000): - assert isinstance(app, Application) - super().__init__(app) - - url = f"{host}:{port}" - self._host = host - self._port = port - if "http" not in url: - url = f"http://{url}" - self._url = url - - self._app.router.add_get("/shutdown", self._shutdown_route) - - self._server_thread: Optional[Thread] = None - self._shutdown_event = Event() - self._runner: Optional[AppRunner] = None - self._site: Optional[TCPSite] = None - - @property - def url(self) -> str: - return self._url - - async def _start_server(self) -> None: - try: - assert isinstance(self._app, Application) - - self._runner = AppRunner(self._app) - await self._runner.setup() - self._site = TCPSite(self._runner, self._host, self._port) - await self._site.start() - - # Wait for shutdown signal - while not self._shutdown_event.is_set(): - await asyncio.sleep(0.1) - - # Cleanup - await self._site.stop() - await self._runner.cleanup() - - except Exception as error: - raise error - - async def __aenter__(self): - if self._server_thread: - raise RuntimeError("ResponseClient is already running.") - - self._shutdown_event.clear() - self._server_thread = Thread( - target=lambda: asyncio.run(self._start_server()), daemon=True - ) - self._server_thread.start() - - # Wait a moment to ensure the server starts - await asyncio.sleep(0.5) - - return self - - async def _stop_server(self): - if not self._server_thread: - raise RuntimeError("ResponseClient is not running.") - - try: - async with ClientSession() as session: - async with session.get( - f"http://{self._host}:{self._port}/shutdown" - ) as response: - pass # Just trigger the shutdown - except Exception: - pass # Ignore errors during shutdown request - - # Set shutdown event as fallback - self._shutdown_event.set() - - # Wait for the server thread to finish - self._server_thread.join(timeout=5.0) - self._server_thread = None - - async def _shutdown_route(self, request: Request) -> Response: - """Handle shutdown request by setting the shutdown event""" - self._shutdown_event.set() - return Response(status=200, text="Shutdown initiated") - - async def __aexit__(self, exc_type, exc, tb): - if not self._server_thread: - raise RuntimeError("ResponseClient is not running.") - try: - async with ClientSession() as session: - async with session.get( - f"http://{self._host}:{self._port}/shutdown" - ) as response: - pass # Just trigger the shutdown - except Exception: - pass # Ignore errors during shutdown request - - # Set shutdown event as fallback - self._shutdown_event.set() - - # Wait for the server thread to finish - self._server_thread.join(timeout=5.0) - self._server_thread = None diff --git a/dev/integration/src/core/application_runner.py b/dev/integration/src/core/application_runner.py deleted file mode 100644 index ebbc56f9..00000000 --- a/dev/integration/src/core/application_runner.py +++ /dev/null @@ -1,42 +0,0 @@ -import asyncio -from abc import ABC, abstractmethod -from typing import Any, Optional -from threading import Thread - - -class ApplicationRunner(ABC): - """Base class for application runners.""" - - def __init__(self, app: Any): - self._app = app - self._thread: Optional[Thread] = None - - @abstractmethod - async def _start_server(self) -> None: - raise NotImplementedError( - "Start server method must be implemented by subclasses" - ) - - async def _stop_server(self) -> None: - pass - - async def __aenter__(self) -> None: - - if self._thread: - raise RuntimeError("Server is already running") - - def target(): - asyncio.run(self._start_server()) - - self._thread = Thread(target=target, daemon=True) - self._thread.start() - - async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: - - if self._thread: - await self._stop_server() - - self._thread.join() - self._thread = None - else: - raise RuntimeError("Server is not running") diff --git a/dev/integration/src/core/client/__init__.py b/dev/integration/src/core/client/__init__.py deleted file mode 100644 index 1d59411e..00000000 --- a/dev/integration/src/core/client/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .agent_client import AgentClient -from .response_client import ResponseClient - -__all__ = [ - "AgentClient", - "ResponseClient", -] diff --git a/dev/integration/src/core/client/agent_client.py b/dev/integration/src/core/client/agent_client.py deleted file mode 100644 index bbf650b6..00000000 --- a/dev/integration/src/core/client/agent_client.py +++ /dev/null @@ -1,136 +0,0 @@ -import os -import json -import asyncio -from typing import Optional, cast - -from aiohttp import ClientSession -from msal import ConfidentialClientApplication - -from microsoft_agents.activity import Activity, ActivityTypes, DeliveryModes, ConversationAccount - -from ..utils import _populate_incoming_activity - - -class AgentClient: - - def __init__( - self, - agent_url: str, - cid: str, - client_id: str, - tenant_id: str, - client_secret: str, - service_url: Optional[str] = None, - default_timeout: float = 5.0, - ): - self._agent_url = agent_url - self._cid = cid - self._client_id = client_id - self._tenant_id = tenant_id - self._client_secret = client_secret - self._service_url = service_url - self._headers = None - self._default_timeout = default_timeout - - self._client: Optional[ClientSession] = None - - @property - def agent_url(self) -> str: - return self._agent_url - - @property - def service_url(self) -> Optional[str]: - return self._service_url - - async def get_access_token(self) -> str: - - msal_app = ConfidentialClientApplication( - client_id=self._client_id, - client_credential=self._client_secret, - authority=f"https://login.microsoftonline.com/{self._tenant_id}", - ) - - res = msal_app.acquire_token_for_client(scopes=[f"{self._client_id}/.default"]) - token = res.get("access_token") if res else None - if not token: - raise Exception("Could not obtain access token") - return token - - async def _init_client(self) -> None: - if not self._client: - if self._client_secret: - token = await self.get_access_token() - self._headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - } - else: - self._headers = {"Content-Type": "application/json"} - - self._client = ClientSession( - base_url=self._agent_url, headers=self._headers - ) - - async def send_request(self, activity: Activity, sleep: float = 0) -> str: - - await self._init_client() - assert self._client - - if activity.conversation: - activity.conversation.id = self._cid - else: - activity.conversation = ConversationAccount(id=self._cid or "") - - if self.service_url: - activity.service_url = self.service_url - - activity = _populate_incoming_activity(activity) - - async with self._client.post( - "api/messages", - headers=self._headers, - json=activity.model_dump(by_alias=True, exclude_unset=True, exclude_none=True, mode="json"), - ) as response: - content = await response.text() - if not response.ok: - raise Exception(f"Failed to send activity: {response.status}") - await asyncio.sleep(sleep) - return content - - def _to_activity(self, activity_or_text: Activity | str) -> Activity: - if isinstance(activity_or_text, str): - activity = Activity( - type=ActivityTypes.message, - text=activity_or_text, - ) - return activity - else: - return cast(Activity, activity_or_text) - - async def send_activity( - self, activity_or_text: Activity | str, sleep: float = 0, timeout: Optional[float] = None - ) -> str: - timeout = timeout or self._default_timeout - activity = self._to_activity(activity_or_text) - content = await self.send_request(activity, sleep=sleep) - return content - - async def send_expect_replies( - self, activity_or_text: Activity | str, sleep: float = 0, timeout: Optional[float] = None - ) -> list[Activity]: - timeout = timeout or self._default_timeout - activity = self._to_activity(activity_or_text) - activity.delivery_mode = DeliveryModes.expect_replies - activity.service_url = activity.service_url or "http://localhost" # temporary fix - - content = await self.send_request(activity, sleep=sleep) - - activities_data = json.loads(content).get("activities", []) - activities = [Activity.model_validate(act) for act in activities_data] - - return activities - - async def close(self) -> None: - if self._client: - await self._client.close() - self._client = None diff --git a/dev/integration/src/core/client/auto_client.py b/dev/integration/src/core/client/auto_client.py deleted file mode 100644 index dcea531b..00000000 --- a/dev/integration/src/core/client/auto_client.py +++ /dev/null @@ -1,18 +0,0 @@ -# from microsoft_agents.activity import Activity - -# from ..agent_client import AgentClient - -# class AutoClient: - -# def __init__(self, agent_client: AgentClient): -# self._agent_client = agent_client - -# async def generate_message(self) -> str: -# pass - -# async def run(self, max_turns: int = 10, time_between_turns: float = 2.0) -> None: - -# for i in range(max_turns): -# await self._agent_client.send_activity( -# Activity(type="message", text=self.generate_message()) -# ) diff --git a/dev/integration/src/core/client/response_client.py b/dev/integration/src/core/client/response_client.py deleted file mode 100644 index d93bfb80..00000000 --- a/dev/integration/src/core/client/response_client.py +++ /dev/null @@ -1,92 +0,0 @@ -from __future__ import annotations - -import sys -from io import StringIO -from typing import Optional -from threading import Lock, Thread, Event -import asyncio - -from aiohttp.web import Application, Request, Response - -from microsoft_agents.activity import ( - Activity, - ActivityTypes, -) - -from ..aiohttp import AiohttpRunner - - -class ResponseClient: - - def __init__( - self, - host: str = "localhost", - port: int = 9873, - ): - self._app: Application = Application() - self._prev_stdout = None - service_endpoint = f"{host}:{port}" - self._host = host - self._port = port - if "http" not in service_endpoint: - service_endpoint = f"http://{service_endpoint}" - self._service_endpoint = service_endpoint - self._activities_list = [] - self._activities_list_lock = Lock() - - self._app.router.add_post( - "/v3/conversations/{path:.*}", self._handle_conversation - ) - - self._app_runner = AiohttpRunner(self._app, host, port) - - @property - def service_endpoint(self) -> str: - return self._service_endpoint - - async def __aenter__(self) -> ResponseClient: - self._prev_stdout = sys.stdout - sys.stdout = StringIO() - - await self._app_runner.__aenter__() - - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: - if self._prev_stdout is not None: - sys.stdout = self._prev_stdout - - await self._app_runner.__aexit__(exc_type, exc_val, exc_tb) - - async def _handle_conversation(self, request: Request) -> Response: - try: - data = await request.json() - activity = Activity.model_validate(data) - - conversation_id = ( - activity.conversation.id if activity.conversation else None - ) - - with self._activities_list_lock: - self._activities_list.append(activity) - - if any(map(lambda x: x.type == "streaminfo", activity.entities or [])): - await self._handle_streamed_activity(activity) - return Response(status=200, text="Stream info handled") - else: - if activity.type != ActivityTypes.typing: - await asyncio.sleep(0.1) # Simulate processing delay - return Response(status=200, text="Activity received") - except Exception as e: - return Response(status=500, text=str(e)) - - async def _handle_streamed_activity( - self, activity: Activity, *args, **kwargs - ) -> bool: - raise NotImplementedError("_handle_streamed_activity is not implemented yet.") - - async def pop(self) -> list[Activity]: - with self._activities_list_lock: - activities = self._activities_list[:] - self._activities_list.clear() - return activities diff --git a/dev/integration/src/core/environment.py b/dev/integration/src/core/environment.py deleted file mode 100644 index 2c9b1ae1..00000000 --- a/dev/integration/src/core/environment.py +++ /dev/null @@ -1,37 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Awaitable, Callable - -from microsoft_agents.hosting.core import ( - AgentApplication, - ChannelAdapter, - Connections, - Authorization, - Storage, - TurnState, -) - -from .application_runner import ApplicationRunner - - -class Environment(ABC): - """A sample data object for integration tests.""" - - agent_application: AgentApplication[TurnState] - storage: Storage - adapter: ChannelAdapter - connection_manager: Connections - authorization: Authorization - - config: dict - - driver: Callable[[], Awaitable[None]] - - @abstractmethod - async def init_env(self, environ_config: dict) -> None: - """Initialize the environment.""" - raise NotImplementedError() - - @abstractmethod - def create_runner(self) -> ApplicationRunner: - """Create an application runner for the environment.""" - raise NotImplementedError() diff --git a/dev/integration/src/core/integration.py b/dev/integration/src/core/integration.py deleted file mode 100644 index 59747d4c..00000000 --- a/dev/integration/src/core/integration.py +++ /dev/null @@ -1,158 +0,0 @@ -import pytest -import asyncio - -import os -from typing import ( - Optional, - TypeVar, - Union, - Callable, - Any, - AsyncGenerator, -) - -import aiohttp.web -from dotenv import load_dotenv - -from .application_runner import ApplicationRunner -from .environment import Environment -from .client import AgentClient, ResponseClient -from .sample import Sample -from .utils import get_host_and_port - -T = TypeVar("T", bound=type) -AppT = TypeVar("AppT", bound=aiohttp.web.Application) # for future extension w/ Union - - -class IntegrationFixtures: - """Provides integration test fixtures.""" - - _sample_cls: Optional[type[Sample]] = None - _environment_cls: Optional[type[Environment]] = None - - _config: dict[str, Any] = {} - - _service_url: Optional[str] = None - _agent_url: Optional[str] = None - _cid: Optional[str] = None - _client_id: Optional[str] = None - _tenant_id: Optional[str] = None - _client_secret: Optional[str] = None - - _environment: Environment - _sample: Sample - _agent_client: AgentClient - _response_client: ResponseClient - - @property - def service_url(self) -> str: - return self._service_url or self._config.get("service_url", "") - - @property - def agent_url(self) -> str: - return self._agent_url or self._config.get("agent_url", "") - - @pytest.fixture - async def environment(self): - """Provides the test environment instance.""" - if self._environment_cls: - assert self._sample_cls - environment = self._environment_cls() - await environment.init_env(await self._sample_cls.get_config()) - yield environment - else: - yield None - - @pytest.fixture - async def sample(self, environment): - """Provides the sample instance.""" - if environment: - assert self._sample_cls - sample = self._sample_cls(environment) - await sample.init_app() - host, port = get_host_and_port(self.agent_url) - async with environment.create_runner(host, port): - await asyncio.sleep(1) # Give time for the app to start - yield sample - else: - yield None - - def create_agent_client(self) -> AgentClient: - if not self._config: - self._config = {} - - load_dotenv("./src/tests/.env") - self._config.update( - { - "client_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", ""), - "tenant_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID", ""), - "client_secret": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET", ""), - } - ) - - agent_client = AgentClient( - agent_url=self.agent_url, - cid=self._cid or self._config.get("cid", ""), - client_id=self._client_id or self._config.get("client_id", ""), - tenant_id=self._tenant_id or self._config.get("tenant_id", ""), - client_secret=self._client_secret or self._config.get("client_secret", ""), - ) - return agent_client - - @pytest.fixture - async def agent_client(self, sample, environment) -> AsyncGenerator[AgentClient, None]: - agent_client = self.create_agent_client() - yield agent_client - await agent_client.close() - - async def _create_response_client(self) -> ResponseClient: - host, port = get_host_and_port(self.service_url) - assert host and port - return ResponseClient(host=host, port=port) - - @pytest.fixture - async def response_client(self) -> AsyncGenerator[ResponseClient, None]: - """Provides the response client instance.""" - async with await self._create_response_client() as response_client: - yield response_client - - -def integration( - agent_url: Optional[str] = "http://localhost:3978/", - sample: Optional[type[Sample]] = None, - environment: Optional[type[Environment]] = None, - app: Optional[AppT] = None, - **kwargs -) -> Callable[[T], T]: - """Factory function to create an Integration instance based on provided parameters. - - Essentially resolves to one of the static methods of Integration: - `from_service_url`, `from_sample`, or `from_app`, - based on the provided parameters. - - If a service URL is provided, it creates the Integration using that. - If both sample and environment are provided, it creates the Integration using them. - If an aiohttp application is provided, it creates the Integration using that. - - :param cls: The Integration class type. - :param service_url: Optional service URL to connect to. - :param sample: Optional Sample instance. - :param environment: Optional Environment instance. - :param host_agent: Flag to indicate if the agent should be hosted. - :param app: Optional aiohttp application instance. - :return: An instance of the Integration class. - """ - - def decorator(target_cls: T) -> T: - - if agent_url: - target_cls._agent_url = agent_url - if sample and environment: - target_cls._sample_cls = sample - target_cls._environment_cls = environment - - target_cls._config = kwargs - - return target_cls - - return decorator diff --git a/dev/integration/src/core/sample.py b/dev/integration/src/core/sample.py deleted file mode 100644 index 6dde3668..00000000 --- a/dev/integration/src/core/sample.py +++ /dev/null @@ -1,19 +0,0 @@ -from abc import ABC, abstractmethod - -from .environment import Environment - - -class Sample(ABC): - """Base class for all samples.""" - - def __init__(self, environment: Environment, **kwargs): - self.env = environment - - @classmethod - async def get_config(cls) -> dict: - """Retrieve the configuration for the sample.""" - return {} - - @abstractmethod - async def init_app(self): - """Initialize the application for the sample.""" diff --git a/dev/integration/src/core/utils.py b/dev/integration/src/core/utils.py deleted file mode 100644 index 2a183169..00000000 --- a/dev/integration/src/core/utils.py +++ /dev/null @@ -1,44 +0,0 @@ -from urllib.parse import urlparse - -from microsoft_agents.activity import ( - Activity, - ActivityTypes, - DeliveryModes, - ConversationAccount, - ChannelAccount, -) - -def get_host_and_port(url: str) -> tuple[str, int]: - """Extract host and port from a URL.""" - - parsed_url = urlparse(url) - host = parsed_url.hostname or "localhost" - port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) - return host, port - -def _populate_incoming_activity(activity: Activity) -> Activity: - - activity = activity.model_copy() - - if not activity.locale: - activity.locale = "en-US" - - if not activity.channel_id: - activity.channel_id = "emulator" - - if not activity.delivery_mode: - activity.delivery_mode = DeliveryModes.normal - - if not activity.service_url: - activity.service_url = "http://localhost" - - if not activity.recipient: - activity.recipient = ChannelAccount(id="agent", name="Agent") - - if not activity.from_property: - activity.from_property = ChannelAccount(id="user", name="User") - - if not activity.conversation: - activity.conversation = ConversationAccount(id="conversation1") - - return activity \ No newline at end of file diff --git a/dev/integration/src/samples/__init__.py b/dev/integration/src/samples/__init__.py deleted file mode 100644 index a77ee72e..00000000 --- a/dev/integration/src/samples/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .quickstart_sample import QuickstartSample - -__all__ = ["QuickstartSample"] diff --git a/dev/integration/src/samples/quickstart_sample.py b/dev/integration/src/samples/quickstart_sample.py deleted file mode 100644 index a18d7553..00000000 --- a/dev/integration/src/samples/quickstart_sample.py +++ /dev/null @@ -1,60 +0,0 @@ -import re -import os -import sys -import traceback - -from dotenv import load_dotenv - -from microsoft_agents.activity import ConversationUpdateTypes - -from microsoft_agents.hosting.core import AgentApplication, TurnContext, TurnState - -from ..core.sample import Sample - - -class QuickstartSample(Sample): - """A quickstart sample implementation.""" - - @classmethod - async def get_config(cls) -> dict: - """Retrieve the configuration for the sample.""" - - load_dotenv("./src/tests/.env") - - - return { - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID"), - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET"), - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID"), - } - - async def init_app(self): - """Initialize the application for the quickstart sample.""" - - app: AgentApplication[TurnState] = self.env.agent_application - - @app.conversation_update(ConversationUpdateTypes.MEMBERS_ADDED) - async def on_members_added(context: TurnContext, state: TurnState) -> None: - await context.send_activity( - "Welcome to the empty agent! " - "This agent is designed to be a starting point for your own agent development." - ) - - @app.message(re.compile(r"^hello$")) - async def on_hello(context: TurnContext, state: TurnState) -> None: - await context.send_activity("Hello!") - - @app.activity("message") - async def on_message(context: TurnContext, state: TurnState) -> None: - await context.send_activity(f"you said: {context.activity.text}") - - @app.error - async def on_error(context: TurnContext, error: Exception): - # This check writes out errors to console log .vs. app insights. - # NOTE: In production environment, you should consider logging this to Azure - # application insights. - print(f"\n [on_turn_error] unhandled error: {error}", file=sys.stderr) - traceback.print_exc() - - # Send a message to the user - await context.send_activity("The bot encountered an error or bug.") From 054a3a28af347f4e27da411731ffa71943f8f405 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 09:56:56 -0800 Subject: [PATCH 44/81] Adding README --- dev/README.md | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/dev/README.md b/dev/README.md index e69de29b..496e833a 100644 --- a/dev/README.md +++ b/dev/README.md @@ -0,0 +1,64 @@ +# Development Tools + +This directory contains development tools and utilities for the Microsoft Agents for Python project. + +## Contents + +- [`install.sh`](install.sh) - Installation script for development dependencies +- [`benchmark/`](benchmark/) - Performance benchmarking tools +- [`microsoft-agents-testing/`](microsoft-agents-testing/) - Testing framework package + +## Quick Setup + +To set up the development environment, run the installation script: + +```bash +./install.sh +``` + +This script installs the testing framework package in editable mode, allowing you to make changes and test them immediately. + +## Benchmarking + +The [`benchmark/`](benchmark/) directory contains tools for performance testing and stress testing agents. See the [benchmark README](benchmark/README.md) for detailed setup and usage instructions. + +### Key Features: +- Support for both standard Python and free-threaded Python 3.13+ +- Configurable worker threads for stress testing +- Token-based authentication for realistic testing scenarios +- Basic payload sending stress tests + +### Quick Start: +1. Navigate to the benchmark directory +2. Set up a Python virtual environment +3. Configure authentication settings in `.env` (use `env.template` as a reference) +4. Run tests with `python -m src.main --num_workers=` + +## Testing Framework + +The [`microsoft-agents-testing/`](microsoft-agents-testing/) directory contains a specialized testing framework for Microsoft Agents. This package provides: + +- Testing utilities and helpers +- Mock objects and test fixtures +- Integration testing tools + +The package can be installed in editable mode using the provided installation script, making it easy to develop and test changes. + +## Prerequisites + +- Python 3.10 or higher +- Virtual environment (recommended) +- For benchmarking: Valid Azure Bot Service credentials + +## Development Workflow + +1. Install development dependencies using `./install.sh` +2. Make changes to the testing framework or benchmark tools +3. Test your changes using the benchmark tools +4. Run the full test suite to ensure compatibility + +## Notes + +- The benchmark tool currently uses threading rather than async coroutines +- Free-threaded Python 3.13+ support is available for improved performance +- All tools require proper authentication configuration for realistic testing \ No newline at end of file From 8a5f22e1b629d22af745400daf7540f75ae1a6df Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 09:58:45 -0800 Subject: [PATCH 45/81] Revising README --- dev/README.md | 53 +++++++-------------------------------------------- 1 file changed, 7 insertions(+), 46 deletions(-) diff --git a/dev/README.md b/dev/README.md index 496e833a..11a470b9 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,64 +1,25 @@ # Development Tools -This directory contains development tools and utilities for the Microsoft Agents for Python project. +Development utilities for the Microsoft Agents for Python project. ## Contents -- [`install.sh`](install.sh) - Installation script for development dependencies -- [`benchmark/`](benchmark/) - Performance benchmarking tools -- [`microsoft-agents-testing/`](microsoft-agents-testing/) - Testing framework package +- **[`install.sh`](install.sh)** - Installs testing framework in editable mode +- **[`benchmark/`](benchmark/)** - Performance testing and stress testing tools +- **[`microsoft-agents-testing/`](microsoft-agents-testing/)** - Testing framework package ## Quick Setup -To set up the development environment, run the installation script: - ```bash ./install.sh ``` -This script installs the testing framework package in editable mode, allowing you to make changes and test them immediately. - ## Benchmarking -The [`benchmark/`](benchmark/) directory contains tools for performance testing and stress testing agents. See the [benchmark README](benchmark/README.md) for detailed setup and usage instructions. - -### Key Features: -- Support for both standard Python and free-threaded Python 3.13+ -- Configurable worker threads for stress testing -- Token-based authentication for realistic testing scenarios -- Basic payload sending stress tests +Performance testing tools with support for concurrent workers and authentication. Requires a running agent instance and Azure Bot Service credentials. -### Quick Start: -1. Navigate to the benchmark directory -2. Set up a Python virtual environment -3. Configure authentication settings in `.env` (use `env.template` as a reference) -4. Run tests with `python -m src.main --num_workers=` +See [benchmark/README.md](benchmark/README.md) for setup and usage details. ## Testing Framework -The [`microsoft-agents-testing/`](microsoft-agents-testing/) directory contains a specialized testing framework for Microsoft Agents. This package provides: - -- Testing utilities and helpers -- Mock objects and test fixtures -- Integration testing tools - -The package can be installed in editable mode using the provided installation script, making it easy to develop and test changes. - -## Prerequisites - -- Python 3.10 or higher -- Virtual environment (recommended) -- For benchmarking: Valid Azure Bot Service credentials - -## Development Workflow - -1. Install development dependencies using `./install.sh` -2. Make changes to the testing framework or benchmark tools -3. Test your changes using the benchmark tools -4. Run the full test suite to ensure compatibility - -## Notes - -- The benchmark tool currently uses threading rather than async coroutines -- Free-threaded Python 3.13+ support is available for improved performance -- All tools require proper authentication configuration for realistic testing \ No newline at end of file +Provides testing utilities and helpers for Microsoft Agents development. Installed in editable mode for active development. \ No newline at end of file From 4b7c673feaa8cafe7b38c53152eb5958758156cf Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 09:59:48 -0800 Subject: [PATCH 46/81] Formatting --- dev/benchmark/src/main.py | 3 +- dev/benchmark/src/output.py | 9 ++++-- .../microsoft_agents/testing/__init__.py | 16 +++------- .../microsoft_agents/testing/auth/__init__.py | 5 +-- .../testing/auth/generate_token.py | 12 ++++--- .../testing/integration/__init__.py | 2 +- .../core/aiohttp/aiohttp_environment.py | 4 ++- .../integration/core/client/agent_client.py | 29 ++++++++++++----- .../testing/integration/core/integration.py | 18 ++++++++--- .../microsoft_agents/testing/sdk_config.py | 31 +++++++++++-------- .../testing/utils/__init__.py | 2 +- .../testing/utils/populate_activity.py | 10 +++--- .../microsoft_agents/testing/utils/urls.py | 3 +- .../core/test_application_runner.py | 1 + .../core/test_integration_from_sample.py | 8 ++--- .../core/test_integration_from_service_url.py | 5 ++- .../tests/manual_test/main.py | 16 +++++++--- .../tests/samples/quickstart_sample.py | 14 ++++++--- 18 files changed, 115 insertions(+), 73 deletions(-) diff --git a/dev/benchmark/src/main.py b/dev/benchmark/src/main.py index c27b49ca..4e7de6de 100644 --- a/dev/benchmark/src/main.py +++ b/dev/benchmark/src/main.py @@ -17,6 +17,7 @@ BenchmarkConfig.load_from_env() + @click.command() @click.option( "--payload_path", "-p", default="./payload.json", help="Path to the payload file." @@ -38,7 +39,7 @@ def main(payload_path: str, num_workers: int, verbose: bool, async_mode: bool): func = create_payload_sender(payload) executor: Executor = CoroutineExecutor() if async_mode else ThreadExecutor() - + start_time = datetime.now(timezone.utc).timestamp() results = executor.run(func, num_workers=num_workers) end_time = datetime.now(timezone.utc).timestamp() diff --git a/dev/benchmark/src/output.py b/dev/benchmark/src/output.py index b7efa12d..a0d3d76a 100644 --- a/dev/benchmark/src/output.py +++ b/dev/benchmark/src/output.py @@ -1,9 +1,12 @@ from .executor import ExecutionResult + def output_results(results: list[ExecutionResult]) -> None: """Output the results of the benchmark to the console.""" - + for result in results: status = "Success" if result.success else "Failure" - print(f"Execution ID: {result.exe_id}, Duration: {result.duration:.4f} seconds, Status: {status}") - print(result.result) \ No newline at end of file + print( + f"Execution ID: {result.exe_id}, Duration: {result.duration:.4f} seconds, Status: {status}" + ) + print(result.result) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py index 0de64c77..c8364fff 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py @@ -1,14 +1,8 @@ from .sdk_config import SDKConfig -from .auth import ( - generate_token, - generate_token_from_config -) +from .auth import generate_token, generate_token_from_config -from .utils import ( - populate_activity, - get_host_and_port -) +from .utils import populate_activity, get_host_and_port from .integration import ( Sample, @@ -17,7 +11,7 @@ AgentClient, ResponseClient, AiohttpEnvironment, - Integration + Integration, ) __all__ = [ @@ -32,5 +26,5 @@ "AiohttpEnvironment", "Integration", "populate_activity", - "get_host_and_port" -] \ No newline at end of file + "get_host_and_port", +] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py index a34ef21f..3fe2a78f 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py @@ -1,6 +1,3 @@ from .generate_token import generate_token, generate_token_from_config -__all__ = [ - "generate_token", - "generate_token_from_config" -] \ No newline at end of file +__all__ = ["generate_token", "generate_token_from_config"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py index 029b05ab..948c2584 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py @@ -3,16 +3,19 @@ from microsoft_agents.hosting.core import AgentAuthConfiguration from microsoft_agents.testing.sdk_config import SDKConfig + def generate_token(app_id: str, app_secret: str, tenant_id: str) -> str: """Generate a token using the provided app credentials. - + :param app_id: Application (client) ID. :param app_secret: Application client secret. :param tenant_id: Directory (tenant) ID. :return: Generated access token as a string. """ - authority_endpoint = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" + authority_endpoint = ( + f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" + ) res = requests.post( authority_endpoint, @@ -29,9 +32,10 @@ def generate_token(app_id: str, app_secret: str, tenant_id: str) -> str: ) return res.json().get("access_token") + def generate_token_from_config(sdk_config: SDKConfig) -> str: """Generates a token using a provided config object. - + :param config: Configuration dictionary containing connection settings. :return: Generated access token as a string. """ @@ -44,4 +48,4 @@ def generate_token_from_config(sdk_config: SDKConfig) -> str: if not app_id or not app_secret or not tenant_id: raise ValueError("Incorrect configuration provided for token generation.") - return generate_token(app_id, app_secret, tenant_id) \ No newline at end of file + return generate_token(app_id, app_secret, tenant_id) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py index a283062f..3ad1e376 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py @@ -16,4 +16,4 @@ "Environment", "Integration", "Sample", -] \ No newline at end of file +] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py index c8256618..9185aed3 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py @@ -51,7 +51,9 @@ async def entry_point(req: Request) -> Response: APP = Application(middlewares=[jwt_authorization_middleware]) APP.router.add_post("/api/messages", entry_point) - APP["agent_configuration"] = self.connection_manager.get_default_connection_configuration() + APP["agent_configuration"] = ( + self.connection_manager.get_default_connection_configuration() + ) APP["agent_app"] = self.agent_application APP["adapter"] = self.adapter diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py index 7330bc98..de3a9ca0 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py @@ -12,7 +12,7 @@ ActivityTypes, DeliveryModes, ChannelAccount, - ConversationAccount + ConversationAccount, ) from microsoft_agents.testing.utils import populate_activity @@ -24,6 +24,7 @@ "locale": "en-US", } + class AgentClient: def __init__( @@ -48,7 +49,9 @@ def __init__( self._client: Optional[ClientSession] = None - self._default_activity_data: Activity | dict = default_activity_data or _DEFAULT_ACTIVITY_VALUES + self._default_activity_data: Activity | dict = ( + default_activity_data or _DEFAULT_ACTIVITY_VALUES + ) @property def agent_url(self) -> str: @@ -95,7 +98,9 @@ async def send_request(self, activity: Activity, sleep: float = 0) -> str: if activity.conversation: activity.conversation.id = self._cid else: - activity.conversation = ConversationAccount(id=self._cid or "") + activity.conversation = ConversationAccount( + id=self._cid or "" + ) if self.service_url: activity.service_url = self.service_url @@ -105,7 +110,9 @@ async def send_request(self, activity: Activity, sleep: float = 0) -> str: async with self._client.post( "api/messages", headers=self._headers, - json=activity.model_dump(by_alias=True, exclude_unset=True, exclude_none=True, mode="json"), + json=activity.model_dump( + by_alias=True, exclude_unset=True, exclude_none=True, mode="json" + ), ) as response: content = await response.text() if not response.ok: @@ -124,7 +131,10 @@ def _to_activity(self, activity_or_text: Activity | str) -> Activity: return cast(Activity, activity_or_text) async def send_activity( - self, activity_or_text: Activity | str, sleep: float = 0, timeout: Optional[float] = None + self, + activity_or_text: Activity | str, + sleep: float = 0, + timeout: Optional[float] = None, ) -> str: timeout = timeout or self._default_timeout activity = self._to_activity(activity_or_text) @@ -132,12 +142,17 @@ async def send_activity( return content async def send_expect_replies( - self, activity_or_text: Activity | str, sleep: float = 0, timeout: Optional[float] = None + self, + activity_or_text: Activity | str, + sleep: float = 0, + timeout: Optional[float] = None, ) -> list[Activity]: timeout = timeout or self._default_timeout activity = self._to_activity(activity_or_text) activity.delivery_mode = DeliveryModes.expect_replies - activity.service_url = activity.service_url or "http://localhost" # temporary fix + activity.service_url = ( + activity.service_url or "http://localhost" + ) # temporary fix content = await self.send_request(activity, sleep=sleep) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py index c3105c74..3b459617 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py @@ -80,9 +80,15 @@ def create_agent_client(self) -> AgentClient: load_dotenv("./src/tests/.env") self._config.update( { - "client_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", ""), - "tenant_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID", ""), - "client_secret": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET", ""), + "client_id": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", "" + ), + "tenant_id": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID", "" + ), + "client_secret": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET", "" + ), } ) @@ -96,7 +102,9 @@ def create_agent_client(self) -> AgentClient: return agent_client @pytest.fixture - async def agent_client(self, sample, environment) -> AsyncGenerator[AgentClient, None]: + async def agent_client( + self, sample, environment + ) -> AsyncGenerator[AgentClient, None]: agent_client = self.create_agent_client() yield agent_client await agent_client.close() @@ -110,4 +118,4 @@ async def _create_response_client(self) -> ResponseClient: async def response_client(self) -> AsyncGenerator[ResponseClient, None]: """Provides the response client instance.""" async with await self._create_response_client() as response_client: - yield response_client \ No newline at end of file + yield response_client diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py index 6352b42c..c661823a 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py @@ -3,37 +3,42 @@ from dotenv import load_dotenv, dotenv_values from typing import Optional -from microsoft_agents.activity import ( - load_configuration_from_env -) -from microsoft_agents.hosting.core import ( - AgentAuthConfiguration -) +from microsoft_agents.activity import load_configuration_from_env +from microsoft_agents.hosting.core import AgentAuthConfiguration + class SDKConfig: """Loads and provides access to SDK configuration from a .env file or environment variables. - + Immutable access to the configuration dictionary is provided via the `config` property. """ - def __init__(self, env_path: Optional[str] = None, load_into_environment: bool = False): + def __init__( + self, env_path: Optional[str] = None, load_into_environment: bool = False + ): """Initializes the SDKConfig by loading configuration from a .env file or environment variables. - + :param env_path: Optional path to the .env file. If None, defaults to '.env' in the current directory. :param load_into_environment: If True, loads the .env file directly into the configuration dictionary. """ if load_into_environment: - self._config = load_configuration_from_env(dotenv_values(env_path)) # Load .env file + self._config = load_configuration_from_env( + dotenv_values(env_path) + ) # Load .env file else: load_dotenv(env_path) # Load .env file into environment variables - self._config = load_configuration_from_env(os.environ) # Load from environment variables + self._config = load_configuration_from_env( + os.environ + ) # Load from environment variables @property def config(self) -> dict: """Returns the loaded configuration dictionary.""" return deepcopy(self._config) - def get_connection(self, connection_name: str = "SERVICE_CONNECTION") -> AgentAuthConfiguration: + def get_connection( + self, connection_name: str = "SERVICE_CONNECTION" + ) -> AgentAuthConfiguration: """Creates an AgentAuthConfiguration from a provided config object.""" data = self._config["CONNECTIONS"][connection_name]["SETTINGS"] - return AgentAuthConfiguration(**data) \ No newline at end of file + return AgentAuthConfiguration(**data) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py index c8c26208..0c902992 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -4,4 +4,4 @@ __all__ = [ "populate_activity", "get_host_and_port", -] \ No newline at end of file +] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py index 5ba73ec6..69e815e7 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py @@ -1,10 +1,8 @@ from microsoft_agents.activity import Activity -def populate_activity( - original: Activity, - defaults: Activity | dict -) -> Activity: - + +def populate_activity(original: Activity, defaults: Activity | dict) -> Activity: + if isinstance(defaults, Activity): defaults = Activity.model_dump(exclude_unset=True) @@ -42,4 +40,4 @@ def populate_activity( # if not activity.conversation: # activity.conversation = ConversationAccount(id="conversation1") -# return activity \ No newline at end of file +# return activity diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py index df452416..d964ebd2 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py @@ -1,9 +1,10 @@ from urllib.parse import urlparse + def get_host_and_port(url: str) -> tuple[str, int]: """Extract host and port from a URL.""" parsed_url = urlparse(url) host = parsed_url.hostname or "localhost" port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) - return host, port \ No newline at end of file + return host, port diff --git a/dev/microsoft-agents-testing/tests/integration/core/test_application_runner.py b/dev/microsoft-agents-testing/tests/integration/core/test_application_runner.py index 65090594..719203b7 100644 --- a/dev/microsoft-agents-testing/tests/integration/core/test_application_runner.py +++ b/dev/microsoft-agents-testing/tests/integration/core/test_application_runner.py @@ -3,6 +3,7 @@ from ._common import SimpleRunner, OtherSimpleRunner + class TestApplicationRunner: @pytest.mark.asyncio diff --git a/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_sample.py b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_sample.py index b7019573..998c0928 100644 --- a/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_sample.py +++ b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_sample.py @@ -2,12 +2,7 @@ import asyncio from copy import copy -from microsoft_agents.testing import ( - ApplicationRunner, - Environment, - Integration, - Sample -) +from microsoft_agents.testing import ApplicationRunner, Environment, Integration, Sample from ._common import SimpleRunner @@ -43,6 +38,7 @@ async def init_app(self): def app(self) -> None: return None + class TestIntegrationFromSample(Integration): _sample_cls = SimpleSample _environment_cls = SimpleEnvironment diff --git a/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py index 4ff707cc..49b4d6fc 100644 --- a/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py +++ b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py @@ -6,6 +6,7 @@ from microsoft_agents.testing import Integration + class TestIntegrationFromURL(Integration): _agent_url = "http://localhost:8000/" _service_url = "http://localhost:8001/" @@ -16,7 +17,9 @@ async def test_service_url_integration(self, agent_client): with aioresponses() as mocked: - mocked.post(f"{self.agent_url}api/messages", status=200, body="Service response") + mocked.post( + f"{self.agent_url}api/messages", status=200, body="Service response" + ) res = await agent_client.send_activity("Hello, service!") assert res == "Service response" diff --git a/dev/microsoft-agents-testing/tests/manual_test/main.py b/dev/microsoft-agents-testing/tests/manual_test/main.py index 3246c51d..7201dfef 100644 --- a/dev/microsoft-agents-testing/tests/manual_test/main.py +++ b/dev/microsoft-agents-testing/tests/manual_test/main.py @@ -9,6 +9,7 @@ from dotenv import load_dotenv + async def main(): env = AiohttpEnvironment() @@ -20,9 +21,15 @@ async def main(): load_dotenv("./src/tests/.env") config = { - "client_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", ""), - "tenant_id": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID", ""), - "client_secret": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET", ""), + "client_id": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", "" + ), + "tenant_id": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID", "" + ), + "client_secret": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET", "" + ), } client = AgentClient( @@ -42,5 +49,6 @@ async def main(): await client.close() + if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/dev/microsoft-agents-testing/tests/samples/quickstart_sample.py b/dev/microsoft-agents-testing/tests/samples/quickstart_sample.py index 04da32c1..26b1fef0 100644 --- a/dev/microsoft-agents-testing/tests/samples/quickstart_sample.py +++ b/dev/microsoft-agents-testing/tests/samples/quickstart_sample.py @@ -9,6 +9,7 @@ from microsoft_agents.hosting.core import AgentApplication, TurnContext, TurnState from microsoft_agents.testing.integration.core import Sample + class QuickstartSample(Sample): """A quickstart sample implementation.""" @@ -18,11 +19,16 @@ async def get_config(cls) -> dict: load_dotenv("./src/tests/.env") - return { - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID"), - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET"), - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID": os.getenv("CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID"), + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID" + ), + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET" + ), + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID" + ), } async def init_app(self): From 01f8f5aec2b6bdc8d9f38f71508c9f2d05977c8e Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 11:14:52 -0800 Subject: [PATCH 47/81] Addressing PR comments --- dev/benchmark/src/main.py | 4 +-- .../core/aiohttp/aiohttp_environment.py | 3 +- .../core/aiohttp/aiohttp_runner.py | 33 ++++++++----------- .../integration/core/client/agent_client.py | 2 -- .../core/client/response_client.py | 9 +++-- .../testing/utils/populate_activity.py | 2 +- .../core/client/test_agent_client.py | 2 -- .../core/test_integration_from_service_url.py | 2 +- 8 files changed, 22 insertions(+), 35 deletions(-) diff --git a/dev/benchmark/src/main.py b/dev/benchmark/src/main.py index 4e7de6de..d8a31c83 100644 --- a/dev/benchmark/src/main.py +++ b/dev/benchmark/src/main.py @@ -1,8 +1,6 @@ -import json, sys -from io import StringIO +import json import logging from datetime import datetime, timezone -from contextlib import contextmanager import click diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py index 9185aed3..7ff83dc0 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py @@ -1,5 +1,4 @@ -from tkinter import E -from aiohttp.web import Request, Response, Application, run_app +from aiohttp.web import Request, Response, Application from microsoft_agents.hosting.aiohttp import ( CloudAdapter, diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py index 3b6780d4..c8fe23c2 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py @@ -1,5 +1,4 @@ from typing import Optional -from typing import Optional from threading import Thread, Event import asyncio @@ -36,28 +35,24 @@ def url(self) -> str: return self._url async def _start_server(self) -> None: - try: - assert isinstance(self._app, Application) - - self._runner = AppRunner(self._app) - await self._runner.setup() - self._site = TCPSite(self._runner, self._host, self._port) - await self._site.start() + assert isinstance(self._app, Application) - # Wait for shutdown signal - while not self._shutdown_event.is_set(): - await asyncio.sleep(0.1) + self._runner = AppRunner(self._app) + await self._runner.setup() + self._site = TCPSite(self._runner, self._host, self._port) + await self._site.start() - # Cleanup - await self._site.stop() - await self._runner.cleanup() + # Wait for shutdown signal + while not self._shutdown_event.is_set(): + await asyncio.sleep(0.1) - except Exception as error: - raise error + # Cleanup + await self._site.stop() + await self._runner.cleanup() async def __aenter__(self): if self._server_thread: - raise RuntimeError("ResponseClient is already running.") + raise RuntimeError("AiohttpRunner is already running.") self._shutdown_event.clear() self._server_thread = Thread( @@ -72,7 +67,7 @@ async def __aenter__(self): async def _stop_server(self): if not self._server_thread: - raise RuntimeError("ResponseClient is not running.") + raise RuntimeError("AiohttpRunner is not running.") try: async with ClientSession() as session: @@ -97,7 +92,7 @@ async def _shutdown_route(self, request: Request) -> Response: async def __aexit__(self, exc_type, exc, tb): if not self._server_thread: - raise RuntimeError("ResponseClient is not running.") + raise RuntimeError("AiohttpRunner is not running.") try: async with ClientSession() as session: async with session.get( diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py index de3a9ca0..73067207 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py @@ -1,5 +1,3 @@ -from email.policy import default -import os import json import asyncio from typing import Optional, cast diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py index d93bfb80..b283efdf 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py @@ -2,8 +2,7 @@ import sys from io import StringIO -from typing import Optional -from threading import Lock, Thread, Event +from threading import Lock import asyncio from aiohttp.web import Application, Request, Response @@ -63,9 +62,9 @@ async def _handle_conversation(self, request: Request) -> Response: data = await request.json() activity = Activity.model_validate(data) - conversation_id = ( - activity.conversation.id if activity.conversation else None - ) + # conversation_id = ( + # activity.conversation.id if activity.conversation else None + # ) with self._activities_list_lock: self._activities_list.append(activity) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py index 69e815e7..2e62479c 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py @@ -4,7 +4,7 @@ def populate_activity(original: Activity, defaults: Activity | dict) -> Activity: if isinstance(defaults, Activity): - defaults = Activity.model_dump(exclude_unset=True) + defaults = defaults.model_dump(exclude_unset=True) new_activity = original.model_copy() diff --git a/dev/microsoft-agents-testing/tests/integration/core/client/test_agent_client.py b/dev/microsoft-agents-testing/tests/integration/core/client/test_agent_client.py index 1a8055dc..3bc59452 100644 --- a/dev/microsoft-agents-testing/tests/integration/core/client/test_agent_client.py +++ b/dev/microsoft-agents-testing/tests/integration/core/client/test_agent_client.py @@ -1,6 +1,4 @@ import json -from contextlib import contextmanager -import re import pytest from aioresponses import aioresponses diff --git a/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py index 49b4d6fc..6e3402a6 100644 --- a/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py +++ b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py @@ -33,7 +33,7 @@ async def test_service_url_integration_with_response_side_effect( with aioresponses() as mocked: def callback(url, **kwargs): - a = requests.post( + requests.post( f"{self.service_url}/v3/conversations/test-conv", json=kwargs.get("json"), ) From 8f87aff94c1354f87dcd531779692f97a26041f2 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 11:22:18 -0800 Subject: [PATCH 48/81] Removing unused import --- .../tests/integration/core/test_integration_from_service_url.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py index 6e3402a6..4262a624 100644 --- a/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py +++ b/dev/microsoft-agents-testing/tests/integration/core/test_integration_from_service_url.py @@ -1,7 +1,6 @@ import pytest import asyncio import requests -from copy import copy from aioresponses import aioresponses, CallbackResult from microsoft_agents.testing import Integration From e742e00b84da7a15abee51f3a3b8922d07f691dd Mon Sep 17 00:00:00 2001 From: rodrigobr-msft Date: Thu, 6 Nov 2025 11:28:05 -0800 Subject: [PATCH 49/81] Update dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../microsoft_agents/testing/sdk_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py index c661823a..c1824ae5 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py @@ -19,7 +19,7 @@ def __init__( """Initializes the SDKConfig by loading configuration from a .env file or environment variables. :param env_path: Optional path to the .env file. If None, defaults to '.env' in the current directory. - :param load_into_environment: If True, loads the .env file directly into the configuration dictionary. + :param load_into_environment: If True, loads the .env file directly into the configuration dictionary (does NOT load into environment variables). If False, loads the .env file into environment variables first, then loads the configuration from those environment variables. """ if load_into_environment: self._config = load_configuration_from_env( From a6e305ac5fc4261f695d99ac7c68ed5ff18ccd63 Mon Sep 17 00:00:00 2001 From: rodrigobr-msft Date: Thu, 6 Nov 2025 11:28:18 -0800 Subject: [PATCH 50/81] Update dev/microsoft-agents-testing/pyproject.toml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- dev/microsoft-agents-testing/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/microsoft-agents-testing/pyproject.toml b/dev/microsoft-agents-testing/pyproject.toml index c231347b..cf659e6f 100644 --- a/dev/microsoft-agents-testing/pyproject.toml +++ b/dev/microsoft-agents-testing/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "microsoft-agents-hosting-core" dynamic = ["version", "dependencies"] description = "Core library for Microsoft Agents" -readme = {file = "readme.md", content-type = "text/markdown"} +readme = {file = "README.md", content-type = "text/markdown"} authors = [{name = "Microsoft Corporation"}] license = "MIT" license-files = ["LICENSE"] From c3877a368d83606ab764c70343a8948d2137f997 Mon Sep 17 00:00:00 2001 From: rodrigobr-msft Date: Thu, 6 Nov 2025 11:28:27 -0800 Subject: [PATCH 51/81] Update dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../microsoft_agents/testing/auth/generate_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py index 948c2584..83106639 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py @@ -36,7 +36,7 @@ def generate_token(app_id: str, app_secret: str, tenant_id: str) -> str: def generate_token_from_config(sdk_config: SDKConfig) -> str: """Generates a token using a provided config object. - :param config: Configuration dictionary containing connection settings. + :param sdk_config: Configuration dictionary containing connection settings. :return: Generated access token as a string. """ From fb5b76d57a4268fa13978360026a1281a445e693 Mon Sep 17 00:00:00 2001 From: rodrigobr-msft Date: Thu, 6 Nov 2025 11:28:49 -0800 Subject: [PATCH 52/81] Update dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../testing/integration/core/environment.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py index 2c9b1ae1..0aa99f24 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py @@ -32,6 +32,9 @@ async def init_env(self, environ_config: dict) -> None: raise NotImplementedError() @abstractmethod - def create_runner(self) -> ApplicationRunner: - """Create an application runner for the environment.""" + def create_runner(self, *args, **kwargs) -> ApplicationRunner: + """Create an application runner for the environment. + + Subclasses may accept additional arguments as needed. + """ raise NotImplementedError() From 5eb9bfb4d495f78db6bb0d3742d296e79dea42d3 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 11:29:59 -0800 Subject: [PATCH 53/81] Removing unused code --- .../testing/utils/populate_activity.py | 31 ++----------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py index 2e62479c..a6b1c19f 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py @@ -2,6 +2,7 @@ def populate_activity(original: Activity, defaults: Activity | dict) -> Activity: + """Populate an Activity object with default values.""" if isinstance(defaults, Activity): defaults = defaults.model_dump(exclude_unset=True) @@ -12,32 +13,4 @@ def populate_activity(original: Activity, defaults: Activity | dict) -> Activity if getattr(new_activity, key) is None: setattr(new_activity, key, defaults[key]) - return new_activity - - -# def _populate_incoming_activity(activity: Activity) -> Activity: - -# activity = activity.model_copy() - -# if not activity.locale: -# activity.locale = "en-US" - -# if not activity.channel_id: -# activity.channel_id = "emulator" - -# if not activity.delivery_mode: -# activity.delivery_mode = DeliveryModes.normal - -# if not activity.service_url: -# activity.service_url = "http://localhost" - -# if not activity.recipient: -# activity.recipient = ChannelAccount(id="agent", name="Agent") - -# if not activity.from_property: -# activity.from_property = ChannelAccount(id="user", name="User") - -# if not activity.conversation: -# activity.conversation = ConversationAccount(id="conversation1") - -# return activity + return new_activity \ No newline at end of file From e4514597d39663f191b4d59d98164134aa20a436 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 13:40:44 -0800 Subject: [PATCH 54/81] Adding field assertion helpers --- .../testing/asserts/__init__.py | 0 .../testing/asserts/assert_activity.py | 32 +++++++++++++++++++ .../testing/asserts/check_field.py | 4 +++ .../testing/asserts/type_defs.py | 14 ++++++++ .../testing/utils/__init__.py | 6 +++- .../testing/utils/{urls.py => misc.py} | 8 +++++ 6 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/asserts/__init__.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/asserts/check_field.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/asserts/type_defs.py rename dev/microsoft-agents-testing/microsoft_agents/testing/utils/{urls.py => misc.py} (52%) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py new file mode 100644 index 00000000..b9563731 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py @@ -0,0 +1,32 @@ +from microsoft_agents.activity import ( + Activity, +) + +from microsoft_agents.testing.utils import ( + normalize_activity_data +) + +from .check_field import check_field +from .type_defs import _UNSET_FIELD, FieldAssertionTypes + +def assert_activity(activity: Activity, baseline: Activity | dict) -> None: + + baseline = normalize_activity_data(baseline) + + for key in baseline.keys(): + + assertion_type = FieldAssertionTypes.EQUALS + assertion_info = baseline[key] + if isinstance(assertion_info, dict) and "assertion_type" in assertion_info: + assertion_type = assertion_info["assertion_type"] + baseline_value = assertion_info["value"] + else: + baseline_value = assertion_info + + target_value = getattr(activity, key, _UNSET_FIELD) + + assert check_field( + baseline_value, + target_value, + assertion_type + ) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/check_field.py new file mode 100644 index 00000000..24f73776 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/check_field.py @@ -0,0 +1,4 @@ +from .type_defs import FieldAssertionTypes + +def check_field(baseline_value: Any, target_value: Any, assertion_type: FieldAssertionType) -> bool: + return False \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/type_defs.py new file mode 100644 index 00000000..e37311a8 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/type_defs.py @@ -0,0 +1,14 @@ +from enum import Enum + +_UNSET_FIELD = object() + +class FieldAssertionTypes(Enum): + """Defines the types of assertions that can be made on fields.""" + + EQUALS = "equals" + NOT_EQUALS = "not_equals" + GREATER_THAN = "greater_than" + LESS_THAN = "less_than" + CONTAINS = "contains" + NOT_CONTAINS = "not_contains" + RE_MATCH = "re_match" \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py index 0c902992..8e1b279d 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -1,7 +1,11 @@ from .populate_activity import populate_activity -from .urls import get_host_and_port +from .misc import ( + get_host_and_port, + normalize_activity_data +) __all__ = [ "populate_activity", "get_host_and_port", + "normalize_activity_data", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py similarity index 52% rename from dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py index d964ebd2..d5764476 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py @@ -1,5 +1,6 @@ from urllib.parse import urlparse +from microsoft_agents.activity import Activity def get_host_and_port(url: str) -> tuple[str, int]: """Extract host and port from a URL.""" @@ -8,3 +9,10 @@ def get_host_and_port(url: str) -> tuple[str, int]: host = parsed_url.hostname or "localhost" port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) return host, port + +def normalize_activity_data(source: Activity | dict) -> dict: + """Normalize Activity data to a dictionary format.""" + + if isinstance(source, Activity): + return source.model_dump(exclude_unset=True) + return source \ No newline at end of file From f241df800843b7acab8aa7d1aeae6557df4f7848 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 14:21:02 -0800 Subject: [PATCH 55/81] Implementing field assertion types --- .../testing/asserts/assert_activity.py | 27 +++++++++---------- .../testing/asserts/check_field.py | 26 +++++++++++++++--- .../testing/asserts/type_defs.py | 7 ++--- .../testing/integration/core/environment.py | 2 +- .../testing/utils/__init__.py | 5 +--- .../microsoft_agents/testing/utils/misc.py | 4 ++- .../testing/utils/populate_activity.py | 2 +- 7 files changed, 45 insertions(+), 28 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py index b9563731..4edca756 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py @@ -1,21 +1,22 @@ -from microsoft_agents.activity import ( - Activity, -) - -from microsoft_agents.testing.utils import ( - normalize_activity_data -) +from microsoft_agents.activity import Activity +from microsoft_agents.testing.utils import normalize_activity_data from .check_field import check_field -from .type_defs import _UNSET_FIELD, FieldAssertionTypes +from .type_defs import _UNSET_FIELD, FieldAssertionType + def assert_activity(activity: Activity, baseline: Activity | dict) -> None: - + """Asserts that the given activity matches the baseline activity. + + :param activity: The activity to be tested. + :param baseline: The baseline activity or a dictionary representing the expected activity data. + """ + baseline = normalize_activity_data(baseline) for key in baseline.keys(): - assertion_type = FieldAssertionTypes.EQUALS + assertion_type = FieldAssertionType.EQUALS assertion_info = baseline[key] if isinstance(assertion_info, dict) and "assertion_type" in assertion_info: assertion_type = assertion_info["assertion_type"] @@ -25,8 +26,4 @@ def assert_activity(activity: Activity, baseline: Activity | dict) -> None: target_value = getattr(activity, key, _UNSET_FIELD) - assert check_field( - baseline_value, - target_value, - assertion_type - ) \ No newline at end of file + assert check_field(baseline_value, target_value, assertion_type) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/check_field.py index 24f73776..c958cf61 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/check_field.py @@ -1,4 +1,24 @@ -from .type_defs import FieldAssertionTypes +import re +from typing import Any -def check_field(baseline_value: Any, target_value: Any, assertion_type: FieldAssertionType) -> bool: - return False \ No newline at end of file +from .type_defs import FieldAssertionType + +_OPERATIONS = { + FieldAssertionType.EQUALS: lambda a, b: a == b, + FieldAssertionType.NOT_EQUALS: lambda a, b: a != b, + FieldAssertionType.GREATER_THAN: lambda a, b: a > b, + FieldAssertionType.LESS_THAN: lambda a, b: a < b, + FieldAssertionType.CONTAINS: lambda a, b: b in a, + FieldAssertionType.NOT_CONTAINS: lambda a, b: b not in a, + FieldAssertionType.RE_MATCH: lambda a, b: re.match(b, a) is not None, +} + + +def check_field( + actual_value: Any, baseline_value: Any, assertion_type: FieldAssertionType +) -> bool: + + operation = _OPERATIONS.get(assertion_type) + if not operation: + return False # missing operation for the assertion type + return operation(actual_value, baseline_value) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/type_defs.py index e37311a8..d8f6dfa0 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/type_defs.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/type_defs.py @@ -2,13 +2,14 @@ _UNSET_FIELD = object() -class FieldAssertionTypes(Enum): + +class FieldAssertionType(Enum): """Defines the types of assertions that can be made on fields.""" - + EQUALS = "equals" NOT_EQUALS = "not_equals" GREATER_THAN = "greater_than" LESS_THAN = "less_than" CONTAINS = "contains" NOT_CONTAINS = "not_contains" - RE_MATCH = "re_match" \ No newline at end of file + RE_MATCH = "re_match" diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py index 0aa99f24..2611da36 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py @@ -34,7 +34,7 @@ async def init_env(self, environ_config: dict) -> None: @abstractmethod def create_runner(self, *args, **kwargs) -> ApplicationRunner: """Create an application runner for the environment. - + Subclasses may accept additional arguments as needed. """ raise NotImplementedError() diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py index 8e1b279d..1875db71 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -1,8 +1,5 @@ from .populate_activity import populate_activity -from .misc import ( - get_host_and_port, - normalize_activity_data -) +from .misc import get_host_and_port, normalize_activity_data __all__ = [ "populate_activity", diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py index d5764476..0331b942 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py @@ -2,6 +2,7 @@ from microsoft_agents.activity import Activity + def get_host_and_port(url: str) -> tuple[str, int]: """Extract host and port from a URL.""" @@ -10,9 +11,10 @@ def get_host_and_port(url: str) -> tuple[str, int]: port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) return host, port + def normalize_activity_data(source: Activity | dict) -> dict: """Normalize Activity data to a dictionary format.""" if isinstance(source, Activity): return source.model_dump(exclude_unset=True) - return source \ No newline at end of file + return source diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py index a6b1c19f..6aa7a265 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py @@ -13,4 +13,4 @@ def populate_activity(original: Activity, defaults: Activity | dict) -> Activity if getattr(new_activity, key) is None: setattr(new_activity, key, defaults[key]) - return new_activity \ No newline at end of file + return new_activity From cb3c6aaf7e9e5976a6db08fa93e662a1bbeff1d8 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 14:56:08 -0800 Subject: [PATCH 56/81] Adding data driven test foundation --- .../data_driven_tests/basic_test.json | 0 dev/integration/integration/README.md | 2 + .../integration/integration/__init__.py | 0 .../integration/components/__init__.py | 0 .../components/test_typing_indicator.py | 36 +++++ .../integration/foundational/__init__.py | 0 .../integration/foundational/_common.py | 10 ++ .../integration/foundational/test_suite.py | 141 ++++++++++++++++++ .../integration/test_quickstart.py | 21 +++ dev/integration/integration/requirements.txt | 1 + dev/integration/test_basics.py | 7 + .../testing/asserts/select_activity.py | 2 + .../testing/integration/data_driven_test.py | 39 +++++ .../testing/integration/data_driven_tester.py | 33 ++++ 14 files changed, 292 insertions(+) create mode 100644 dev/integration/data_driven_tests/basic_test.json create mode 100644 dev/integration/integration/README.md create mode 100644 dev/integration/integration/integration/__init__.py create mode 100644 dev/integration/integration/integration/components/__init__.py create mode 100644 dev/integration/integration/integration/components/test_typing_indicator.py create mode 100644 dev/integration/integration/integration/foundational/__init__.py create mode 100644 dev/integration/integration/integration/foundational/_common.py create mode 100644 dev/integration/integration/integration/foundational/test_suite.py create mode 100644 dev/integration/integration/integration/test_quickstart.py create mode 100644 dev/integration/integration/requirements.txt create mode 100644 dev/integration/test_basics.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/asserts/select_activity.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py diff --git a/dev/integration/data_driven_tests/basic_test.json b/dev/integration/data_driven_tests/basic_test.json new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/integration/README.md b/dev/integration/integration/README.md new file mode 100644 index 00000000..614462c9 --- /dev/null +++ b/dev/integration/integration/README.md @@ -0,0 +1,2 @@ +# Microsoft 365 Agents SDK for Python Integration Testing Framework + diff --git a/dev/integration/integration/integration/__init__.py b/dev/integration/integration/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/integration/integration/components/__init__.py b/dev/integration/integration/integration/components/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/integration/integration/components/test_typing_indicator.py b/dev/integration/integration/integration/components/test_typing_indicator.py new file mode 100644 index 00000000..a51613c8 --- /dev/null +++ b/dev/integration/integration/integration/components/test_typing_indicator.py @@ -0,0 +1,36 @@ +import pytest +import asyncio + +from microsoft_agents.activity import ( + Activity, + ActivityTypes, + ChannelAccount +) + +from src.core import integration, IntegrationFixtures, AiohttpEnvironment +from src.samples import QuickstartSample + +@integration(sample=QuickstartSample, environment=AiohttpEnvironment) +class TestTypingIndicator(IntegrationFixtures): + + @pytest.mark.asyncio + async def test_typing_indicator(self, agent_client, response_client): + + activity_base = Activity( + type=ActivityTypes.message, + from_property={"id": "user1", "name": "User 1"}, + recipient={"id": "agent", "name": "Agent"}, + conversation={"id": "conv1"}, + channel_id="test_channel" + ) + + activity_a = activity_base.model_copy() + activity_b = activity_base.model_copy() + + activity_a.from_property = ChannelAccount(id="user1", name="User 1") + activity_b.from_property = ChannelAccount(id="user2", name="User 2") + + await asyncio.gather( + agent_client.send_activity(activity_a), + agent_client.send_activity(activity_b) + ) \ No newline at end of file diff --git a/dev/integration/integration/integration/foundational/__init__.py b/dev/integration/integration/integration/foundational/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/integration/integration/foundational/_common.py b/dev/integration/integration/integration/foundational/_common.py new file mode 100644 index 00000000..91672ffd --- /dev/null +++ b/dev/integration/integration/integration/foundational/_common.py @@ -0,0 +1,10 @@ +import json + +from microsoft_agents.activity import Activity + +def load_activity(channel: str, name: str) -> Activity: + + with open("./dev/integration/src/tests/integration/foundational/activities/{}/{}.json".format(channel, name), "r") as f: + activity = json.load(f) + + return Activity.model_validate(activity) \ No newline at end of file diff --git a/dev/integration/integration/integration/foundational/test_suite.py b/dev/integration/integration/integration/foundational/test_suite.py new file mode 100644 index 00000000..a38edf0b --- /dev/null +++ b/dev/integration/integration/integration/foundational/test_suite.py @@ -0,0 +1,141 @@ +import json +import pytest +import asyncio + +from microsoft_agents.activity import ( + ActivityTypes, +) + +from src.core import integration, IntegrationFixtures, AiohttpEnvironment +from src.samples import QuickstartSample + +from ._common import load_activity + +DIRECTLINE = "directline" + +@integration() +class TestFoundation(IntegrationFixtures): + + def load_activity(self, activity_name) -> Activity: + return load_activity(DIRECTLINE, activity_name) + + @pytest.mark.asyncio + async def test__send_activity__sends_hello_world__returns_hello_world(self, agent_client): + activity = load_activity(DIRECTLINE, "hello_world.json") + result = await agent_client.send_activity(activity) + assert result is not None + last = result[-1] + assert last.type == ActivityTypes.message + assert last.text.lower() == "you said: {activity.text}".lower() + + @pytest.mark.asyncio + async def test__send_invoke__send_basic_invoke_activity__receive_invoke_response(self, agent_client): + activity = load_activity(DIRECTLINE, "basic_invoke.json") + result = await agent_client.send_activity(activity) + assert result + data = json.loads(result) + message = data.get("message", {}) + assert "Invoke received." in message + assert "data" in data + assert data["parameters"] and len(data["parameters"]) > 0 + assert "hi" in data["value"] + + @pytest.mark.asyncio + async def test__send_activity__sends_message_activity_to_ac_submit__return_valid_response(self, agent_client): + activity = load_activity(DIRECTLINE, "ac_submit.json") + result = await agent_client.send_activity(activity) + assert result is not None + last = result[-1] + assert last.type == ActivityTypes.message + assert "doStuff" in last.text + assert "Action.Submit" in last.text + assert "hello" in last.text + + @pytest.mark.asyncio + async def test__send_invoke_sends_invoke_activity_to_ac_execute__returns_valid_adaptive_card_invoke_response(self, agent_client): + activity = load_activity(DIRECTLINE, "ac_execute.json") + result = await agent_client.send_invoke(activity) + + result = json.loads(result) + + assert result.status == 200 + assert result.value + + assert "application/vnd.microsoft.card.adaptive" in result.type + + activity_data = json.loads(activity.value) + assert activity_data.get("action") + user_text = activity_data.get("usertext") + assert user_text in result.value + + @pytest.mark.asyncio + async def test__send_activity_sends_text__returns_poem(self, agent_client): + pass + + @pytest.mark.asyncio + async def test__send_expected_replies_activity__sends_text__returns_poem(self, agent_client): + activity = self.load_activity("expected_replies.json") + result = await agent_client.send_expected_replies(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert "Apollo" in last.text + assert "\n" in last.text + + @pytest.mark.asyncio + async def test__send_invoke__query_link__returns_text(self, agent_client): + activity = self.load_activity("query_link.json") + result = await agent_client.send_invoke(activity) + pass # TODO + + @pytest.mark.asyncio + async def test__send_invoke__select_item__receive_item(self, agent_client): + activity = self.load_activity("select_item.json") + result = await agent_client.send_invoke(activity) + pass # TODO + + @pytest.mark.asyncio + async def test__send_activity__conversation_update__returns_welcome_message(self, agent_client): + activity = self.load_activity("conversation_update.json") + result = await agent_client.send_activity(activity) + last = result[-1] + assert "Hello and Welcome!" in last.text + + @pytest.mark.asyncio + async def test__send_activity__send_heart_message_reaction__returns_message_reaction_heart(self, agent_client): + activity = self.load_activity("message_reaction_heart.json") + result = await agent_client.send_activity(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert "Message Reaction Added: heart" in last.text + + @pytest.mark.asyncio + async def test__send_activity__remove_heart_message_reaction__returns_message_reaction_heart(self, agent_client): + activity = self.load_activity + result = await agent_client.send_activity(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert "Message Reaction Removed: heart" in last.text + + @pytest.mark.asyncio + async def test__send_expected_replies_activity__send_seattle_today_weather__returns_weather(self, agent_client): + activity = self.load_activity("expected_replies_seattle_weather.json") + result = await agent_client.send_expected_replies(activity) + last = result[-1] + assert last.type == ActivityTypes.message + assert last.attachments and len(last.attachments) > 0 + + adaptive_card = last.attachments.first() + assert adaptive_card + assert "application/vnd.microsoft.card.adaptive" == adaptive_card.content_type + assert adaptive_card.content + + assert \ + "�" in adaptive_card.content or \ + "\\u00B0" in adaptive_card.content or \ + f"Missing temperature inside adaptive card: {adaptive_card.content}" in adaptive_card.content + + @pytest.mark.asyncio + async def test__send_activity__simulate_message_loop__expect_question_about_time_and_returns_weather(self, agent_client): + activities = self.load_activity("message_loop_1.json") + fresult = await agent_client.send_activity(activities[0]) + assert \ No newline at end of file diff --git a/dev/integration/integration/integration/test_quickstart.py b/dev/integration/integration/integration/test_quickstart.py new file mode 100644 index 00000000..7738b9b6 --- /dev/null +++ b/dev/integration/integration/integration/test_quickstart.py @@ -0,0 +1,21 @@ +import pytest +import asyncio + +from src.core import IntegrationFixtures, AiohttpEnvironment +from src.samples import QuickstartSample + +class TestQuickstart(Integration): + _sample_cls = QuickstartSample + _environment_cls = AiohttpEnvironment + + @pytest.mark.asyncio + async def test_welcome_message(self, agent_client, response_client): + res = await agent_client.send_expect_replies("hi") + await asyncio.sleep(1) # Wait for processing + responses = await response_client.pop() + + assert len(responses) == 0 + + first_non_typing = next((r for r in res if r.type != "typing"), None) + assert first_non_typing is not None + assert first_non_typing.text == "you said: hi" \ No newline at end of file diff --git a/dev/integration/integration/requirements.txt b/dev/integration/integration/requirements.txt new file mode 100644 index 00000000..d524e63a --- /dev/null +++ b/dev/integration/integration/requirements.txt @@ -0,0 +1 @@ +aioresponses \ No newline at end of file diff --git a/dev/integration/test_basics.py b/dev/integration/test_basics.py new file mode 100644 index 00000000..e9e16d3d --- /dev/null +++ b/dev/integration/test_basics.py @@ -0,0 +1,7 @@ +from microsoft_agents.testing import DataDrivenTester + +class TestBasics(DataDrivenTester): + _input_dir = "./data_driven_tests" + + def test(self): + self._run_data_driven_test("input_file.json") \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/select_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/select_activity.py new file mode 100644 index 00000000..2f396dde --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/select_activity.py @@ -0,0 +1,2 @@ +def select_activity() -> bool: + return False \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py new file mode 100644 index 00000000..3f86a01b --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py @@ -0,0 +1,39 @@ +import json +import asyncio + +from microsoft_agents.activity import Activity +from microsoft_agents.testing.core import ( + AgentClient, + ResponseClient +) + +from microsoft_agents.testing.asserts import assert_activity + +class DataDrivenTestModule: + + def __init__( + self, + test_file_path: str, + agent_client: AgentClient, + response_client: ResponseClient + ) -> None: + + data = json.load(open(test_file_path, "r")) + + self._agent_client = agent_client + self._response_client = response_client + + self._input_activities: list[Activity] = [] + self._activity_assertions: list[dict] = [] + + async def run_test(self) -> bool: + + for input_activity in self._input_activities: + await self._agent_client.send_activity(input_activity) + + await asyncio.sleep(1) + + for activity_assertion in self._activity_assertions: + # select first + selected_activity = None + assert_activity(selected_activity, activity_assertion) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py new file mode 100644 index 00000000..3e26b3f4 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py @@ -0,0 +1,33 @@ +from typing import Callable, TypeVar + +from .core import Integration + +async def run_data_driven_test(input_file: str) -> None: + """ + Run data-driven tests based on the provided input file. + """ + pass + +T = TypeVar("T", bound=Integration) + +def factory(tests_path: str = "./") -> Callable[T, T]: + + # for file in file + + files = [] + + def decorator(test_cls: T) -> T: + + for file_name in files: + + test_case_name = f"test_data_driven__{file_name}" + + def func(self, agent_client, response_client) -> None: + + + + setattr(test_cls, test_case_name, func) + + return test_cls + + return decorator \ No newline at end of file From b8f3304a43b17b62362f3760c67e4a73c9e166b2 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 15:16:57 -0800 Subject: [PATCH 57/81] Separating assertions from checkers --- .../{asserts => assertions}/__init__.py | 0 .../testing/assertions/assertions.py | 24 +++++++++++++++++++ .../check_activity.py} | 2 +- .../{asserts => assertions}/check_field.py | 0 .../select_activity.py | 0 .../{asserts => assertions}/type_defs.py | 0 .../testing/integration/data_driven_test.py | 2 +- 7 files changed, 26 insertions(+), 2 deletions(-) rename dev/microsoft-agents-testing/microsoft_agents/testing/{asserts => assertions}/__init__.py (100%) create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py rename dev/microsoft-agents-testing/microsoft_agents/testing/{asserts/assert_activity.py => assertions/check_activity.py} (92%) rename dev/microsoft-agents-testing/microsoft_agents/testing/{asserts => assertions}/check_field.py (100%) rename dev/microsoft-agents-testing/microsoft_agents/testing/{asserts => assertions}/select_activity.py (100%) rename dev/microsoft-agents-testing/microsoft_agents/testing/{asserts => assertions}/type_defs.py (100%) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/asserts/__init__.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py new file mode 100644 index 00000000..a4fbdc5e --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -0,0 +1,24 @@ +from typing import Any + +from microsoft_agents.activity import Activity + +from .type_defs import FieldAssertionType +from .check_activity import check_activity +from .check_field import check_field + +def assert_activity(activity: Activity, baseline: Activity | dict) -> None: + """Asserts that the given activity matches the baseline activity. + + :param activity: The activity to be tested. + :param baseline: The baseline activity or a dictionary representing the expected activity data. + """ + assert check_activity(activity, baseline) + +def assert_field(actual_value: Any, baseline_value: Any, assertion_type: FieldAssertionType) -> None: + """Asserts that a specific field in the target matches the baseline. + + :param key_in_baseline: The key of the field to be tested. + :param target: The target dictionary containing the actual values. + :param baseline: The baseline dictionary containing the expected values. + """ + assert check_field(actual_value, baseline_value, assertion_type) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py similarity index 92% rename from dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index 4edca756..a9b97554 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/assert_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -5,7 +5,7 @@ from .type_defs import _UNSET_FIELD, FieldAssertionType -def assert_activity(activity: Activity, baseline: Activity | dict) -> None: +def check_activity(activity: Activity, baseline: Activity | dict) -> None: """Asserts that the given activity matches the baseline activity. :param activity: The activity to be tested. diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/asserts/check_field.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/select_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/asserts/select_activity.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/asserts/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py similarity index 100% rename from dev/microsoft-agents-testing/microsoft_agents/testing/asserts/type_defs.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py index 3f86a01b..daee68b2 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py @@ -26,7 +26,7 @@ def __init__( self._input_activities: list[Activity] = [] self._activity_assertions: list[dict] = [] - async def run_test(self) -> bool: + async def assert(self) -> bool: for input_activity in self._input_activities: await self._agent_client.send_activity(input_activity) From 42b4ea1f8333de41d804ae3aac3aecc1b2e05fea Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 6 Nov 2025 15:22:06 -0800 Subject: [PATCH 58/81] Adjusting return values for check_activity with refactor --- .../microsoft_agents/testing/assertions/check_activity.py | 7 +++++-- .../microsoft_agents/testing/assertions/check_field.py | 2 +- .../testing/integration/data_driven_test.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index a9b97554..cb0a616a 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -5,7 +5,7 @@ from .type_defs import _UNSET_FIELD, FieldAssertionType -def check_activity(activity: Activity, baseline: Activity | dict) -> None: +def check_activity(activity: Activity, baseline: Activity | dict) -> bool: """Asserts that the given activity matches the baseline activity. :param activity: The activity to be tested. @@ -26,4 +26,7 @@ def check_activity(activity: Activity, baseline: Activity | dict) -> None: target_value = getattr(activity, key, _UNSET_FIELD) - assert check_field(baseline_value, target_value, assertion_type) + if not check_field(baseline_value, target_value, assertion_type): + return False + + return True diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index c958cf61..e5c0e13d 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -17,7 +17,7 @@ def check_field( actual_value: Any, baseline_value: Any, assertion_type: FieldAssertionType ) -> bool: - + operation = _OPERATIONS.get(assertion_type) if not operation: return False # missing operation for the assertion type diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py index daee68b2..46a59c03 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py @@ -7,7 +7,7 @@ ResponseClient ) -from microsoft_agents.testing.asserts import assert_activity +from microsoft_agents.testing.asserts import check_activity class DataDrivenTestModule: @@ -36,4 +36,4 @@ async def assert(self) -> bool: for activity_assertion in self._activity_assertions: # select first selected_activity = None - assert_activity(selected_activity, activity_assertion) \ No newline at end of file + check_activity(selected_activity, activity_assertion) \ No newline at end of file From 2f79824008c2e40a3ddf5ca173856ca97b4847ce Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Wed, 12 Nov 2025 08:21:11 -0800 Subject: [PATCH 59/81] Fixing package name in pyproject.toml --- dev/microsoft-agents-testing/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/microsoft-agents-testing/pyproject.toml b/dev/microsoft-agents-testing/pyproject.toml index cf659e6f..5557ac38 100644 --- a/dev/microsoft-agents-testing/pyproject.toml +++ b/dev/microsoft-agents-testing/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "microsoft-agents-hosting-core" +name = "microsoft-agents-testing" dynamic = ["version", "dependencies"] description = "Core library for Microsoft Agents" readme = {file = "README.md", content-type = "text/markdown"} From cfded79e0d4f25799a02486055cca32b2623f304 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Wed, 12 Nov 2025 09:17:44 -0800 Subject: [PATCH 60/81] Assertion test cases --- .../components/test_typing_indicator.py | 36 ---------- .../integration/test_quickstart.py | 32 ++++----- .../testing/assertions/__init__.py | 15 ++++ .../testing/assertions/check_activity.py | 44 +++++++++--- .../testing/assertions/check_field.py | 4 +- .../testing/assertions/type_defs.py | 16 ++--- .../testing/integration/data_driven_test.py | 56 +++++++-------- .../testing/integration/data_driven_tester.py | 36 +++++----- dev/microsoft-agents-testing/pytest.ini | 39 +++++++++++ .../tests/assertions}/__init__.py | 0 .../tests/assertions/_common.py | 14 ++++ .../tests/assertions/test_check_activity.py | 70 +++++++++++++++++++ .../tests/assertions/test_check_field.py | 0 .../tests/assertions/test_select_activity.py | 0 .../integration/test_data_driven_tester.py | 0 15 files changed, 244 insertions(+), 118 deletions(-) delete mode 100644 dev/integration/integration/integration/components/test_typing_indicator.py create mode 100644 dev/microsoft-agents-testing/pytest.ini rename dev/{integration/integration/integration/components => microsoft-agents-testing/tests/assertions}/__init__.py (100%) create mode 100644 dev/microsoft-agents-testing/tests/assertions/_common.py create mode 100644 dev/microsoft-agents-testing/tests/assertions/test_check_activity.py create mode 100644 dev/microsoft-agents-testing/tests/assertions/test_check_field.py create mode 100644 dev/microsoft-agents-testing/tests/assertions/test_select_activity.py create mode 100644 dev/microsoft-agents-testing/tests/integration/test_data_driven_tester.py diff --git a/dev/integration/integration/integration/components/test_typing_indicator.py b/dev/integration/integration/integration/components/test_typing_indicator.py deleted file mode 100644 index a51613c8..00000000 --- a/dev/integration/integration/integration/components/test_typing_indicator.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest -import asyncio - -from microsoft_agents.activity import ( - Activity, - ActivityTypes, - ChannelAccount -) - -from src.core import integration, IntegrationFixtures, AiohttpEnvironment -from src.samples import QuickstartSample - -@integration(sample=QuickstartSample, environment=AiohttpEnvironment) -class TestTypingIndicator(IntegrationFixtures): - - @pytest.mark.asyncio - async def test_typing_indicator(self, agent_client, response_client): - - activity_base = Activity( - type=ActivityTypes.message, - from_property={"id": "user1", "name": "User 1"}, - recipient={"id": "agent", "name": "Agent"}, - conversation={"id": "conv1"}, - channel_id="test_channel" - ) - - activity_a = activity_base.model_copy() - activity_b = activity_base.model_copy() - - activity_a.from_property = ChannelAccount(id="user1", name="User 1") - activity_b.from_property = ChannelAccount(id="user2", name="User 2") - - await asyncio.gather( - agent_client.send_activity(activity_a), - agent_client.send_activity(activity_b) - ) \ No newline at end of file diff --git a/dev/integration/integration/integration/test_quickstart.py b/dev/integration/integration/integration/test_quickstart.py index 7738b9b6..7f37a336 100644 --- a/dev/integration/integration/integration/test_quickstart.py +++ b/dev/integration/integration/integration/test_quickstart.py @@ -1,21 +1,21 @@ -import pytest -import asyncio +# import pytest +# import asyncio -from src.core import IntegrationFixtures, AiohttpEnvironment -from src.samples import QuickstartSample +# from src.core import IntegrationFixtures, AiohttpEnvironment +# from src.samples import QuickstartSample -class TestQuickstart(Integration): - _sample_cls = QuickstartSample - _environment_cls = AiohttpEnvironment +# class TestQuickstart(Integration): +# _sample_cls = QuickstartSample +# _environment_cls = AiohttpEnvironment - @pytest.mark.asyncio - async def test_welcome_message(self, agent_client, response_client): - res = await agent_client.send_expect_replies("hi") - await asyncio.sleep(1) # Wait for processing - responses = await response_client.pop() +# @pytest.mark.asyncio +# async def test_welcome_message(self, agent_client, response_client): +# res = await agent_client.send_expect_replies("hi") +# await asyncio.sleep(1) # Wait for processing +# responses = await response_client.pop() - assert len(responses) == 0 +# assert len(responses) == 0 - first_non_typing = next((r for r in res if r.type != "typing"), None) - assert first_non_typing is not None - assert first_non_typing.text == "you said: hi" \ No newline at end of file +# first_non_typing = next((r for r in res if r.type != "typing"), None) +# assert first_non_typing is not None +# assert first_non_typing.text == "you said: hi" \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py index e69de29b..95f4bfc9 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py @@ -0,0 +1,15 @@ +from .assertions import ( + assert_activity, + assert_field, +) +from .check_activity import check_activity +from .check_field import check_field +from .type_defs import FieldAssertionType + +__all__ = [ + "assert_activity", + "assert_field", + "check_activity", + "check_field", + "FieldAssertionType", +] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index cb0a616a..852d5cdd 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -1,3 +1,5 @@ +from typing import Any + from microsoft_agents.activity import Activity from microsoft_agents.testing.utils import normalize_activity_data @@ -5,6 +7,35 @@ from .type_defs import _UNSET_FIELD, FieldAssertionType +def _parse_assertion(assertion_info: Any) -> tuple[Any, FieldAssertionType]: + """Parses the assertion information and returns the assertion type and baseline value. + + :param assertion_info: The assertion information to be parsed. + :return: A tuple containing the assertion type and baseline value. + """ + + assertion_type = FieldAssertionType.EQUALS + + if isinstance(assertion_info, dict) and "assertion_type" in assertion_info: + # format: + # {"assertion_type": "__EQ__", "assertion": "value"} + assertion_type = assertion_info["assertion_type"] + assertion = assertion_info.get("assertion") + + elif isinstance(assertion_info, list) and \ + len(assertion_info) >= 1 and \ + isinstance(assertion_info[0], str) and \ + assertion_info[0] in FieldAssertionType.__members__: + # format: + # ["__EQ__", "assertion"] + assertion_type = FieldAssertionType[assertion_info[0]] + assertion = assertion_info[1] if len(assertion_info) > 1 else None + else: + # default format: direct value + assertion = assertion_info + + return assertion, assertion_type + def check_activity(activity: Activity, baseline: Activity | dict) -> bool: """Asserts that the given activity matches the baseline activity. @@ -15,18 +46,11 @@ def check_activity(activity: Activity, baseline: Activity | dict) -> bool: baseline = normalize_activity_data(baseline) for key in baseline.keys(): - - assertion_type = FieldAssertionType.EQUALS - assertion_info = baseline[key] - if isinstance(assertion_info, dict) and "assertion_type" in assertion_info: - assertion_type = assertion_info["assertion_type"] - baseline_value = assertion_info["value"] - else: - baseline_value = assertion_info - + # support for different assertion formats + assertion, assertion_type = _parse_assertion(baseline[key]) target_value = getattr(activity, key, _UNSET_FIELD) - if not check_field(baseline_value, target_value, assertion_type): + if not check_field(target_value, assertion, assertion_type): return False return True diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index e5c0e13d..2f7e3273 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -15,10 +15,10 @@ def check_field( - actual_value: Any, baseline_value: Any, assertion_type: FieldAssertionType + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType ) -> bool: operation = _OPERATIONS.get(assertion_type) if not operation: return False # missing operation for the assertion type - return operation(actual_value, baseline_value) + return operation(actual_value, assertion) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py index d8f6dfa0..bd1eb3b7 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py @@ -3,13 +3,13 @@ _UNSET_FIELD = object() -class FieldAssertionType(Enum): +class FieldAssertionType(str, Enum): """Defines the types of assertions that can be made on fields.""" - EQUALS = "equals" - NOT_EQUALS = "not_equals" - GREATER_THAN = "greater_than" - LESS_THAN = "less_than" - CONTAINS = "contains" - NOT_CONTAINS = "not_contains" - RE_MATCH = "re_match" + EQUALS = "__EQ__" + NOT_EQUALS = "__NEQ__" + GREATER_THAN = "__GT__" + LESS_THAN = "__LT__" + CONTAINS = "__IN__" + NOT_CONTAINS = "__NIN__" + RE_MATCH = "__RE_MATCH__" diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py index 46a59c03..7267dcf0 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py @@ -1,39 +1,39 @@ -import json -import asyncio +# import json +# import asyncio -from microsoft_agents.activity import Activity -from microsoft_agents.testing.core import ( - AgentClient, - ResponseClient -) +# from microsoft_agents.activity import Activity +# from microsoft_agents.testing.core import ( +# AgentClient, +# ResponseClient +# ) -from microsoft_agents.testing.asserts import check_activity +# from microsoft_agents.testing.asserts import check_activity -class DataDrivenTestModule: +# class DataDrivenTestModule: - def __init__( - self, - test_file_path: str, - agent_client: AgentClient, - response_client: ResponseClient - ) -> None: +# def __init__( +# self, +# test_file_path: str, +# agent_client: AgentClient, +# response_client: ResponseClient +# ) -> None: - data = json.load(open(test_file_path, "r")) +# data = json.load(open(test_file_path, "r")) - self._agent_client = agent_client - self._response_client = response_client +# self._agent_client = agent_client +# self._response_client = response_client - self._input_activities: list[Activity] = [] - self._activity_assertions: list[dict] = [] +# self._input_activities: list[Activity] = [] +# self._activity_assertions: list[dict] = [] - async def assert(self) -> bool: +# async def assert(self) -> bool: - for input_activity in self._input_activities: - await self._agent_client.send_activity(input_activity) +# for input_activity in self._input_activities: +# await self._agent_client.send_activity(input_activity) - await asyncio.sleep(1) +# await asyncio.sleep(1) - for activity_assertion in self._activity_assertions: - # select first - selected_activity = None - check_activity(selected_activity, activity_assertion) \ No newline at end of file +# for activity_assertion in self._activity_assertions: +# # select first +# selected_activity = None +# check_activity(selected_activity, activity_assertion) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py index 3e26b3f4..c57df5c5 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py @@ -1,33 +1,33 @@ -from typing import Callable, TypeVar +# from typing import Callable, TypeVar -from .core import Integration +# from .core import Integration -async def run_data_driven_test(input_file: str) -> None: - """ - Run data-driven tests based on the provided input file. - """ - pass +# async def run_data_driven_test(input_file: str) -> None: +# """ +# Run data-driven tests based on the provided input file. +# """ +# pass -T = TypeVar("T", bound=Integration) +# T = TypeVar("T", bound=Integration) -def factory(tests_path: str = "./") -> Callable[T, T]: +# def factory(tests_path: str = "./") -> Callable[T, T]: - # for file in file +# # for file in file - files = [] +# files = [] - def decorator(test_cls: T) -> T: +# def decorator(test_cls: T) -> T: - for file_name in files: +# for file_name in files: - test_case_name = f"test_data_driven__{file_name}" +# test_case_name = f"test_data_driven__{file_name}" - def func(self, agent_client, response_client) -> None: +# def func(self, agent_client, response_client) -> None: - setattr(test_cls, test_case_name, func) +# setattr(test_cls, test_case_name, func) - return test_cls +# return test_cls - return decorator \ No newline at end of file +# return decorator \ No newline at end of file diff --git a/dev/microsoft-agents-testing/pytest.ini b/dev/microsoft-agents-testing/pytest.ini new file mode 100644 index 00000000..479894a8 --- /dev/null +++ b/dev/microsoft-agents-testing/pytest.ini @@ -0,0 +1,39 @@ +[pytest] +# Pytest configuration for Microsoft Agents for Python + +# Treat all warnings as errors by default +# This ensures that any code generating warnings will fail tests, +# promoting cleaner code and early detection of issues +filterwarnings = + error + # Ignore specific warnings that are not actionable or are from dependencies + ignore::DeprecationWarning:pkg_resources.* + ignore::DeprecationWarning:setuptools.* + ignore::PendingDeprecationWarning + # pytest-asyncio warnings that are safe to ignore + ignore:.*deprecated.*asyncio.*:DeprecationWarning:pytest_asyncio.* + +# Test discovery configuration +testpaths = tests +python_files = test_*.py *_test.py +python_classes = Test* +python_functions = test_* + +# Output configuration +addopts = + --strict-markers + --strict-config + --verbose + --tb=short + --durations=10 + +# Minimum version requirement +minversion = 6.0 + +# Markers for test categorization +markers = + unit: Unit tests + integration: Integration tests + slow: Slow tests that may take longer to run + requires_network: Tests that require network access + requires_auth: Tests that require authentication \ No newline at end of file diff --git a/dev/integration/integration/integration/components/__init__.py b/dev/microsoft-agents-testing/tests/assertions/__init__.py similarity index 100% rename from dev/integration/integration/integration/components/__init__.py rename to dev/microsoft-agents-testing/tests/assertions/__init__.py diff --git a/dev/microsoft-agents-testing/tests/assertions/_common.py b/dev/microsoft-agents-testing/tests/assertions/_common.py new file mode 100644 index 00000000..4049b90a --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/_common.py @@ -0,0 +1,14 @@ +import pytest + +from microsoft_agents.activity import Activity + +@pytest.fixture +def activity(): + return Activity(type="message", text="Hello, World!") + +@pytest.fixture(params=[ + Activity(type="message", text="Hello, World!"), + {"type": "message", "text": "Hello, World!"} +]) +def baseline(request): + return request.param \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py new file mode 100644 index 00000000..78609621 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py @@ -0,0 +1,70 @@ +import pytest + +from microsoft_agents.testing.assertions.type_defs import FieldAssertionType +from microsoft_agents.testing.assertions.check_activity import _parse_assertion + +class TestParseAssertion: + + @pytest.fixture( + params=[ + FieldAssertionType.EQUALS.value, + FieldAssertionType.NOT_EQUALS.value, + FieldAssertionType.GREATER_THAN.value, + ] + ) + def assertion_type_str(self, request): + return request.param + + @pytest.fixture( + params=[ + "simple_value", + {"key": "value"}, + 42 + ] + ) + def assertion_value(self, request): + return request.param + + def test_parse_assertion_dict(self, assertion_value, assertion_type_str): + + assertion, assertion_type = _parse_assertion( + {"assertion_type": assertion_type_str, "assertion": assertion_value} + ) + assert assertion == assertion_value + assert assertion_type.name == FieldAssertionType(assertion_type_str).name + + def test_parse_assertion_dict_complex(): + + assertion, assertion_type = _parse_assertion( + {"assertion_type": "__EQ__", "assertion": {"key": "value"}} + ) + + assert assertion == {"key": "value"} + assert assertion_type.name == "__EQ__" + + def test_parse_assertion_dict_no_value(): + + assertion, assertion_type = _parse_assertion( + {"assertion_type": "__NEQ__"} + ) + + assert assertion is None + assert assertion_type.name == "__NEQ__" + + def test_parse_assertion_dict_extra_value(): + + assertion, assertion_type = _parse_assertion( + {"assertion_type": "__RE_MATCH__", "key": "value"} + ) + + assert assertion is None + assert assertion_type.name == "__RE_MATCH__" + + def test_parse_assertion_list(): + + assertion, assertion_type = _parse_assertion( + ["__GT__", 10] + ) + + assert assertion == 10 + assert assertion_type.name == "__GT__" \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/assertions/test_select_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_select_activity.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/integration/test_data_driven_tester.py b/dev/microsoft-agents-testing/tests/integration/test_data_driven_tester.py new file mode 100644 index 00000000..e69de29b From b6618b2e09f7c7a9dffcd5a62846c45fcd369de4 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Wed, 12 Nov 2025 10:15:22 -0800 Subject: [PATCH 61/81] _parse_assertion unit tests --- .../integration/foundational/_common.py | 10 ++- .../integration/test_quickstart.py | 2 +- dev/integration/test_basics.py | 3 +- .../testing/assertions/__init__.py | 2 +- .../testing/assertions/assertions.py | 8 ++- .../testing/assertions/check_activity.py | 21 ++++-- .../testing/assertions/check_field.py | 2 +- .../testing/assertions/select_activity.py | 2 +- .../testing/assertions/type_defs.py | 16 +++-- .../testing/integration/data_driven_test.py | 4 +- .../testing/integration/data_driven_tester.py | 7 +- .../tests/assertions/_common.py | 14 ++-- .../tests/assertions/test_check_activity.py | 68 +++++++------------ 13 files changed, 82 insertions(+), 77 deletions(-) diff --git a/dev/integration/integration/integration/foundational/_common.py b/dev/integration/integration/integration/foundational/_common.py index 91672ffd..f4d22304 100644 --- a/dev/integration/integration/integration/foundational/_common.py +++ b/dev/integration/integration/integration/foundational/_common.py @@ -2,9 +2,15 @@ from microsoft_agents.activity import Activity + def load_activity(channel: str, name: str) -> Activity: - with open("./dev/integration/src/tests/integration/foundational/activities/{}/{}.json".format(channel, name), "r") as f: + with open( + "./dev/integration/src/tests/integration/foundational/activities/{}/{}.json".format( + channel, name + ), + "r", + ) as f: activity = json.load(f) - return Activity.model_validate(activity) \ No newline at end of file + return Activity.model_validate(activity) diff --git a/dev/integration/integration/integration/test_quickstart.py b/dev/integration/integration/integration/test_quickstart.py index 7f37a336..967d4ebc 100644 --- a/dev/integration/integration/integration/test_quickstart.py +++ b/dev/integration/integration/integration/test_quickstart.py @@ -18,4 +18,4 @@ # first_non_typing = next((r for r in res if r.type != "typing"), None) # assert first_non_typing is not None -# assert first_non_typing.text == "you said: hi" \ No newline at end of file +# assert first_non_typing.text == "you said: hi" diff --git a/dev/integration/test_basics.py b/dev/integration/test_basics.py index e9e16d3d..28c2244e 100644 --- a/dev/integration/test_basics.py +++ b/dev/integration/test_basics.py @@ -1,7 +1,8 @@ from microsoft_agents.testing import DataDrivenTester + class TestBasics(DataDrivenTester): _input_dir = "./data_driven_tests" def test(self): - self._run_data_driven_test("input_file.json") \ No newline at end of file + self._run_data_driven_test("input_file.json") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py index 95f4bfc9..e36bff9f 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py @@ -12,4 +12,4 @@ "check_activity", "check_field", "FieldAssertionType", -] \ No newline at end of file +] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py index a4fbdc5e..92b9786a 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -6,6 +6,7 @@ from .check_activity import check_activity from .check_field import check_field + def assert_activity(activity: Activity, baseline: Activity | dict) -> None: """Asserts that the given activity matches the baseline activity. @@ -14,11 +15,14 @@ def assert_activity(activity: Activity, baseline: Activity | dict) -> None: """ assert check_activity(activity, baseline) -def assert_field(actual_value: Any, baseline_value: Any, assertion_type: FieldAssertionType) -> None: + +def assert_field( + actual_value: Any, baseline_value: Any, assertion_type: FieldAssertionType +) -> None: """Asserts that a specific field in the target matches the baseline. :param key_in_baseline: The key of the field to be tested. :param target: The target dictionary containing the actual values. :param baseline: The baseline dictionary containing the expected values. """ - assert check_field(actual_value, baseline_value, assertion_type) \ No newline at end of file + assert check_field(actual_value, baseline_value, assertion_type) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index 852d5cdd..e0ff0d2b 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -16,26 +16,33 @@ def _parse_assertion(assertion_info: Any) -> tuple[Any, FieldAssertionType]: assertion_type = FieldAssertionType.EQUALS - if isinstance(assertion_info, dict) and "assertion_type" in assertion_info: + if ( + isinstance(assertion_info, dict) + and "assertion_type" in assertion_info + and "assertion" in assertion_info + ): # format: # {"assertion_type": "__EQ__", "assertion": "value"} assertion_type = assertion_info["assertion_type"] assertion = assertion_info.get("assertion") - elif isinstance(assertion_info, list) and \ - len(assertion_info) >= 1 and \ - isinstance(assertion_info[0], str) and \ - assertion_info[0] in FieldAssertionType.__members__: + elif ( + isinstance(assertion_info, list) + and len(assertion_info) >= 2 + and isinstance(assertion_info[0], str) + and assertion_info[0] in FieldAssertionType.__members__ + ): # format: # ["__EQ__", "assertion"] assertion_type = FieldAssertionType[assertion_info[0]] - assertion = assertion_info[1] if len(assertion_info) > 1 else None + assertion = assertion_info[1] else: # default format: direct value assertion = assertion_info return assertion, assertion_type + def check_activity(activity: Activity, baseline: Activity | dict) -> bool: """Asserts that the given activity matches the baseline activity. @@ -52,5 +59,5 @@ def check_activity(activity: Activity, baseline: Activity | dict) -> bool: if not check_field(target_value, assertion, assertion_type): return False - + return True diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index 2f7e3273..a771d336 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -17,7 +17,7 @@ def check_field( actual_value: Any, assertion: Any, assertion_type: FieldAssertionType ) -> bool: - + operation = _OPERATIONS.get(assertion_type) if not operation: return False # missing operation for the assertion type diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py index 2f396dde..85a2b03b 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py @@ -1,2 +1,2 @@ def select_activity() -> bool: - return False \ No newline at end of file + return False diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py index bd1eb3b7..1954fb35 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py @@ -6,10 +6,12 @@ class FieldAssertionType(str, Enum): """Defines the types of assertions that can be made on fields.""" - EQUALS = "__EQ__" - NOT_EQUALS = "__NEQ__" - GREATER_THAN = "__GT__" - LESS_THAN = "__LT__" - CONTAINS = "__IN__" - NOT_CONTAINS = "__NIN__" - RE_MATCH = "__RE_MATCH__" + EQUALS = "EQUALS" + NOT_EQUALS = "NOT_EQUALS" + GREATER_THAN = "GREATER_THAN" + LESS_THAN = "LESS_THAN" + CONTAINS = "CONTAINS" + NOT_CONTAINS = "NOT_CONTAINS" + IN = "IN" + NOT_IN = "NOT_IN" + RE_MATCH = "RE_MATCH" diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py index 7267dcf0..0e13db4a 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py @@ -17,7 +17,7 @@ # agent_client: AgentClient, # response_client: ResponseClient # ) -> None: - + # data = json.load(open(test_file_path, "r")) # self._agent_client = agent_client @@ -36,4 +36,4 @@ # for activity_assertion in self._activity_assertions: # # select first # selected_activity = None -# check_activity(selected_activity, activity_assertion) \ No newline at end of file +# check_activity(selected_activity, activity_assertion) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py index c57df5c5..af36f9d8 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py @@ -23,11 +23,10 @@ # test_case_name = f"test_data_driven__{file_name}" # def func(self, agent_client, response_client) -> None: - - + # setattr(test_cls, test_case_name, func) # return test_cls - -# return decorator \ No newline at end of file + +# return decorator diff --git a/dev/microsoft-agents-testing/tests/assertions/_common.py b/dev/microsoft-agents-testing/tests/assertions/_common.py index 4049b90a..0a69960c 100644 --- a/dev/microsoft-agents-testing/tests/assertions/_common.py +++ b/dev/microsoft-agents-testing/tests/assertions/_common.py @@ -2,13 +2,17 @@ from microsoft_agents.activity import Activity + @pytest.fixture def activity(): return Activity(type="message", text="Hello, World!") -@pytest.fixture(params=[ - Activity(type="message", text="Hello, World!"), - {"type": "message", "text": "Hello, World!"} -]) + +@pytest.fixture( + params=[ + Activity(type="message", text="Hello, World!"), + {"type": "message", "text": "Hello, World!"}, + ] +) def baseline(request): - return request.param \ No newline at end of file + return request.param diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py index 78609621..250a379c 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py @@ -3,6 +3,7 @@ from microsoft_agents.testing.assertions.type_defs import FieldAssertionType from microsoft_agents.testing.assertions.check_activity import _parse_assertion + class TestParseAssertion: @pytest.fixture( @@ -15,56 +16,37 @@ class TestParseAssertion: def assertion_type_str(self, request): return request.param - @pytest.fixture( - params=[ - "simple_value", - {"key": "value"}, - 42 - ] - ) + @pytest.fixture(params=["simple_value", {"key": "value"}, 42]) def assertion_value(self, request): return request.param - + def test_parse_assertion_dict(self, assertion_value, assertion_type_str): - + assertion, assertion_type = _parse_assertion( {"assertion_type": assertion_type_str, "assertion": assertion_value} ) assert assertion == assertion_value - assert assertion_type.name == FieldAssertionType(assertion_type_str).name - - def test_parse_assertion_dict_complex(): - - assertion, assertion_type = _parse_assertion( - {"assertion_type": "__EQ__", "assertion": {"key": "value"}} - ) - - assert assertion == {"key": "value"} - assert assertion_type.name == "__EQ__" + assert assertion_type.name == FieldAssertionType(assertion_type_str).name - def test_parse_assertion_dict_no_value(): - + def test_parse_assertion_list(self, assertion_value, assertion_type_str): assertion, assertion_type = _parse_assertion( - {"assertion_type": "__NEQ__"} + [assertion_type_str, assertion_value] ) - - assert assertion is None - assert assertion_type.name == "__NEQ__" - - def test_parse_assertion_dict_extra_value(): - - assertion, assertion_type = _parse_assertion( - {"assertion_type": "__RE_MATCH__", "key": "value"} - ) - - assert assertion is None - assert assertion_type.name == "__RE_MATCH__" - - def test_parse_assertion_list(): - - assertion, assertion_type = _parse_assertion( - ["__GT__", 10] - ) - - assert assertion == 10 - assert assertion_type.name == "__GT__" \ No newline at end of file + assert assertion == assertion_value + assert assertion_type.name == assertion_type_str + + @pytest.mark.parametrize( + "field", + [ + "value", + {"assertion_type": FieldAssertionType.IN}, + {"assertion_type": FieldAssertionType.IN, "key": "value"}, + [FieldAssertionType.RE_MATCH], + [], + {"assertion_type": "invalid", "assertion": "test"}, + ], + ) + def test_parse_assertion_default(self, field): + assertion, assertion_type = _parse_assertion(field) + assert assertion == field + assert assertion_type == FieldAssertionType.EQUALS From 1e8f19ea9aa14a5ad4dbd7e38c3cd423380f0763 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Wed, 12 Nov 2025 11:33:44 -0800 Subject: [PATCH 62/81] More assertion tests and fixes to data normalization logic --- .../testing/assertions/check_activity.py | 75 ++-- .../testing/assertions/check_field.py | 7 + .../testing/assertions/type_defs.py | 6 +- .../microsoft_agents/testing/utils/misc.py | 2 +- .../tests/assertions/test_check_activity.py | 368 +++++++++++++++++- .../tests/assertions/test_check_field.py | 214 ++++++++++ 6 files changed, 633 insertions(+), 39 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index e0ff0d2b..17c9ef12 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -1,13 +1,15 @@ -from typing import Any +from typing import Any, TypeVar, Optional + +from pydantic import BaseModel from microsoft_agents.activity import Activity from microsoft_agents.testing.utils import normalize_activity_data from .check_field import check_field -from .type_defs import _UNSET_FIELD, FieldAssertionType +from .type_defs import UNSET_FIELD, FieldAssertionType -def _parse_assertion(assertion_info: Any) -> tuple[Any, FieldAssertionType]: +def _parse_assertion(field: Any) -> tuple[Any, Optional[FieldAssertionType]]: """Parses the assertion information and returns the assertion type and baseline value. :param assertion_info: The assertion information to be parsed. @@ -15,33 +17,63 @@ def _parse_assertion(assertion_info: Any) -> tuple[Any, FieldAssertionType]: """ assertion_type = FieldAssertionType.EQUALS + assertion = None if ( - isinstance(assertion_info, dict) - and "assertion_type" in assertion_info - and "assertion" in assertion_info + isinstance(field, dict) + and "assertion_type" in field + and "assertion" in field + and field["assertion_type"] in FieldAssertionType.__members__ ): # format: # {"assertion_type": "__EQ__", "assertion": "value"} - assertion_type = assertion_info["assertion_type"] - assertion = assertion_info.get("assertion") + assertion_type = field["assertion_type"] + assertion = field.get("assertion") elif ( - isinstance(assertion_info, list) - and len(assertion_info) >= 2 - and isinstance(assertion_info[0], str) - and assertion_info[0] in FieldAssertionType.__members__ + isinstance(field, list) + and len(field) >= 2 + and isinstance(field[0], str) + and field[0] in FieldAssertionType.__members__ ): # format: # ["__EQ__", "assertion"] - assertion_type = FieldAssertionType[assertion_info[0]] - assertion = assertion_info[1] + assertion_type = FieldAssertionType[field[0]] + assertion = field[1] + elif isinstance(field, list) or isinstance(field, dict): + assertion_type = None else: # default format: direct value - assertion = assertion_info + assertion = field return assertion, assertion_type +def _check(actual: Any, baseline: Any) -> bool: + + assertion, assertion_type = _parse_assertion(baseline) + + if assertion_type is None: + if isinstance(baseline, dict): + for key in baseline: + new_actual = actual.get(key, UNSET_FIELD) + new_baseline = baseline[key] + if not _check(new_actual, new_baseline): + return False + + elif isinstance(baseline, list): + for index, item in enumerate(baseline): + new_actual = actual[index] if index < len(actual) else UNSET_FIELD + new_baseline = item + if not _check(new_actual, new_baseline): + return False + else: + raise ValueError("Unsupported baseline type for complex assertion.") + else: + assert assertion_type + assertion, assertion_type = _parse_assertion(baseline) + if not check_field(actual, assertion, assertion_type): + return False + def check_activity(activity: Activity, baseline: Activity | dict) -> bool: """Asserts that the given activity matches the baseline activity. @@ -49,15 +81,6 @@ def check_activity(activity: Activity, baseline: Activity | dict) -> bool: :param activity: The activity to be tested. :param baseline: The baseline activity or a dictionary representing the expected activity data. """ - + actual_activity = normalize_activity_data(activity) baseline = normalize_activity_data(baseline) - - for key in baseline.keys(): - # support for different assertion formats - assertion, assertion_type = _parse_assertion(baseline[key]) - target_value = getattr(activity, key, _UNSET_FIELD) - - if not check_field(target_value, assertion, assertion_type): - return False - - return True + return _check(actual_activity, baseline) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index a771d336..9ed52f0e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -17,6 +17,13 @@ def check_field( actual_value: Any, assertion: Any, assertion_type: FieldAssertionType ) -> bool: + """Checks if the actual value satisfies the given assertion based on the assertion type. + + :param actual_value: The value to be checked. + :param assertion: The expected value or pattern to check against. + :param assertion_type: The type of assertion to perform. + :return: True if the assertion is satisfied, False otherwise. + """ operation = _OPERATIONS.get(assertion_type) if not operation: diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py index 1954fb35..362e9a27 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py @@ -1,7 +1,9 @@ from enum import Enum -_UNSET_FIELD = object() - +class UNSET_FIELD: + @staticmethod + def get(*args, **kwargs): + return UNSET_FIELD class FieldAssertionType(str, Enum): """Defines the types of assertions that can be made on fields.""" diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py index 0331b942..3eac2dc0 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py @@ -16,5 +16,5 @@ def normalize_activity_data(source: Activity | dict) -> dict: """Normalize Activity data to a dictionary format.""" if isinstance(source, Activity): - return source.model_dump(exclude_unset=True) + return source.model_dump(exclude_unset=True, mode="json") return source diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py index 250a379c..a8211f46 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py @@ -1,16 +1,21 @@ import pytest +from microsoft_agents.activity import Activity + from microsoft_agents.testing.assertions.type_defs import FieldAssertionType -from microsoft_agents.testing.assertions.check_activity import _parse_assertion +from microsoft_agents.testing.assertions.check_activity import ( + _parse_assertion, + check_activity +) class TestParseAssertion: @pytest.fixture( params=[ - FieldAssertionType.EQUALS.value, - FieldAssertionType.NOT_EQUALS.value, - FieldAssertionType.GREATER_THAN.value, + FieldAssertionType.EQUALS, + FieldAssertionType.NOT_EQUALS, + FieldAssertionType.GREATER_THAN ] ) def assertion_type_str(self, request): @@ -26,27 +31,370 @@ def test_parse_assertion_dict(self, assertion_value, assertion_type_str): {"assertion_type": assertion_type_str, "assertion": assertion_value} ) assert assertion == assertion_value - assert assertion_type.name == FieldAssertionType(assertion_type_str).name + assert assertion_type == FieldAssertionType(assertion_type_str) def test_parse_assertion_list(self, assertion_value, assertion_type_str): assertion, assertion_type = _parse_assertion( [assertion_type_str, assertion_value] ) assert assertion == assertion_value - assert assertion_type.name == assertion_type_str + assert assertion_type.value == assertion_type_str @pytest.mark.parametrize( "field", [ "value", + 123, + 12.34 + ], + ) + def test_parse_assertion_default(self, field): + assertion, assertion_type = _parse_assertion(field) + assert assertion == field + assert assertion_type == FieldAssertionType.EQUALS + + @pytest.mark.parametrize( + "field", + [ {"assertion_type": FieldAssertionType.IN}, {"assertion_type": FieldAssertionType.IN, "key": "value"}, [FieldAssertionType.RE_MATCH], [], {"assertion_type": "invalid", "assertion": "test"}, - ], + ] ) - def test_parse_assertion_default(self, field): + def test_parse_assertion_none(self, field): assertion, assertion_type = _parse_assertion(field) - assert assertion == field - assert assertion_type == FieldAssertionType.EQUALS + assert assertion is None + assert assertion_type is None + +class TestCheckActivity: + """Tests for check_activity function.""" + + def test_check_activity_with_matching_simple_fields(self): + """Test that activity matches baseline with simple equal fields.""" + activity = Activity(type="message", text="Hello, World!") + baseline = {"type": "message", "text": "Hello, World!"} + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_non_matching_fields(self): + """Test that activity doesn't match baseline with different field values.""" + activity = Activity(type="message", text="Hello") + baseline = {"type": "message", "text": "Goodbye"} + + assert check_activity(activity, baseline) is False + + def test_check_activity_with_activity_baseline(self): + """Test that baseline can be an Activity object.""" + activity = Activity(type="message", text="Hello") + baseline = Activity(type="message", text="Hello") + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_partial_baseline(self): + """Test that only fields in baseline are checked.""" + activity = Activity( + type="message", + text="Hello", + channel_id="test-channel", + conversation={"id": "conv123"} + ) + baseline = {"type": "message", "text": "Hello"} + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_missing_field(self): + """Test that activity with missing field doesn't match baseline.""" + activity = Activity(type="message") + baseline = {"type": "message", "text": "Hello"} + + assert check_activity(activity, baseline) is False + + def test_check_activity_with_none_values(self): + """Test that None values are handled correctly.""" + activity = Activity(type="message") + baseline = {"type": "message", "text": None} + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_empty_baseline(self): + """Test that empty baseline always matches.""" + activity = Activity(type="message", text="Hello") + baseline = {} + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_dict_assertion_format(self): + """Test using dict format for assertions.""" + activity = Activity(type="message", text="Hello, World!") + baseline = { + "type": "message", + "text": {"assertion_type": "CONTAINS", "assertion": "Hello"} + } + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_list_assertion_format(self): + """Test using list format for assertions.""" + activity = Activity(type="message", text="Hello, World!") + baseline = { + "type": "message", + "text": ["CONTAINS", "World"] + } + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_not_equals_assertion(self): + """Test NOT_EQUALS assertion type.""" + activity = Activity(type="message", text="Hello") + baseline = { + "type": "message", + "text": {"assertion_type": "NOT_EQUALS", "assertion": "Goodbye"} + } + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_contains_assertion(self): + """Test CONTAINS assertion type.""" + activity = Activity(type="message", text="Hello, World!") + baseline = { + "text": {"assertion_type": "CONTAINS", "assertion": "World"} + } + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_not_contains_assertion(self): + """Test NOT_CONTAINS assertion type.""" + activity = Activity(type="message", text="Hello") + baseline = { + "text": {"assertion_type": "NOT_CONTAINS", "assertion": "Goodbye"} + } + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_regex_assertion(self): + """Test RE_MATCH assertion type.""" + activity = Activity(type="message", text="msg_20250112_001") + baseline = { + "text": {"assertion_type": "RE_MATCH", "assertion": r"^msg_\d{8}_\d{3}$"} + } + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_multiple_fields_and_mixed_assertions(self): + """Test multiple fields with different assertion types.""" + activity = Activity( + type="message", + text="Hello, World!", + channel_id="test-channel" + ) + baseline = { + "type": "message", + "text": ["CONTAINS", "Hello"], + "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "prod-channel"} + } + + assert check_activity(activity, baseline) is True + + def test_check_activity_fails_on_any_field_mismatch(self): + """Test that activity check fails if any field doesn't match.""" + activity = Activity( + type="message", + text="Hello", + channel_id="test-channel" + ) + baseline = { + "type": "message", + "text": "Hello", + "channel_id": "prod-channel" + } + + assert check_activity(activity, baseline) is False + + def test_check_activity_with_numeric_fields(self): + """Test with numeric field values.""" + activity = Activity(type="message", locale="en-US") + activity.channel_data = {"timestamp": 1234567890} + baseline = { + "type": "message", + "channel_data": {"timestamp": 1234567890} + } + + assert check_activity(activity, baseline) is True + + def test_check_activity_with_greater_than_assertion(self): + """Test GREATER_THAN assertion on numeric fields.""" + activity = Activity(type="message") + activity.channel_data = {"count": 100} + baseline = { + "channel_data": {"count": {"assertion_type": "GREATER_THAN", "assertion": 50}} + } + + # This test depends on how nested dicts are handled + # If channel_data is compared as a whole dict, this might not work as expected + # Keeping this test to illustrate the concept + result = check_activity(activity, baseline) + # The actual behavior depends on implementation details + assert isinstance(result, bool) + + def test_check_activity_with_complex_nested_structures(self): + """Test with complex nested structures in baseline.""" + activity = Activity( + type="message", + conversation={"id": "conv123", "name": "Test Conversation"} + ) + baseline = { + "type": "message", + "conversation": {"id": "conv123", "name": "Test Conversation"} + } + assert check_activity(activity, baseline) is True + + def test_check_activity_with_boolean_fields(self): + """Test with boolean field values.""" + activity = Activity(type="message") + activity.channel_data = {"is_active": True} + baseline = { + "channel_data": {"is_active": True} + } + assert check_activity(activity, baseline) is True + + def test_check_activity_type_mismatch(self): + """Test that different activity types don't match.""" + activity = Activity(type="message", text="Hello") + baseline = {"type": "event", "text": "Hello"} + + assert check_activity(activity, baseline) is False + + def test_check_activity_with_list_fields(self): + """Test with list field values.""" + activity = Activity(type="message") + activity.attachments = [{"contentType": "text/plain", "content": "test"}] + baseline = { + "type": "message", + "attachments": [{"contentType": "text/plain", "content": "test"}] + } + + assert check_activity(activity, baseline) is True + + +class TestCheckActivityRealWorldScenarios: + """Tests simulating real-world usage scenarios.""" + + def test_validate_bot_response_message(self): + """Test validating a typical bot response.""" + activity = Activity( + type="message", + text="I found 3 results for your query.", + from_property={"id": "bot123", "name": "HelpBot"} + ) + baseline = { + "type": "message", + "text": ["RE_MATCH", r"I found \d+ results"], + "from_property": {"id": "bot123"} + } + + assert check_activity(activity, baseline) is True + + def test_validate_user_message(self): + """Test validating a user message with flexible text matching.""" + activity = Activity( + type="message", + text="help me with something", + from_property={"id": "user456"} + ) + baseline = { + "type": "message", + "text": {"assertion_type": "CONTAINS", "assertion": "help"} + } + + assert check_activity(activity, baseline) is True + + def test_validate_event_activity(self): + """Test validating an event activity.""" + activity = Activity( + type="event", + name="conversationUpdate", + value={"action": "add"} + ) + baseline = { + "type": "event", + "name": "conversationUpdate" + } + + assert check_activity(activity, baseline) is True + + def test_partial_match_allows_extra_fields(self): + """Test that extra fields in activity don't cause failure.""" + activity = Activity( + type="message", + text="Hello", + channel_id="teams", + conversation={"id": "conv123"}, + from_property={"id": "user123"}, + timestamp="2025-01-12T10:00:00Z" + ) + baseline = { + "type": "message", + "text": "Hello" + } + + assert check_activity(activity, baseline) is True + + def test_strict_match_with_multiple_fields(self): + """Test strict matching with multiple fields specified.""" + activity = Activity( + type="message", + text="Hello", + channel_id="teams" + ) + baseline = { + "type": "message", + "text": "Hello", + "channel_id": "teams" + } + + assert check_activity(activity, baseline) is True + + def test_flexible_text_matching_with_regex(self): + """Test flexible text matching using regex patterns.""" + activity = Activity( + type="message", + text="Order #12345 has been confirmed" + ) + baseline = { + "type": "message", + "text": ["RE_MATCH", r"Order #\d+ has been"] + } + + assert check_activity(activity, baseline) is True + + def test_negative_assertions(self): + """Test using negative assertions to ensure fields don't match.""" + activity = Activity( + type="message", + text="Success", + channel_id="teams" + ) + baseline = { + "type": "message", + "text": {"assertion_type": "NOT_CONTAINS", "assertion": "Error"}, + "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "slack"} + } + + assert check_activity(activity, baseline) is True + + def test_combined_positive_and_negative_assertions(self): + """Test combining positive and negative assertions.""" + activity = Activity( + type="message", + text="Operation completed successfully", + channel_id="teams" + ) + baseline = { + "type": "message", + "text": ["CONTAINS", "completed"], + "channel_id": ["NOT_EQUALS", "slack"] + } + + assert check_activity(activity, baseline) is True \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py index e69de29b..99de0866 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py @@ -0,0 +1,214 @@ +from microsoft_agents.testing.assertions.check_field import check_field +from microsoft_agents.testing.assertions.type_defs import FieldAssertionType + + +class TestCheckFieldEquals: + """Tests for EQUALS assertion type.""" + + def test_equals_with_matching_strings(self): + assert check_field("hello", "hello", FieldAssertionType.EQUALS) is True + + def test_equals_with_non_matching_strings(self): + assert check_field("hello", "world", FieldAssertionType.EQUALS) is False + + def test_equals_with_matching_integers(self): + assert check_field(42, 42, FieldAssertionType.EQUALS) is True + + def test_equals_with_non_matching_integers(self): + assert check_field(42, 43, FieldAssertionType.EQUALS) is False + + def test_equals_with_none_values(self): + assert check_field(None, None, FieldAssertionType.EQUALS) is True + + def test_equals_with_boolean_values(self): + assert check_field(True, True, FieldAssertionType.EQUALS) is True + assert check_field(False, False, FieldAssertionType.EQUALS) is True + assert check_field(True, False, FieldAssertionType.EQUALS) is False + + +class TestCheckFieldNotEquals: + """Tests for NOT_EQUALS assertion type.""" + + def test_not_equals_with_different_strings(self): + assert check_field("hello", "world", FieldAssertionType.NOT_EQUALS) is True + + def test_not_equals_with_matching_strings(self): + assert check_field("hello", "hello", FieldAssertionType.NOT_EQUALS) is False + + def test_not_equals_with_different_integers(self): + assert check_field(42, 43, FieldAssertionType.NOT_EQUALS) is True + + def test_not_equals_with_matching_integers(self): + assert check_field(42, 42, FieldAssertionType.NOT_EQUALS) is False + + +class TestCheckFieldGreaterThan: + """Tests for GREATER_THAN assertion type.""" + + def test_greater_than_with_larger_value(self): + assert check_field(10, 5, FieldAssertionType.GREATER_THAN) is True + + def test_greater_than_with_smaller_value(self): + assert check_field(5, 10, FieldAssertionType.GREATER_THAN) is False + + def test_greater_than_with_equal_value(self): + assert check_field(10, 10, FieldAssertionType.GREATER_THAN) is False + + def test_greater_than_with_floats(self): + assert check_field(10.5, 10.2, FieldAssertionType.GREATER_THAN) is True + assert check_field(10.2, 10.5, FieldAssertionType.GREATER_THAN) is False + + def test_greater_than_with_negative_numbers(self): + assert check_field(-5, -10, FieldAssertionType.GREATER_THAN) is True + assert check_field(-10, -5, FieldAssertionType.GREATER_THAN) is False + + +class TestCheckFieldLessThan: + """Tests for LESS_THAN assertion type.""" + + def test_less_than_with_smaller_value(self): + assert check_field(5, 10, FieldAssertionType.LESS_THAN) is True + + def test_less_than_with_larger_value(self): + assert check_field(10, 5, FieldAssertionType.LESS_THAN) is False + + def test_less_than_with_equal_value(self): + assert check_field(10, 10, FieldAssertionType.LESS_THAN) is False + + def test_less_than_with_floats(self): + assert check_field(10.2, 10.5, FieldAssertionType.LESS_THAN) is True + assert check_field(10.5, 10.2, FieldAssertionType.LESS_THAN) is False + + +class TestCheckFieldContains: + """Tests for CONTAINS assertion type.""" + + def test_contains_substring_in_string(self): + assert check_field("hello world", "world", FieldAssertionType.CONTAINS) is True + + def test_contains_substring_not_in_string(self): + assert check_field("hello world", "foo", FieldAssertionType.CONTAINS) is False + + def test_contains_element_in_list(self): + assert check_field([1, 2, 3, 4], 3, FieldAssertionType.CONTAINS) is True + + def test_contains_element_not_in_list(self): + assert check_field([1, 2, 3, 4], 5, FieldAssertionType.CONTAINS) is False + + def test_contains_key_in_dict(self): + assert check_field({"a": 1, "b": 2}, "a", FieldAssertionType.CONTAINS) is True + + def test_contains_key_not_in_dict(self): + assert check_field({"a": 1, "b": 2}, "c", FieldAssertionType.CONTAINS) is False + + def test_contains_empty_string(self): + assert check_field("hello", "", FieldAssertionType.CONTAINS) is True + + +class TestCheckFieldNotContains: + """Tests for NOT_CONTAINS assertion type.""" + + def test_not_contains_substring_not_in_string(self): + assert check_field("hello world", "foo", FieldAssertionType.NOT_CONTAINS) is True + + def test_not_contains_substring_in_string(self): + assert check_field("hello world", "world", FieldAssertionType.NOT_CONTAINS) is False + + def test_not_contains_element_not_in_list(self): + assert check_field([1, 2, 3, 4], 5, FieldAssertionType.NOT_CONTAINS) is True + + def test_not_contains_element_in_list(self): + assert check_field([1, 2, 3, 4], 3, FieldAssertionType.NOT_CONTAINS) is False + + +class TestCheckFieldReMatch: + """Tests for RE_MATCH assertion type.""" + + def test_re_match_simple_pattern(self): + assert check_field("hello123", r"hello\d+", FieldAssertionType.RE_MATCH) is True + + def test_re_match_no_match(self): + assert check_field("hello", r"\d+", FieldAssertionType.RE_MATCH) is False + + def test_re_match_email_pattern(self): + pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + assert check_field("test@example.com", pattern, FieldAssertionType.RE_MATCH) is True + assert check_field("invalid-email", pattern, FieldAssertionType.RE_MATCH) is False + + def test_re_match_anchored_pattern(self): + assert check_field("hello world", r"^hello", FieldAssertionType.RE_MATCH) is True + assert check_field("hello world", r"^world", FieldAssertionType.RE_MATCH) is False + + def test_re_match_full_string(self): + assert check_field("abc", r"^abc$", FieldAssertionType.RE_MATCH) is True + assert check_field("abcd", r"^abc$", FieldAssertionType.RE_MATCH) is False + + def test_re_match_case_sensitive(self): + assert check_field("Hello", r"hello", FieldAssertionType.RE_MATCH) is False + assert check_field("Hello", r"Hello", FieldAssertionType.RE_MATCH) is True + + +class TestCheckFieldEdgeCases: + """Tests for edge cases and error handling.""" + + def test_invalid_assertion_type(self): + # Passing an unsupported assertion type should return False + assert check_field("test", "test", "INVALID_TYPE") is False + + def test_none_actual_value_with_equals(self): + assert check_field(None, "test", FieldAssertionType.EQUALS) is False + assert check_field(None, None, FieldAssertionType.EQUALS) is True + + def test_empty_string_comparisons(self): + assert check_field("", "", FieldAssertionType.EQUALS) is True + assert check_field("", "test", FieldAssertionType.EQUALS) is False + + def test_empty_list_contains(self): + assert check_field([], "item", FieldAssertionType.CONTAINS) is False + + def test_zero_comparisons(self): + assert check_field(0, 0, FieldAssertionType.EQUALS) is True + assert check_field(0, 1, FieldAssertionType.LESS_THAN) is True + assert check_field(0, -1, FieldAssertionType.GREATER_THAN) is True + + def test_type_mismatch_comparisons(self): + # Different types should work with equality checks + assert check_field("42", 42, FieldAssertionType.EQUALS) is False + assert check_field("42", 42, FieldAssertionType.NOT_EQUALS) is True + + def test_complex_data_structures(self): + actual = {"nested": {"value": 123}} + expected = {"nested": {"value": 123}} + assert check_field(actual, expected, FieldAssertionType.EQUALS) is True + + def test_list_equality(self): + assert check_field([1, 2, 3], [1, 2, 3], FieldAssertionType.EQUALS) is True + assert check_field([1, 2, 3], [3, 2, 1], FieldAssertionType.EQUALS) is False + + +class TestCheckFieldWithRealWorldScenarios: + """Tests simulating real-world usage scenarios.""" + + def test_validate_response_status_code(self): + assert check_field(200, 200, FieldAssertionType.EQUALS) is True + assert check_field(404, 200, FieldAssertionType.NOT_EQUALS) is True + + def test_validate_response_contains_keyword(self): + response = "Success: Operation completed successfully" + assert check_field(response, "Success", FieldAssertionType.CONTAINS) is True + assert check_field(response, "Error", FieldAssertionType.NOT_CONTAINS) is True + + def test_validate_numeric_threshold(self): + temperature = 72.5 + assert check_field(temperature, 100, FieldAssertionType.LESS_THAN) is True + assert check_field(temperature, 0, FieldAssertionType.GREATER_THAN) is True + + def test_validate_message_format(self): + message_id = "msg_20250112_001" + pattern = r"^msg_\d{8}_\d{3}$" + assert check_field(message_id, pattern, FieldAssertionType.RE_MATCH) is True + + def test_validate_list_membership(self): + allowed_roles = ["admin", "user", "guest"] + assert check_field(allowed_roles, "admin", FieldAssertionType.CONTAINS) is True + assert check_field(allowed_roles, "superuser", FieldAssertionType.NOT_CONTAINS) is True \ No newline at end of file From d9eb5fa2fedd3f30e9b8381a8f8731bbed4b7eb5 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Wed, 12 Nov 2025 12:42:59 -0800 Subject: [PATCH 63/81] Fixing more test cases with newest refactor --- .../microsoft_agents/testing/assertions/check_activity.py | 7 ++++--- .../microsoft_agents/testing/assertions/check_field.py | 6 +++--- .../tests/assertions/test_check_activity.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index 17c9ef12..da461096 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -59,6 +59,7 @@ def _check(actual: Any, baseline: Any) -> bool: new_baseline = baseline[key] if not _check(new_actual, new_baseline): return False + return True elif isinstance(baseline, list): for index, item in enumerate(baseline): @@ -66,14 +67,14 @@ def _check(actual: Any, baseline: Any) -> bool: new_baseline = item if not _check(new_actual, new_baseline): return False + return True else: raise ValueError("Unsupported baseline type for complex assertion.") else: assert assertion_type assertion, assertion_type = _parse_assertion(baseline) - if not check_field(actual, assertion, assertion_type): - return False - + return check_field(actual, assertion, assertion_type) + def check_activity(activity: Activity, baseline: Activity | dict) -> bool: """Asserts that the given activity matches the baseline activity. diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index 9ed52f0e..2a3bcfce 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -1,11 +1,11 @@ import re from typing import Any -from .type_defs import FieldAssertionType +from .type_defs import FieldAssertionType, UNSET_FIELD _OPERATIONS = { - FieldAssertionType.EQUALS: lambda a, b: a == b, - FieldAssertionType.NOT_EQUALS: lambda a, b: a != b, + FieldAssertionType.EQUALS: lambda a, b: a == b or (a is UNSET_FIELD and b is None), + FieldAssertionType.NOT_EQUALS: lambda a, b: a != b or (a is UNSET_FIELD and b is not None), FieldAssertionType.GREATER_THAN: lambda a, b: a > b, FieldAssertionType.LESS_THAN: lambda a, b: a < b, FieldAssertionType.CONTAINS: lambda a, b: b in a, diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py index a8211f46..61cc4b3a 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py @@ -75,7 +75,7 @@ def test_check_activity_with_matching_simple_fields(self): """Test that activity matches baseline with simple equal fields.""" activity = Activity(type="message", text="Hello, World!") baseline = {"type": "message", "text": "Hello, World!"} - + assert check_activity(activity, baseline) is True def test_check_activity_with_non_matching_fields(self): From 57b87eb9eade0fc52616519d01056d675ee9ff4b Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 13 Nov 2025 08:01:58 -0800 Subject: [PATCH 64/81] assert_activity tests --- .../tests/assertions/test_check_activity.py | 95 +++++++------------ 1 file changed, 34 insertions(+), 61 deletions(-) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py index 61cc4b3a..8aefb0ce 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py @@ -1,8 +1,9 @@ import pytest -from microsoft_agents.activity import Activity +from microsoft_agents.activity import Activity, Attachment from microsoft_agents.testing.assertions.type_defs import FieldAssertionType +from microsoft_agents.testing.assertions.assertions import assert_activity from microsoft_agents.testing.assertions.check_activity import ( _parse_assertion, check_activity @@ -75,22 +76,19 @@ def test_check_activity_with_matching_simple_fields(self): """Test that activity matches baseline with simple equal fields.""" activity = Activity(type="message", text="Hello, World!") baseline = {"type": "message", "text": "Hello, World!"} - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_non_matching_fields(self): """Test that activity doesn't match baseline with different field values.""" activity = Activity(type="message", text="Hello") baseline = {"type": "message", "text": "Goodbye"} - - assert check_activity(activity, baseline) is False + assert_activity(activity, baseline) def test_check_activity_with_activity_baseline(self): """Test that baseline can be an Activity object.""" activity = Activity(type="message", text="Hello") baseline = Activity(type="message", text="Hello") - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_partial_baseline(self): """Test that only fields in baseline are checked.""" @@ -101,29 +99,25 @@ def test_check_activity_with_partial_baseline(self): conversation={"id": "conv123"} ) baseline = {"type": "message", "text": "Hello"} - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_missing_field(self): """Test that activity with missing field doesn't match baseline.""" activity = Activity(type="message") baseline = {"type": "message", "text": "Hello"} - - assert check_activity(activity, baseline) is False + assert_activity(activity, baseline) def test_check_activity_with_none_values(self): """Test that None values are handled correctly.""" activity = Activity(type="message") baseline = {"type": "message", "text": None} - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_empty_baseline(self): """Test that empty baseline always matches.""" activity = Activity(type="message", text="Hello") baseline = {} - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_dict_assertion_format(self): """Test using dict format for assertions.""" @@ -132,8 +126,7 @@ def test_check_activity_with_dict_assertion_format(self): "type": "message", "text": {"assertion_type": "CONTAINS", "assertion": "Hello"} } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_list_assertion_format(self): """Test using list format for assertions.""" @@ -142,8 +135,7 @@ def test_check_activity_with_list_assertion_format(self): "type": "message", "text": ["CONTAINS", "World"] } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_not_equals_assertion(self): """Test NOT_EQUALS assertion type.""" @@ -152,8 +144,7 @@ def test_check_activity_with_not_equals_assertion(self): "type": "message", "text": {"assertion_type": "NOT_EQUALS", "assertion": "Goodbye"} } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_contains_assertion(self): """Test CONTAINS assertion type.""" @@ -161,8 +152,7 @@ def test_check_activity_with_contains_assertion(self): baseline = { "text": {"assertion_type": "CONTAINS", "assertion": "World"} } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_not_contains_assertion(self): """Test NOT_CONTAINS assertion type.""" @@ -170,8 +160,7 @@ def test_check_activity_with_not_contains_assertion(self): baseline = { "text": {"assertion_type": "NOT_CONTAINS", "assertion": "Goodbye"} } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_regex_assertion(self): """Test RE_MATCH assertion type.""" @@ -179,8 +168,7 @@ def test_check_activity_with_regex_assertion(self): baseline = { "text": {"assertion_type": "RE_MATCH", "assertion": r"^msg_\d{8}_\d{3}$"} } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_multiple_fields_and_mixed_assertions(self): """Test multiple fields with different assertion types.""" @@ -194,8 +182,7 @@ def test_check_activity_with_multiple_fields_and_mixed_assertions(self): "text": ["CONTAINS", "Hello"], "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "prod-channel"} } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_fails_on_any_field_mismatch(self): """Test that activity check fails if any field doesn't match.""" @@ -209,8 +196,7 @@ def test_check_activity_fails_on_any_field_mismatch(self): "text": "Hello", "channel_id": "prod-channel" } - - assert check_activity(activity, baseline) is False + assert_activity(activity, baseline) def test_check_activity_with_numeric_fields(self): """Test with numeric field values.""" @@ -220,8 +206,7 @@ def test_check_activity_with_numeric_fields(self): "type": "message", "channel_data": {"timestamp": 1234567890} } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_greater_than_assertion(self): """Test GREATER_THAN assertion on numeric fields.""" @@ -234,9 +219,7 @@ def test_check_activity_with_greater_than_assertion(self): # This test depends on how nested dicts are handled # If channel_data is compared as a whole dict, this might not work as expected # Keeping this test to illustrate the concept - result = check_activity(activity, baseline) - # The actual behavior depends on implementation details - assert isinstance(result, bool) + assert_activity(activity, baseline) def test_check_activity_with_complex_nested_structures(self): """Test with complex nested structures in baseline.""" @@ -248,7 +231,7 @@ def test_check_activity_with_complex_nested_structures(self): "type": "message", "conversation": {"id": "conv123", "name": "Test Conversation"} } - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_with_boolean_fields(self): """Test with boolean field values.""" @@ -257,27 +240,24 @@ def test_check_activity_with_boolean_fields(self): baseline = { "channel_data": {"is_active": True} } - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_check_activity_type_mismatch(self): """Test that different activity types don't match.""" activity = Activity(type="message", text="Hello") baseline = {"type": "event", "text": "Hello"} - - assert check_activity(activity, baseline) is False - + assert_activity(activity, baseline) + def test_check_activity_with_list_fields(self): """Test with list field values.""" activity = Activity(type="message") - activity.attachments = [{"contentType": "text/plain", "content": "test"}] + activity.attachments = [Attachment(content_type="text/plain", content="test")] baseline = { "type": "message", - "attachments": [{"contentType": "text/plain", "content": "test"}] + "attachments": [{"content_type": "text/plain", "content": "test"}] } - - assert check_activity(activity, baseline) is True - - + assert_activity(activity, baseline) + class TestCheckActivityRealWorldScenarios: """Tests simulating real-world usage scenarios.""" @@ -293,8 +273,7 @@ def test_validate_bot_response_message(self): "text": ["RE_MATCH", r"I found \d+ results"], "from_property": {"id": "bot123"} } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_validate_user_message(self): """Test validating a user message with flexible text matching.""" @@ -307,8 +286,7 @@ def test_validate_user_message(self): "type": "message", "text": {"assertion_type": "CONTAINS", "assertion": "help"} } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_validate_event_activity(self): """Test validating an event activity.""" @@ -338,8 +316,7 @@ def test_partial_match_allows_extra_fields(self): "type": "message", "text": "Hello" } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_strict_match_with_multiple_fields(self): """Test strict matching with multiple fields specified.""" @@ -353,8 +330,7 @@ def test_strict_match_with_multiple_fields(self): "text": "Hello", "channel_id": "teams" } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_flexible_text_matching_with_regex(self): """Test flexible text matching using regex patterns.""" @@ -366,8 +342,7 @@ def test_flexible_text_matching_with_regex(self): "type": "message", "text": ["RE_MATCH", r"Order #\d+ has been"] } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_negative_assertions(self): """Test using negative assertions to ensure fields don't match.""" @@ -381,8 +356,7 @@ def test_negative_assertions(self): "text": {"assertion_type": "NOT_CONTAINS", "assertion": "Error"}, "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "slack"} } - - assert check_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_combined_positive_and_negative_assertions(self): """Test combining positive and negative assertions.""" @@ -396,5 +370,4 @@ def test_combined_positive_and_negative_assertions(self): "text": ["CONTAINS", "completed"], "channel_id": ["NOT_EQUALS", "slack"] } - - assert check_activity(activity, baseline) is True \ No newline at end of file + assert_activity(activity, baseline) \ No newline at end of file From 6925093541c089bfb4193ba21ffd996c89d2d78d Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 13 Nov 2025 08:04:33 -0800 Subject: [PATCH 65/81] Reorganizing tests --- .../testing/assertions/assertions.py | 5 +- .../testing/assertions/check_activity.py | 90 ++++++++----------- .../testing/assertions/check_field.py | 40 ++++++++- .../testing/assertions/type_defs.py | 22 +++++ ...ck_activity.py => test_assert_activity.py} | 55 ++++++------ 5 files changed, 126 insertions(+), 86 deletions(-) rename dev/microsoft-agents-testing/tests/assertions/{test_check_activity.py => test_assert_activity.py} (88%) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py index 92b9786a..6977748e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -3,7 +3,7 @@ from microsoft_agents.activity import Activity from .type_defs import FieldAssertionType -from .check_activity import check_activity +from .check_activity import check_activity_verbose from .check_field import check_field @@ -13,7 +13,8 @@ def assert_activity(activity: Activity, baseline: Activity | dict) -> None: :param activity: The activity to be tested. :param baseline: The baseline activity or a dictionary representing the expected activity data. """ - assert check_activity(activity, baseline) + res, assertion_error_data = check_activity_verbose(activity, baseline) + assert res, str(assertion_error_data) def assert_field( diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index da461096..7c3a4ac2 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -5,83 +5,65 @@ from microsoft_agents.activity import Activity from microsoft_agents.testing.utils import normalize_activity_data -from .check_field import check_field -from .type_defs import UNSET_FIELD, FieldAssertionType +from .check_field import check_field, _parse_assertion +from .type_defs import UNSET_FIELD, FieldAssertionType, AssertionErrorData - -def _parse_assertion(field: Any) -> tuple[Any, Optional[FieldAssertionType]]: - """Parses the assertion information and returns the assertion type and baseline value. - - :param assertion_info: The assertion information to be parsed. - :return: A tuple containing the assertion type and baseline value. - """ - - assertion_type = FieldAssertionType.EQUALS - assertion = None - - if ( - isinstance(field, dict) - and "assertion_type" in field - and "assertion" in field - and field["assertion_type"] in FieldAssertionType.__members__ - ): - # format: - # {"assertion_type": "__EQ__", "assertion": "value"} - assertion_type = field["assertion_type"] - assertion = field.get("assertion") - - elif ( - isinstance(field, list) - and len(field) >= 2 - and isinstance(field[0], str) - and field[0] in FieldAssertionType.__members__ - ): - # format: - # ["__EQ__", "assertion"] - assertion_type = FieldAssertionType[field[0]] - assertion = field[1] - elif isinstance(field, list) or isinstance(field, dict): - assertion_type = None - else: - # default format: direct value - assertion = field - - return assertion, assertion_type - -def _check(actual: Any, baseline: Any) -> bool: +def _check(actual: Any, baseline: Any, field_path: str = "") -> tuple[bool, Optional[AssertionErrorData]]: assertion, assertion_type = _parse_assertion(baseline) if assertion_type is None: if isinstance(baseline, dict): for key in baseline: + new_field_path = f"{field_path}.{key}" if field_path else key new_actual = actual.get(key, UNSET_FIELD) new_baseline = baseline[key] - if not _check(new_actual, new_baseline): - return False - return True + + res, assertion_error_data = _check(new_actual, new_baseline, new_field_path) + if not res: + return False, assertion_error_data + return True, None elif isinstance(baseline, list): for index, item in enumerate(baseline): + new_field_path = f"{field_path}[{index}]" if field_path else f"[{index}]" new_actual = actual[index] if index < len(actual) else UNSET_FIELD new_baseline = item - if not _check(new_actual, new_baseline): - return False - return True + + res, assertion_error_data = _check(new_actual, new_baseline, new_field_path) + if not res: + return False, assertion_error_data + return True, None else: raise ValueError("Unsupported baseline type for complex assertion.") else: - assert assertion_type - assertion, assertion_type = _parse_assertion(baseline) - return check_field(actual, assertion, assertion_type) - + assert isinstance(assertion_type, FieldAssertionType) + res = check_field(actual, assertion, assertion_type) + if res: + return True, None + else: + assertion_error_data = AssertionErrorData( + field_path=field_path, + actual_value=actual, + assertion=assertion, + assertion_type=assertion_type, + ) + return False, assertion_error_data def check_activity(activity: Activity, baseline: Activity | dict) -> bool: """Asserts that the given activity matches the baseline activity. + :param activity: The activity to be tested. + :param baseline: The baseline activity or a dictionary representing the expected activity data. + """ + return check_activity_verbose(activity, baseline)[0] + +def check_activity_verbose(activity: Activity, baseline: Activity | dict) -> tuple[bool, Optional[AssertionErrorData]]: + """Asserts that the given activity matches the baseline activity. + :param activity: The activity to be tested. :param baseline: The baseline activity or a dictionary representing the expected activity data. """ actual_activity = normalize_activity_data(activity) baseline = normalize_activity_data(baseline) - return _check(actual_activity, baseline) \ No newline at end of file + return _check(actual_activity, baseline, "activity") \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index 2a3bcfce..612ea5c9 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -1,5 +1,5 @@ import re -from typing import Any +from typing import Any, Optional from .type_defs import FieldAssertionType, UNSET_FIELD @@ -13,6 +13,44 @@ FieldAssertionType.RE_MATCH: lambda a, b: re.match(b, a) is not None, } +def _parse_assertion(field: Any) -> tuple[Any, Optional[FieldAssertionType]]: + """Parses the assertion information and returns the assertion type and baseline value. + + :param assertion_info: The assertion information to be parsed. + :return: A tuple containing the assertion type and baseline value. + """ + + assertion_type = FieldAssertionType.EQUALS + assertion = None + + if ( + isinstance(field, dict) + and "assertion_type" in field + and "assertion" in field + and field["assertion_type"] in FieldAssertionType.__members__ + ): + # format: + # {"assertion_type": "__EQ__", "assertion": "value"} + assertion_type = FieldAssertionType[field["assertion_type"]] + assertion = field.get("assertion") + + elif ( + isinstance(field, list) + and len(field) >= 2 + and isinstance(field[0], str) + and field[0] in FieldAssertionType.__members__ + ): + # format: + # ["__EQ__", "assertion"] + assertion_type = FieldAssertionType[field[0]] + assertion = field[1] + elif isinstance(field, list) or isinstance(field, dict): + assertion_type = None + else: + # default format: direct value + assertion = field + + return assertion, assertion_type def check_field( actual_value: Any, assertion: Any, assertion_type: FieldAssertionType diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py index 362e9a27..844c3146 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py @@ -1,8 +1,13 @@ from enum import Enum +from dataclasses import dataclass +from typing import Any class UNSET_FIELD: + """Singleton to represent an unset field in activity comparisons.""" + @staticmethod def get(*args, **kwargs): + """Returns the singleton instance.""" return UNSET_FIELD class FieldAssertionType(str, Enum): @@ -17,3 +22,20 @@ class FieldAssertionType(str, Enum): IN = "IN" NOT_IN = "NOT_IN" RE_MATCH = "RE_MATCH" + +@dataclass +class AssertionErrorData: + """Data class to hold information about assertion errors.""" + + field_path: str + actual_value: Any + assertion: Any + assertion_type: FieldAssertionType + + def __str__(self) -> str: + return ( + f"Assertion failed at '{self.field_path}': " + f"actual value '{self.actual_value}' " + f"does not satisfy assertion '{self.assertion}' " + f"of type '{self.assertion_type}'." + ) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py similarity index 88% rename from dev/microsoft-agents-testing/tests/assertions/test_check_activity.py rename to dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py index 8aefb0ce..611a58d9 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_activity.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py @@ -4,10 +4,7 @@ from microsoft_agents.testing.assertions.type_defs import FieldAssertionType from microsoft_agents.testing.assertions.assertions import assert_activity -from microsoft_agents.testing.assertions.check_activity import ( - _parse_assertion, - check_activity -) +from microsoft_agents.testing.assertions.check_field import _parse_assertion class TestParseAssertion: @@ -69,28 +66,28 @@ def test_parse_assertion_none(self, field): assert assertion is None assert assertion_type is None -class TestCheckActivity: - """Tests for check_activity function.""" +class TestAssertActivity: + """Tests for assert_activity function.""" - def test_check_activity_with_matching_simple_fields(self): + def test_assert_activity_with_matching_simple_fields(self): """Test that activity matches baseline with simple equal fields.""" activity = Activity(type="message", text="Hello, World!") baseline = {"type": "message", "text": "Hello, World!"} assert_activity(activity, baseline) - def test_check_activity_with_non_matching_fields(self): + def test_assert_activity_with_non_matching_fields(self): """Test that activity doesn't match baseline with different field values.""" activity = Activity(type="message", text="Hello") baseline = {"type": "message", "text": "Goodbye"} assert_activity(activity, baseline) - def test_check_activity_with_activity_baseline(self): + def test_assert_activity_with_activity_baseline(self): """Test that baseline can be an Activity object.""" activity = Activity(type="message", text="Hello") baseline = Activity(type="message", text="Hello") assert_activity(activity, baseline) - def test_check_activity_with_partial_baseline(self): + def test_assert_activity_with_partial_baseline(self): """Test that only fields in baseline are checked.""" activity = Activity( type="message", @@ -101,25 +98,25 @@ def test_check_activity_with_partial_baseline(self): baseline = {"type": "message", "text": "Hello"} assert_activity(activity, baseline) - def test_check_activity_with_missing_field(self): + def test_assert_activity_with_missing_field(self): """Test that activity with missing field doesn't match baseline.""" activity = Activity(type="message") baseline = {"type": "message", "text": "Hello"} assert_activity(activity, baseline) - def test_check_activity_with_none_values(self): + def test_assert_activity_with_none_values(self): """Test that None values are handled correctly.""" activity = Activity(type="message") baseline = {"type": "message", "text": None} assert_activity(activity, baseline) - def test_check_activity_with_empty_baseline(self): + def test_assert_activity_with_empty_baseline(self): """Test that empty baseline always matches.""" activity = Activity(type="message", text="Hello") baseline = {} assert_activity(activity, baseline) - def test_check_activity_with_dict_assertion_format(self): + def test_assert_activity_with_dict_assertion_format(self): """Test using dict format for assertions.""" activity = Activity(type="message", text="Hello, World!") baseline = { @@ -128,7 +125,7 @@ def test_check_activity_with_dict_assertion_format(self): } assert_activity(activity, baseline) - def test_check_activity_with_list_assertion_format(self): + def test_assert_activity_with_list_assertion_format(self): """Test using list format for assertions.""" activity = Activity(type="message", text="Hello, World!") baseline = { @@ -137,7 +134,7 @@ def test_check_activity_with_list_assertion_format(self): } assert_activity(activity, baseline) - def test_check_activity_with_not_equals_assertion(self): + def test_assert_activity_with_not_equals_assertion(self): """Test NOT_EQUALS assertion type.""" activity = Activity(type="message", text="Hello") baseline = { @@ -146,7 +143,7 @@ def test_check_activity_with_not_equals_assertion(self): } assert_activity(activity, baseline) - def test_check_activity_with_contains_assertion(self): + def test_assert_activity_with_contains_assertion(self): """Test CONTAINS assertion type.""" activity = Activity(type="message", text="Hello, World!") baseline = { @@ -154,7 +151,7 @@ def test_check_activity_with_contains_assertion(self): } assert_activity(activity, baseline) - def test_check_activity_with_not_contains_assertion(self): + def test_assert_activity_with_not_contains_assertion(self): """Test NOT_CONTAINS assertion type.""" activity = Activity(type="message", text="Hello") baseline = { @@ -162,7 +159,7 @@ def test_check_activity_with_not_contains_assertion(self): } assert_activity(activity, baseline) - def test_check_activity_with_regex_assertion(self): + def test_assert_activity_with_regex_assertion(self): """Test RE_MATCH assertion type.""" activity = Activity(type="message", text="msg_20250112_001") baseline = { @@ -170,7 +167,7 @@ def test_check_activity_with_regex_assertion(self): } assert_activity(activity, baseline) - def test_check_activity_with_multiple_fields_and_mixed_assertions(self): + def test_assert_activity_with_multiple_fields_and_mixed_assertions(self): """Test multiple fields with different assertion types.""" activity = Activity( type="message", @@ -184,7 +181,7 @@ def test_check_activity_with_multiple_fields_and_mixed_assertions(self): } assert_activity(activity, baseline) - def test_check_activity_fails_on_any_field_mismatch(self): + def test_assert_activity_fails_on_any_field_mismatch(self): """Test that activity check fails if any field doesn't match.""" activity = Activity( type="message", @@ -198,7 +195,7 @@ def test_check_activity_fails_on_any_field_mismatch(self): } assert_activity(activity, baseline) - def test_check_activity_with_numeric_fields(self): + def test_assert_activity_with_numeric_fields(self): """Test with numeric field values.""" activity = Activity(type="message", locale="en-US") activity.channel_data = {"timestamp": 1234567890} @@ -208,7 +205,7 @@ def test_check_activity_with_numeric_fields(self): } assert_activity(activity, baseline) - def test_check_activity_with_greater_than_assertion(self): + def test_assert_activity_with_greater_than_assertion(self): """Test GREATER_THAN assertion on numeric fields.""" activity = Activity(type="message") activity.channel_data = {"count": 100} @@ -221,7 +218,7 @@ def test_check_activity_with_greater_than_assertion(self): # Keeping this test to illustrate the concept assert_activity(activity, baseline) - def test_check_activity_with_complex_nested_structures(self): + def test_assert_activity_with_complex_nested_structures(self): """Test with complex nested structures in baseline.""" activity = Activity( type="message", @@ -233,7 +230,7 @@ def test_check_activity_with_complex_nested_structures(self): } assert_activity(activity, baseline) - def test_check_activity_with_boolean_fields(self): + def test_assert_activity_with_boolean_fields(self): """Test with boolean field values.""" activity = Activity(type="message") activity.channel_data = {"is_active": True} @@ -242,13 +239,13 @@ def test_check_activity_with_boolean_fields(self): } assert_activity(activity, baseline) - def test_check_activity_type_mismatch(self): + def test_assert_activity_type_mismatch(self): """Test that different activity types don't match.""" activity = Activity(type="message", text="Hello") baseline = {"type": "event", "text": "Hello"} assert_activity(activity, baseline) - def test_check_activity_with_list_fields(self): + def test_assert_activity_with_list_fields(self): """Test with list field values.""" activity = Activity(type="message") activity.attachments = [Attachment(content_type="text/plain", content="test")] @@ -258,7 +255,7 @@ def test_check_activity_with_list_fields(self): } assert_activity(activity, baseline) -class TestCheckActivityRealWorldScenarios: +class TestAssertActivityRealWorldScenarios: """Tests simulating real-world usage scenarios.""" def test_validate_bot_response_message(self): @@ -300,7 +297,7 @@ def test_validate_event_activity(self): "name": "conversationUpdate" } - assert check_activity(activity, baseline) is True + assert assert_activity(activity, baseline) is True def test_partial_match_allows_extra_fields(self): """Test that extra fields in activity don't cause failure.""" From 6d55b29acea7a0d4b110044a94c4f49727e63104 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 13 Nov 2025 08:14:44 -0800 Subject: [PATCH 66/81] Copyright comments --- .../microsoft_agents/testing/__init__.py | 3 +++ .../testing/assertions/__init__.py | 3 +++ .../testing/assertions/assertions.py | 8 +++++-- .../testing/assertions/check_activity.py | 5 +++-- .../testing/assertions/check_field.py | 21 +++++++++++++++++++ .../testing/assertions/type_defs.py | 3 +++ .../microsoft_agents/testing/auth/__init__.py | 3 +++ .../testing/auth/generate_token.py | 3 +++ .../testing/integration/__init__.py | 3 +++ .../testing/integration/core/__init__.py | 3 +++ .../integration/core/aiohttp/__init__.py | 3 +++ .../core/aiohttp/aiohttp_environment.py | 3 +++ .../core/aiohttp/aiohttp_runner.py | 3 +++ .../integration/core/application_runner.py | 3 +++ .../integration/core/client/__init__.py | 3 +++ .../integration/core/client/agent_client.py | 3 +++ .../core/client/response_client.py | 3 +++ .../testing/integration/core/environment.py | 3 +++ .../testing/integration/core/integration.py | 3 +++ .../testing/integration/core/sample.py | 3 +++ .../microsoft_agents/testing/sdk_config.py | 3 +++ .../testing/utils/__init__.py | 3 +++ .../microsoft_agents/testing/utils/misc.py | 3 +++ .../testing/utils/populate_activity.py | 3 +++ .../microsoft_agents/testing/utils/urls.py | 3 +++ 25 files changed, 96 insertions(+), 4 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py index c8364fff..b85d37e9 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .sdk_config import SDKConfig from .auth import generate_token, generate_token_from_config diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py index e36bff9f..462bcd9f 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .assertions import ( assert_activity, assert_field, diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py index 6977748e..5127cecd 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -1,10 +1,13 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from typing import Any from microsoft_agents.activity import Activity from .type_defs import FieldAssertionType from .check_activity import check_activity_verbose -from .check_field import check_field +from .check_field import check_field_verbose def assert_activity(activity: Activity, baseline: Activity | dict) -> None: @@ -26,4 +29,5 @@ def assert_field( :param target: The target dictionary containing the actual values. :param baseline: The baseline dictionary containing the expected values. """ - assert check_field(actual_value, baseline_value, assertion_type) + res, assertion_error_message = check_field_verbose(actual_value, baseline_value, assertion_type) + assert res, assertion_error_message \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index 7c3a4ac2..612b7b71 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -1,6 +1,7 @@ -from typing import Any, TypeVar, Optional +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. -from pydantic import BaseModel +from typing import Any, Optional from microsoft_agents.activity import Activity from microsoft_agents.testing.utils import normalize_activity_data diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index 612ea5c9..7084464c 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -67,3 +67,24 @@ def check_field( if not operation: return False # missing operation for the assertion type return operation(actual_value, assertion) + +def check_field_verbose( + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType +) -> tuple[bool, Optional[str]]: + """Checks if the actual value satisfies the given assertion based on the assertion type. + + :param actual_value: The value to be checked. + :param assertion: The expected value or pattern to check against. + :param assertion_type: The type of assertion to perform. + :return: A tuple containing a boolean indicating if the assertion is satisfied and an optional error message. + """ + + operation = _OPERATIONS.get(assertion_type) + if not operation: + return False, f"Missing operation for assertion type: {assertion_type}" + + result = operation(actual_value, assertion) + if result: + return True, None + else: + return False, f"Assertion failed: actual value '{actual_value}' does not satisfy '{assertion_type.name}' with assertion '{assertion}'" \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py index 844c3146..fd73d65d 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from enum import Enum from dataclasses import dataclass from typing import Any diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py index 3fe2a78f..80bb0402 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .generate_token import generate_token, generate_token_from_config __all__ = ["generate_token", "generate_token_from_config"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py index 83106639..f96f6d9e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import requests from microsoft_agents.hosting.core import AgentAuthConfiguration diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py index 3ad1e376..cdc0f0df 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .core import ( AgentClient, ApplicationRunner, diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py index 9c69a2ae..a1161336 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .application_runner import ApplicationRunner from .aiohttp import AiohttpEnvironment from .client import ( diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py index 82d2d1d0..4625620e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .aiohttp_environment import AiohttpEnvironment from .aiohttp_runner import AiohttpRunner diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py index 7ff83dc0..cd630697 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from aiohttp.web import Request, Response, Application from microsoft_agents.hosting.aiohttp import ( diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py index c8fe23c2..518c9b5a 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from typing import Optional from threading import Thread, Event import asyncio diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py index ebbc56f9..9c77d745 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import asyncio from abc import ABC, abstractmethod from typing import Any, Optional diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py index 1d59411e..7b778407 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .agent_client import AgentClient from .response_client import ResponseClient diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py index 73067207..0928dfff 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import json import asyncio from typing import Optional, cast diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py index b283efdf..e22a10df 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from __future__ import annotations import sys diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py index 2611da36..a351e735 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from abc import ABC, abstractmethod from typing import Awaitable, Callable diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py index 3b459617..1b0673f5 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import pytest import os diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py index 6dde3668..d97298cc 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from abc import ABC, abstractmethod from .environment import Environment diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py index c1824ae5..61e1def8 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import os from copy import deepcopy from dotenv import load_dotenv, dotenv_values diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py index 1875db71..fd1102ff 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .populate_activity import populate_activity from .misc import get_host_and_port, normalize_activity_data diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py index 3eac2dc0..ea6cda35 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from urllib.parse import urlparse from microsoft_agents.activity import Activity diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py index 6aa7a265..7d1edea8 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from microsoft_agents.activity import Activity diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py index d964ebd2..7cc8fe9e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from urllib.parse import urlparse From 6866f5dcc8a8b5c1333de38614d4484c8102f6dc Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 13 Nov 2025 08:50:53 -0800 Subject: [PATCH 67/81] DDT implementation outline --- .../testing/integration/data_driven_tester.py | 23 ++++++++ .../testing/integration/ddt.py | 58 +++++++++++++++++++ .../testing/integration/load_ddt.py | 4 ++ 3 files changed, 85 insertions(+) create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/load_ddt.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py index af36f9d8..d5c9e2f7 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py @@ -1,3 +1,6 @@ +import pytest + +from .data_driven_test import DataDrivenTest # from typing import Callable, TypeVar # from .core import Integration @@ -30,3 +33,23 @@ # return test_cls # return decorator + +class DataDrivenTester: + + _test_path: str + + @pytest.mark.asyncio + async def test_data_driven(self, agent_client, response_client) -> None: + + ddt = DataDrivenTest(self._test_path) + + responses = [] + + await for step in ddt: + if isinstance(step, Activity): + await agent_client.send_activity(step) + elif isinstance(step, dict): + # assertion + responses.extend(await response_client.pop()) + + \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py new file mode 100644 index 00000000..af01e7db --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py @@ -0,0 +1,58 @@ +import asyncio + +from copy import deepcopy +from typing import Awaitable, Callable + +from microsoft_agents.activity import Activity + +from microsoft_agents.testing.assertions import assert_activity + +class DataDrivenTest: + + def __init__(self, test_flow: dict) -> None: + self._description = test_flow.get("description", "") + + defaults = test_flow.get("defaults", {}) + self._input_defaults = defaults.get("input", {}) + self._assertion_defaults = defaults.get("assertion", {}) + self._sleep_defaults = defaults.get("sleep", {}) + + self._test = test_flow.get("test", []) + + def _load_input(self, input_data: dict) -> Activity: + data = deepcopy(self._input_defaults) + data.update(input_data) + return Activity.model_validate(data) + + def _load_assertion(self, assertion_data: dict) -> dict: + data = deepcopy(self._assertion_defaults) + data.update(assertion_data) + return data + + async def _sleep(self, sleep_data: dict) -> None: + duration = sleep_data.get("duration") + if duration is None: + duration = self._sleep_defaults.get("duration", 0) + await asyncio.sleep(duration) + + async def run(self, agent_client, response_client) -> None: + + for step in self._test: + step_type = step.get("type") + if not step_type: + raise ValueError("Each step must have a 'type' field.") + + if step_type == "input": + input_activity = self._load_input(step) + await agent_client.send_activity(input_activity) + elif step_type == "assertion": + assertion = self._load_assertion(step) + responses = await response_client.pop() + + selector = Selector(assertion.get("selector", {})) + selection = selector.select(responses) + + assert_activity(selection, assertion) + + elif step_type == "sleep": + await self._sleep(step) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/load_ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/load_ddt.py new file mode 100644 index 00000000..a69781b9 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/load_ddt.py @@ -0,0 +1,4 @@ +import pyyaml + +def load_ddt(): + pass \ No newline at end of file From 1db2bf5531d3616c3052f414a5cd3df50ed363bf Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 13 Nov 2025 09:47:41 -0800 Subject: [PATCH 68/81] DataDrivenTest class --- .../data_driven_tests/basic_test.yaml | 22 +++++++ .../data_driven_tests/defaults.yaml | 10 ++++ .../integration/data_driven/__init__.py | 7 +++ .../data_driven/data_driven_test.py | 60 +++++++++++++++++++ .../testing/integration/data_driven/ddt.py | 41 +++++++++++++ .../testing/integration/data_driven_test.py | 39 ------------ .../testing/integration/data_driven_tester.py | 55 ----------------- .../testing/integration/load_ddt.py | 4 -- tests/activity/test_sub_channels.py | 5 -- 9 files changed, 140 insertions(+), 103 deletions(-) create mode 100644 dev/integration/data_driven_tests/basic_test.yaml create mode 100644 dev/integration/data_driven_tests/defaults.yaml create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/integration/load_ddt.py delete mode 100644 tests/activity/test_sub_channels.py diff --git a/dev/integration/data_driven_tests/basic_test.yaml b/dev/integration/data_driven_tests/basic_test.yaml new file mode 100644 index 00000000..7e88db72 --- /dev/null +++ b/dev/integration/data_driven_tests/basic_test.yaml @@ -0,0 +1,22 @@ +parent: ./defaults.yaml +description: A basic data driven test example +defaults: + input: + sleep: + assertion: +test: + - type: input + activity: + type: message + text: "Hello, World!" + + - type: sleep + duration: 1 + + - type: assertion + selector: + activity_index: -1 + type: message + activity: + type: message + text: "Hello, World!" \ No newline at end of file diff --git a/dev/integration/data_driven_tests/defaults.yaml b/dev/integration/data_driven_tests/defaults.yaml new file mode 100644 index 00000000..6d65a3b3 --- /dev/null +++ b/dev/integration/data_driven_tests/defaults.yaml @@ -0,0 +1,10 @@ +defaults: + all: + channel_id: "test_channel" + service_url: "http://localhost:3978" + locale: "en-US" + conversation_id: "test_conversation" + input: + user_id: "test_user" + output: + user_id: "test_bot" \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py new file mode 100644 index 00000000..fd6756ca --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py @@ -0,0 +1,7 @@ +from .data_driven_test import DataDrivenTest +from .ddt import ddt + +__all__ = [ + "DataDrivenTest", + "ddt" +] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py new file mode 100644 index 00000000..a50c2cd1 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License.s + +import asyncio + +from copy import deepcopy + +from microsoft_agents.activity import Activity + +from microsoft_agents.testing.assertions import assert_activity + +class DataDrivenTest: + + def __init__(self, test_flow: dict) -> None: + self._description = test_flow.get("description", "") + + defaults = test_flow.get("defaults", {}) + self._input_defaults = defaults.get("input", {}) + self._assertion_defaults = defaults.get("assertion", {}) + self._sleep_defaults = defaults.get("sleep", {}) + + self._test = test_flow.get("test", []) + + def _load_input(self, input_data: dict) -> Activity: + data = deepcopy(self._input_defaults) + data.update(input_data) + return Activity.model_validate(data) + + def _load_assertion(self, assertion_data: dict) -> dict: + data = deepcopy(self._assertion_defaults) + data.update(assertion_data) + return data + + async def _sleep(self, sleep_data: dict) -> None: + duration = sleep_data.get("duration") + if duration is None: + duration = self._sleep_defaults.get("duration", 0) + await asyncio.sleep(duration) + + async def run(self, agent_client, response_client) -> None: + + for step in self._test: + step_type = step.get("type") + if not step_type: + raise ValueError("Each step must have a 'type' field.") + + if step_type == "input": + input_activity = self._load_input(step) + await agent_client.send_activity(input_activity) + elif step_type == "assertion": + assertion = self._load_assertion(step) + responses = await response_client.pop() + + selector = Selector(assertion.get("selector", {})) + selection = selector.select(responses) + + assert_activity(selection, assertion) + + elif step_type == "sleep": + await self._sleep(step) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py new file mode 100644 index 00000000..a627e7e3 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py @@ -0,0 +1,41 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Callable + +import pytest + +from microsoft_agents.testing.integration.core import Integration + +from .data_driven_test import DataDrivenTest + +def _add_test_method(test_cls: Integration, test_path: str, base_dir: str) -> None: + + test_case_name = f"test_data_driven__{test_path.replace('/', '_').replace('.', '_')}" + + @pytest.mark.asyncio + async def _func(self, agent_client, response_client) -> None: + ddt = DataDrivenTest(f"{base_dir}/{test_path}") + await ddt.run(agent_client, response_client) + + setattr(test_cls, test_case_name, func) + +def ddt(test_path: str) -> Callable[[Integration], Integration]: + + def decorator(test_cls: Integration) -> Integration: + + test_case_name = f"test_data_driven__{test_path.replace('/', '_').replace('.', '_')}" + + async def func(self, agent_client, response_client) -> None: + ddt = DataDrivenTest(test_path) + + responses = [] + + await for step in ddt: + if isinstance(step, Activity): + await agent_client.send_activity(step) + elif isinstance(step, dict): + # assertion + responses.extend(await response_client.pop()) + + return decorator \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py deleted file mode 100644 index 0e13db4a..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# import json -# import asyncio - -# from microsoft_agents.activity import Activity -# from microsoft_agents.testing.core import ( -# AgentClient, -# ResponseClient -# ) - -# from microsoft_agents.testing.asserts import check_activity - -# class DataDrivenTestModule: - -# def __init__( -# self, -# test_file_path: str, -# agent_client: AgentClient, -# response_client: ResponseClient -# ) -> None: - -# data = json.load(open(test_file_path, "r")) - -# self._agent_client = agent_client -# self._response_client = response_client - -# self._input_activities: list[Activity] = [] -# self._activity_assertions: list[dict] = [] - -# async def assert(self) -> bool: - -# for input_activity in self._input_activities: -# await self._agent_client.send_activity(input_activity) - -# await asyncio.sleep(1) - -# for activity_assertion in self._activity_assertions: -# # select first -# selected_activity = None -# check_activity(selected_activity, activity_assertion) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py deleted file mode 100644 index d5c9e2f7..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven_tester.py +++ /dev/null @@ -1,55 +0,0 @@ -import pytest - -from .data_driven_test import DataDrivenTest -# from typing import Callable, TypeVar - -# from .core import Integration - -# async def run_data_driven_test(input_file: str) -> None: -# """ -# Run data-driven tests based on the provided input file. -# """ -# pass - -# T = TypeVar("T", bound=Integration) - -# def factory(tests_path: str = "./") -> Callable[T, T]: - -# # for file in file - -# files = [] - -# def decorator(test_cls: T) -> T: - -# for file_name in files: - -# test_case_name = f"test_data_driven__{file_name}" - -# def func(self, agent_client, response_client) -> None: - - -# setattr(test_cls, test_case_name, func) - -# return test_cls - -# return decorator - -class DataDrivenTester: - - _test_path: str - - @pytest.mark.asyncio - async def test_data_driven(self, agent_client, response_client) -> None: - - ddt = DataDrivenTest(self._test_path) - - responses = [] - - await for step in ddt: - if isinstance(step, Activity): - await agent_client.send_activity(step) - elif isinstance(step, dict): - # assertion - responses.extend(await response_client.pop()) - - \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/load_ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/load_ddt.py deleted file mode 100644 index a69781b9..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/load_ddt.py +++ /dev/null @@ -1,4 +0,0 @@ -import pyyaml - -def load_ddt(): - pass \ No newline at end of file diff --git a/tests/activity/test_sub_channels.py b/tests/activity/test_sub_channels.py deleted file mode 100644 index 67c6d92c..00000000 --- a/tests/activity/test_sub_channels.py +++ /dev/null @@ -1,5 +0,0 @@ -from microsoft_agents.activity import Activity, ChannelId, Entity - - -class TestSubChannels: - pass From e44a2da07b9d7d4d2da2028801d006e5dd945a7f Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 13 Nov 2025 12:10:44 -0800 Subject: [PATCH 69/81] select_activity implementation for assertions --- .../data_driven_tests/basic_test.yaml | 6 +- .../testing/assertions/assertions.py | 12 ++-- .../testing/assertions/select_activity.py | 61 ++++++++++++++++++- .../testing/assertions/type_defs.py | 14 +++++ 4 files changed, 83 insertions(+), 10 deletions(-) diff --git a/dev/integration/data_driven_tests/basic_test.yaml b/dev/integration/data_driven_tests/basic_test.yaml index 7e88db72..3f416936 100644 --- a/dev/integration/data_driven_tests/basic_test.yaml +++ b/dev/integration/data_driven_tests/basic_test.yaml @@ -15,8 +15,10 @@ test: - type: assertion selector: - activity_index: -1 - type: message + index: -1 + quantifier: ONE # implied + activity: + type: message activity: type: message text: "Hello, World!" \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py index 5127cecd..5c8efb18 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -10,24 +10,24 @@ from .check_field import check_field_verbose -def assert_activity(activity: Activity, baseline: Activity | dict) -> None: +def assert_activity(activity: Activity, assertion: Activity | dict) -> None: """Asserts that the given activity matches the baseline activity. :param activity: The activity to be tested. - :param baseline: The baseline activity or a dictionary representing the expected activity data. + :param assertion: The baseline activity or a dictionary representing the expected activity data. """ - res, assertion_error_data = check_activity_verbose(activity, baseline) + res, assertion_error_data = check_activity_verbose(activity, assertion) assert res, str(assertion_error_data) def assert_field( - actual_value: Any, baseline_value: Any, assertion_type: FieldAssertionType + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType ) -> None: """Asserts that a specific field in the target matches the baseline. :param key_in_baseline: The key of the field to be tested. :param target: The target dictionary containing the actual values. - :param baseline: The baseline dictionary containing the expected values. + :param assertion: The baseline dictionary containing the expected values. """ - res, assertion_error_message = check_field_verbose(actual_value, baseline_value, assertion_type) + res, assertion_error_message = check_field_verbose(actual_value, assertion, assertion_type) assert res, assertion_error_message \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py index 85a2b03b..24bec0ab 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py @@ -1,2 +1,59 @@ -def select_activity() -> bool: - return False +from typing import Optional + +from microsoft_agents.activity import Activity + +from .check_activity import check_activity +from .type_defs import SelectorQuantifier + +def select_activity( + activities: list[Activity], + selector: Activity, + index: int = 0, +) -> Optional[Activity]: + """Selects a single activity from a list based on the provided selector and index. + + :param activities: List of activities to select from. + :param selector: Activity used as a selector. + :param index: Index of the activity to select when multiple match. + :return: The selected activity or None if no activity matches. + """ + res = select_activities(activities, { + "selector": selector, + "quantifier": SelectorQuantifier.ONE, + "index": index + } + ) + return res[index] if res else None + +def select_activities( + activities: list[Activity], + selector_config: dict +) -> list[Activity]: + """Selects activities from a list based on the provided selector configuration. + + :param activities: List of activities to select from. + :param selector_config: Configuration dict containing 'selector', 'quantifier', and optionally ' + :return: List of selected activities. + """ + + selector = selector_config.get("selector", {}) + + index = selector_config.get("index", None) + if index is not None: + quantifier_name = selector_config.get("quantifier", SelectorQuantifier.ONE) + else: + quantifier_name = selector_config.get("quantifier", SelectorQuantifier.ALL) + quantifier_name = quantifier_name.upper() + + if quantifier_name not in SelectorQuantifier.__members__: + raise ValueError(f"Invalid quantifier: {quantifier_name}") + + quantifier = SelectorQuantifier(quantifier_name) + + if quantifier == SelectorQuantifier.ALL: + return list( + filter(lambda a: check_activity(a, selector), activities) + ) + else: + first = next(filter(lambda a: check_activity(a, selector), activities), None) + return [first] if first is not None else [] \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py index fd73d65d..fc1060bd 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py @@ -26,6 +26,20 @@ class FieldAssertionType(str, Enum): NOT_IN = "NOT_IN" RE_MATCH = "RE_MATCH" +class SelectorQuantifier(str, Enum): + """Defines quantifiers for selecting activities.""" + + ALL = "ALL" + ONE = "ONE" + +class AssertionQuantifier(str, Enum): + """Defines quantifiers for assertions on activities.""" + + ANY = "ANY" + ALL = "ALL" + ONE = "ONE" + NONE = "NONE" + @dataclass class AssertionErrorData: """Data class to hold information about assertion errors.""" From 1cf561baceaa72fa800edeefe7417d982a6ad382 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 13 Nov 2025 12:33:47 -0800 Subject: [PATCH 70/81] inverted assertions for quantifiers --- .../data_driven_tests/basic_test.yaml | 3 +- .../testing/assertions/activity_assertion.py | 52 +++++++++++++++++++ .../testing/assertions/assertions.py | 35 +++++++++---- .../testing/assertions/check_activity.py | 29 ++++++++--- .../testing/assertions/check_field.py | 16 +++--- 5 files changed, 109 insertions(+), 26 deletions(-) create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py diff --git a/dev/integration/data_driven_tests/basic_test.yaml b/dev/integration/data_driven_tests/basic_test.yaml index 3f416936..a9199fbb 100644 --- a/dev/integration/data_driven_tests/basic_test.yaml +++ b/dev/integration/data_driven_tests/basic_test.yaml @@ -14,9 +14,10 @@ test: duration: 1 - type: assertion + quantifier: ONE selector: index: -1 - quantifier: ONE # implied + quantifier: ONE activity: type: message activity: diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py new file mode 100644 index 00000000..210bdbc1 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py @@ -0,0 +1,52 @@ +from typing import Optional + +from microsoft_agents.activity import Activity + +from .select_activity import select_activities +from .type_defs import ( + AssertionQuantifier, + SelectorQuantifier, + FieldAssertionType, + AssertionErrorData +) + +class ActivityAssertion: + + def __init__(self, config: dict) -> None: + quantifier_name = config.get("quantifier", AssertionQuantifier.ALL) + self._quantifier = AssertionQuantifier(quantifier_name) + + self._selector = config.get("selector", {}) + self._assertion = config.get("assertion", {}) + + def check(self, activities: list[Activity]) -> tuple[bool, list[AssertionErrorData]]: + + activities = select_activities(activities, self._selector) + + invert = self._quantifier == AssertionQuantifier.NONE + + count = 0 + assertion_error_data_list: list[AssertionErrorData] = [] + for activity in activities: + res, assertion_error_data = check_activity_verbose(activity, self._assertion, invert=invert) + if self._quantifier == AssertionQuantifier.ALL and not res: + return False, assertion_error_data + if self._quantifier == AssertionQuantifier.ANY and res: + count += 1 + + if self._quantifier == AssertionQuantifier.ANY: + return count > 0, assertion_error_data_list + if self._quantifier == AssertionQuantifier.ONE: + return count == 1, assertion_error_data_list + if self._quantifier == AssertionQuantifier.NONE: + return count == 0, assertion_error_data_list + + + + + + + def assert(self, activities: list[Activity]) -> None: + res, assertion_error_data = self.check(activities) + assertion_error_message = "\n".join(str(err) for err in assertion_error_data) + assert res, assertion_error_message \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py index 5c8efb18..00334b59 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -5,10 +5,21 @@ from microsoft_agents.activity import Activity -from .type_defs import FieldAssertionType +from .type_defs import FieldAssertionType, AssertionQuantifier from .check_activity import check_activity_verbose from .check_field import check_field_verbose +def assert_field( + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType +) -> None: + """Asserts that a specific field in the target matches the baseline. + + :param key_in_baseline: The key of the field to be tested. + :param target: The target dictionary containing the actual values. + :param assertion: The baseline dictionary containing the expected values. + """ + res, assertion_error_message = check_field_verbose(actual_value, assertion, assertion_type) + assert res, assertion_error_message def assert_activity(activity: Activity, assertion: Activity | dict) -> None: """Asserts that the given activity matches the baseline activity. @@ -19,15 +30,17 @@ def assert_activity(activity: Activity, assertion: Activity | dict) -> None: res, assertion_error_data = check_activity_verbose(activity, assertion) assert res, str(assertion_error_data) +def assert_activities(activities: list[Activity], assertion_config: dict) -> None: + """Asserts that the given list of activities matches the baseline activities. -def assert_field( - actual_value: Any, assertion: Any, assertion_type: FieldAssertionType -) -> None: - """Asserts that a specific field in the target matches the baseline. - - :param key_in_baseline: The key of the field to be tested. - :param target: The target dictionary containing the actual values. - :param assertion: The baseline dictionary containing the expected values. + :param activities: The list of activities to be tested. + :param assertion: The baseline dictionary representing the expected activities data. """ - res, assertion_error_message = check_field_verbose(actual_value, assertion, assertion_type) - assert res, assertion_error_message \ No newline at end of file + + quantifier = assertion_config.get("quantifier", ) + selector = assertion_config.get("selector", {}) + + + for activity in activities: + res, assertion_error_data = check_activity_verbose(activity, assertion) + assert res, str(assertion_error_data) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index 612b7b71..054b0265 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -9,7 +9,12 @@ from .check_field import check_field, _parse_assertion from .type_defs import UNSET_FIELD, FieldAssertionType, AssertionErrorData -def _check(actual: Any, baseline: Any, field_path: str = "") -> tuple[bool, Optional[AssertionErrorData]]: +def _check( + actual: Any, + baseline: Any, + invert: bool, + field_path: str = "" + ) -> tuple[bool, Optional[AssertionErrorData]]: assertion, assertion_type = _parse_assertion(baseline) @@ -20,7 +25,7 @@ def _check(actual: Any, baseline: Any, field_path: str = "") -> tuple[bool, Opti new_actual = actual.get(key, UNSET_FIELD) new_baseline = baseline[key] - res, assertion_error_data = _check(new_actual, new_baseline, new_field_path) + res, assertion_error_data = _check(new_actual, new_baseline, invert, new_field_path) if not res: return False, assertion_error_data return True, None @@ -31,7 +36,7 @@ def _check(actual: Any, baseline: Any, field_path: str = "") -> tuple[bool, Opti new_actual = actual[index] if index < len(actual) else UNSET_FIELD new_baseline = item - res, assertion_error_data = _check(new_actual, new_baseline, new_field_path) + res, assertion_error_data = _check(new_actual, new_baseline, invert, new_field_path) if not res: return False, assertion_error_data return True, None @@ -39,7 +44,7 @@ def _check(actual: Any, baseline: Any, field_path: str = "") -> tuple[bool, Opti raise ValueError("Unsupported baseline type for complex assertion.") else: assert isinstance(assertion_type, FieldAssertionType) - res = check_field(actual, assertion, assertion_type) + res = check_field(actual, assertion, assertion_type, invert=invert) if res: return True, None else: @@ -51,15 +56,23 @@ def _check(actual: Any, baseline: Any, field_path: str = "") -> tuple[bool, Opti ) return False, assertion_error_data -def check_activity(activity: Activity, baseline: Activity | dict) -> bool: +def check_activity( + activity: Activity, + baseline: Activity | dict, + invert: bool = False + ) -> bool: """Asserts that the given activity matches the baseline activity. :param activity: The activity to be tested. :param baseline: The baseline activity or a dictionary representing the expected activity data. """ - return check_activity_verbose(activity, baseline)[0] + return check_activity_verbose(activity, baseline, invert=invert)[0] -def check_activity_verbose(activity: Activity, baseline: Activity | dict) -> tuple[bool, Optional[AssertionErrorData]]: +def check_activity_verbose( + activity: Activity, + baseline: Activity | dict, + invert: bool = False + ) -> tuple[bool, Optional[AssertionErrorData]]: """Asserts that the given activity matches the baseline activity. :param activity: The activity to be tested. @@ -67,4 +80,4 @@ def check_activity_verbose(activity: Activity, baseline: Activity | dict) -> tup """ actual_activity = normalize_activity_data(activity) baseline = normalize_activity_data(baseline) - return _check(actual_activity, baseline, "activity") \ No newline at end of file + return _check(actual_activity, baseline, invert, "activity") \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index 7084464c..cefcb59c 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -53,38 +53,42 @@ def _parse_assertion(field: Any) -> tuple[Any, Optional[FieldAssertionType]]: return assertion, assertion_type def check_field( - actual_value: Any, assertion: Any, assertion_type: FieldAssertionType + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType, invert: bool = False ) -> bool: """Checks if the actual value satisfies the given assertion based on the assertion type. :param actual_value: The value to be checked. :param assertion: The expected value or pattern to check against. :param assertion_type: The type of assertion to perform. + :param invert: Whether to invert the result of the assertion. :return: True if the assertion is satisfied, False otherwise. """ operation = _OPERATIONS.get(assertion_type) if not operation: - return False # missing operation for the assertion type - return operation(actual_value, assertion) + raise ValueError(f"Missing operation for assertion type: {assertion_type}") + return operation(actual_value, assertion) ^ invert def check_field_verbose( - actual_value: Any, assertion: Any, assertion_type: FieldAssertionType + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType, invert: bool = False ) -> tuple[bool, Optional[str]]: """Checks if the actual value satisfies the given assertion based on the assertion type. :param actual_value: The value to be checked. :param assertion: The expected value or pattern to check against. :param assertion_type: The type of assertion to perform. + :param invert: Whether to invert the result of the assertion. :return: A tuple containing a boolean indicating if the assertion is satisfied and an optional error message. """ operation = _OPERATIONS.get(assertion_type) if not operation: - return False, f"Missing operation for assertion type: {assertion_type}" + raise ValueError(f"Missing operation for assertion type: {assertion_type}") - result = operation(actual_value, assertion) + result = operation(actual_value, assertion) ^ invert if result: return True, None else: + if invert: + return False, f"INVERTED Assertion failed: actual value '{actual_value}' satisfies '{assertion_type.name}' with assertion '{assertion}'" return False, f"Assertion failed: actual value '{actual_value}' does not satisfy '{assertion_type.name}' with assertion '{assertion}'" \ No newline at end of file From e651f92c3866e755d2eedc6fcad6cc21e393c6db Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 13 Nov 2025 12:44:53 -0800 Subject: [PATCH 71/81] ActivityAssertion class implementation --- .../testing/assertions/activity_assertion.py | 54 ++++++++++--------- .../testing/assertions/check_activity.py | 31 ++++------- .../testing/assertions/check_field.py | 16 +++--- 3 files changed, 44 insertions(+), 57 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py index 210bdbc1..cf9e9adc 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py @@ -3,50 +3,54 @@ from microsoft_agents.activity import Activity from .select_activity import select_activities +from .check_activity import check_activity_verbose from .type_defs import ( AssertionQuantifier, - SelectorQuantifier, - FieldAssertionType, AssertionErrorData ) class ActivityAssertion: def __init__(self, config: dict) -> None: + """Initializes the ActivityAssertion with the given configuration. + + :param config: The configuration dictionary containing quantifier, selector, and assertion. + """ quantifier_name = config.get("quantifier", AssertionQuantifier.ALL) self._quantifier = AssertionQuantifier(quantifier_name) self._selector = config.get("selector", {}) self._assertion = config.get("assertion", {}) - def check(self, activities: list[Activity]) -> tuple[bool, list[AssertionErrorData]]: + @staticmethod + def _combine_assertion_errors(errors: list[AssertionErrorData]) -> str: + """Combines multiple assertion errors into a single string representation. - activities = select_activities(activities, self._selector) + :param errors: The list of assertion errors to be combined. + :return: A string representation of the combined assertion errors. + """ + return "\n".join(str(error) for error in errors) - invert = self._quantifier == AssertionQuantifier.NONE + def check(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: + """Asserts that the given activities match the assertion criteria. + + :param activities: The list of activities to be tested. + :return: A tuple containing a boolean indicating if the assertion passed and an optional error message. + """ + + activities = select_activities(activities, self._selector) count = 0 - assertion_error_data_list: list[AssertionErrorData] = [] for activity in activities: - res, assertion_error_data = check_activity_verbose(activity, self._assertion, invert=invert) + res, assertion_error_data = check_activity_verbose(activity, self._assertion) if self._quantifier == AssertionQuantifier.ALL and not res: - return False, assertion_error_data - if self._quantifier == AssertionQuantifier.ANY and res: - count += 1 - - if self._quantifier == AssertionQuantifier.ANY: - return count > 0, assertion_error_data_list - if self._quantifier == AssertionQuantifier.ONE: - return count == 1, assertion_error_data_list - if self._quantifier == AssertionQuantifier.NONE: - return count == 0, assertion_error_data_list - - - - + return False, f"Activity did not match the assertion: {activity}\nError: {assertion_error_data}" + if self._quantifier == AssertionQuantifier.NONE and res: + return False, f"Activity matched the assertion when none were expected: {activity}" + count += 1 + passes = True + if self._quantifier == AssertionQuantifier.ONE and count != 1: + return False, f"Expected exactly one activity to match the assertion, but found {count}." - def assert(self, activities: list[Activity]) -> None: - res, assertion_error_data = self.check(activities) - assertion_error_message = "\n".join(str(err) for err in assertion_error_data) - assert res, assertion_error_message \ No newline at end of file + return passes, None \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index 054b0265..6ad404e2 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from typing import Any, Optional +from typing import Any, TypeVar, Optional from microsoft_agents.activity import Activity from microsoft_agents.testing.utils import normalize_activity_data @@ -9,12 +9,7 @@ from .check_field import check_field, _parse_assertion from .type_defs import UNSET_FIELD, FieldAssertionType, AssertionErrorData -def _check( - actual: Any, - baseline: Any, - invert: bool, - field_path: str = "" - ) -> tuple[bool, Optional[AssertionErrorData]]: +def _check(actual: Any, baseline: Any, field_path: str = "") -> tuple[bool, Optional[AssertionErrorData]]: assertion, assertion_type = _parse_assertion(baseline) @@ -25,7 +20,7 @@ def _check( new_actual = actual.get(key, UNSET_FIELD) new_baseline = baseline[key] - res, assertion_error_data = _check(new_actual, new_baseline, invert, new_field_path) + res, assertion_error_data = _check(new_actual, new_baseline, new_field_path) if not res: return False, assertion_error_data return True, None @@ -36,7 +31,7 @@ def _check( new_actual = actual[index] if index < len(actual) else UNSET_FIELD new_baseline = item - res, assertion_error_data = _check(new_actual, new_baseline, invert, new_field_path) + res, assertion_error_data = _check(new_actual, new_baseline, new_field_path) if not res: return False, assertion_error_data return True, None @@ -44,7 +39,7 @@ def _check( raise ValueError("Unsupported baseline type for complex assertion.") else: assert isinstance(assertion_type, FieldAssertionType) - res = check_field(actual, assertion, assertion_type, invert=invert) + res = check_field(actual, assertion, assertion_type) if res: return True, None else: @@ -56,23 +51,15 @@ def _check( ) return False, assertion_error_data -def check_activity( - activity: Activity, - baseline: Activity | dict, - invert: bool = False - ) -> bool: +def check_activity(activity: Activity, baseline: Activity | dict) -> bool: """Asserts that the given activity matches the baseline activity. :param activity: The activity to be tested. :param baseline: The baseline activity or a dictionary representing the expected activity data. """ - return check_activity_verbose(activity, baseline, invert=invert)[0] + return check_activity_verbose(activity, baseline)[0] -def check_activity_verbose( - activity: Activity, - baseline: Activity | dict, - invert: bool = False - ) -> tuple[bool, Optional[AssertionErrorData]]: +def check_activity_verbose(activity: Activity, baseline: Activity | dict) -> tuple[bool, Optional[AssertionErrorData]]: """Asserts that the given activity matches the baseline activity. :param activity: The activity to be tested. @@ -80,4 +67,4 @@ def check_activity_verbose( """ actual_activity = normalize_activity_data(activity) baseline = normalize_activity_data(baseline) - return _check(actual_activity, baseline, invert, "activity") \ No newline at end of file + return _check(actual_activity, baseline, "activity") \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index cefcb59c..2ae2ef0f 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -53,42 +53,38 @@ def _parse_assertion(field: Any) -> tuple[Any, Optional[FieldAssertionType]]: return assertion, assertion_type def check_field( - actual_value: Any, assertion: Any, assertion_type: FieldAssertionType, invert: bool = False + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType ) -> bool: """Checks if the actual value satisfies the given assertion based on the assertion type. :param actual_value: The value to be checked. :param assertion: The expected value or pattern to check against. :param assertion_type: The type of assertion to perform. - :param invert: Whether to invert the result of the assertion. :return: True if the assertion is satisfied, False otherwise. """ operation = _OPERATIONS.get(assertion_type) if not operation: - raise ValueError(f"Missing operation for assertion type: {assertion_type}") - return operation(actual_value, assertion) ^ invert + raise ValueError(f"Unsupported assertion type: {assertion_type}") + return operation(actual_value, assertion) def check_field_verbose( - actual_value: Any, assertion: Any, assertion_type: FieldAssertionType, invert: bool = False + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType ) -> tuple[bool, Optional[str]]: """Checks if the actual value satisfies the given assertion based on the assertion type. :param actual_value: The value to be checked. :param assertion: The expected value or pattern to check against. :param assertion_type: The type of assertion to perform. - :param invert: Whether to invert the result of the assertion. :return: A tuple containing a boolean indicating if the assertion is satisfied and an optional error message. """ operation = _OPERATIONS.get(assertion_type) if not operation: - raise ValueError(f"Missing operation for assertion type: {assertion_type}") + raise ValueError(f"Unsupported assertion type: {assertion_type}") - result = operation(actual_value, assertion) ^ invert + result = operation(actual_value, assertion) if result: return True, None else: - if invert: - return False, f"INVERTED Assertion failed: actual value '{actual_value}' satisfies '{assertion_type.name}' with assertion '{assertion}'" return False, f"Assertion failed: actual value '{actual_value}' does not satisfy '{assertion_type.name}' with assertion '{assertion}'" \ No newline at end of file From ed7f2f17658907118e03fe07384f18bab724d823 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 13 Nov 2025 12:48:21 -0800 Subject: [PATCH 72/81] DataDrivenTest implementation --- .../testing/assertions/__init__.py | 3 +- .../testing/assertions/activity_assertion.py | 35 +++-- .../testing/assertions/assertions.py | 14 +- .../testing/assertions/check_activity.py | 27 +++- .../testing/assertions/check_field.py | 17 ++- .../testing/assertions/select_activity.py | 29 ++-- .../testing/assertions/type_defs.py | 9 +- .../integration/data_driven/__init__.py | 5 +- .../data_driven/data_driven_test.py | 27 ++-- .../testing/integration/ddt.py | 13 +- .../tests/assertions/test_assert_activity.py | 139 ++++++------------ .../tests/assertions/test_check_field.py | 31 +++- 12 files changed, 177 insertions(+), 172 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py index 462bcd9f..9e8695be 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py @@ -1,13 +1,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +from .activity_assertion import ActivityAssertion from .assertions import ( assert_activity, assert_field, ) from .check_activity import check_activity from .check_field import check_field -from .type_defs import FieldAssertionType +from .type_defs import FieldAssertionType, SelectorQuantifier, AssertionQuantifier __all__ = [ "assert_activity", diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py index cf9e9adc..57299304 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py @@ -4,16 +4,14 @@ from .select_activity import select_activities from .check_activity import check_activity_verbose -from .type_defs import ( - AssertionQuantifier, - AssertionErrorData -) +from .type_defs import AssertionQuantifier, AssertionErrorData + class ActivityAssertion: def __init__(self, config: dict) -> None: """Initializes the ActivityAssertion with the given configuration. - + :param config: The configuration dictionary containing quantifier, selector, and assertion. """ quantifier_name = config.get("quantifier", AssertionQuantifier.ALL) @@ -25,7 +23,7 @@ def __init__(self, config: dict) -> None: @staticmethod def _combine_assertion_errors(errors: list[AssertionErrorData]) -> str: """Combines multiple assertion errors into a single string representation. - + :param errors: The list of assertion errors to be combined. :return: A string representation of the combined assertion errors. """ @@ -33,24 +31,35 @@ def _combine_assertion_errors(errors: list[AssertionErrorData]) -> str: def check(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: """Asserts that the given activities match the assertion criteria. - + :param activities: The list of activities to be tested. :return: A tuple containing a boolean indicating if the assertion passed and an optional error message. """ - + activities = select_activities(activities, self._selector) count = 0 for activity in activities: - res, assertion_error_data = check_activity_verbose(activity, self._assertion) + res, assertion_error_data = check_activity_verbose( + activity, self._assertion + ) if self._quantifier == AssertionQuantifier.ALL and not res: - return False, f"Activity did not match the assertion: {activity}\nError: {assertion_error_data}" + return ( + False, + f"Activity did not match the assertion: {activity}\nError: {assertion_error_data}", + ) if self._quantifier == AssertionQuantifier.NONE and res: - return False, f"Activity matched the assertion when none were expected: {activity}" + return ( + False, + f"Activity matched the assertion when none were expected: {activity}", + ) count += 1 passes = True if self._quantifier == AssertionQuantifier.ONE and count != 1: - return False, f"Expected exactly one activity to match the assertion, but found {count}." + return ( + False, + f"Expected exactly one activity to match the assertion, but found {count}.", + ) - return passes, None \ No newline at end of file + return passes, None diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py index 00334b59..f778b191 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -9,6 +9,7 @@ from .check_activity import check_activity_verbose from .check_field import check_field_verbose + def assert_field( actual_value: Any, assertion: Any, assertion_type: FieldAssertionType ) -> None: @@ -18,9 +19,12 @@ def assert_field( :param target: The target dictionary containing the actual values. :param assertion: The baseline dictionary containing the expected values. """ - res, assertion_error_message = check_field_verbose(actual_value, assertion, assertion_type) + res, assertion_error_message = check_field_verbose( + actual_value, assertion, assertion_type + ) assert res, assertion_error_message + def assert_activity(activity: Activity, assertion: Activity | dict) -> None: """Asserts that the given activity matches the baseline activity. @@ -30,6 +34,7 @@ def assert_activity(activity: Activity, assertion: Activity | dict) -> None: res, assertion_error_data = check_activity_verbose(activity, assertion) assert res, str(assertion_error_data) + def assert_activities(activities: list[Activity], assertion_config: dict) -> None: """Asserts that the given list of activities matches the baseline activities. @@ -37,10 +42,11 @@ def assert_activities(activities: list[Activity], assertion_config: dict) -> Non :param assertion: The baseline dictionary representing the expected activities data. """ - quantifier = assertion_config.get("quantifier", ) + quantifier = assertion_config.get( + "quantifier", + ) selector = assertion_config.get("selector", {}) - for activity in activities: res, assertion_error_data = check_activity_verbose(activity, assertion) - assert res, str(assertion_error_data) \ No newline at end of file + assert res, str(assertion_error_data) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index 6ad404e2..ba286c22 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -9,7 +9,10 @@ from .check_field import check_field, _parse_assertion from .type_defs import UNSET_FIELD, FieldAssertionType, AssertionErrorData -def _check(actual: Any, baseline: Any, field_path: str = "") -> tuple[bool, Optional[AssertionErrorData]]: + +def _check( + actual: Any, baseline: Any, field_path: str = "" +) -> tuple[bool, Optional[AssertionErrorData]]: assertion, assertion_type = _parse_assertion(baseline) @@ -20,18 +23,24 @@ def _check(actual: Any, baseline: Any, field_path: str = "") -> tuple[bool, Opti new_actual = actual.get(key, UNSET_FIELD) new_baseline = baseline[key] - res, assertion_error_data = _check(new_actual, new_baseline, new_field_path) + res, assertion_error_data = _check( + new_actual, new_baseline, new_field_path + ) if not res: return False, assertion_error_data return True, None - + elif isinstance(baseline, list): for index, item in enumerate(baseline): - new_field_path = f"{field_path}[{index}]" if field_path else f"[{index}]" + new_field_path = ( + f"{field_path}[{index}]" if field_path else f"[{index}]" + ) new_actual = actual[index] if index < len(actual) else UNSET_FIELD new_baseline = item - res, assertion_error_data = _check(new_actual, new_baseline, new_field_path) + res, assertion_error_data = _check( + new_actual, new_baseline, new_field_path + ) if not res: return False, assertion_error_data return True, None @@ -51,6 +60,7 @@ def _check(actual: Any, baseline: Any, field_path: str = "") -> tuple[bool, Opti ) return False, assertion_error_data + def check_activity(activity: Activity, baseline: Activity | dict) -> bool: """Asserts that the given activity matches the baseline activity. @@ -59,7 +69,10 @@ def check_activity(activity: Activity, baseline: Activity | dict) -> bool: """ return check_activity_verbose(activity, baseline)[0] -def check_activity_verbose(activity: Activity, baseline: Activity | dict) -> tuple[bool, Optional[AssertionErrorData]]: + +def check_activity_verbose( + activity: Activity, baseline: Activity | dict +) -> tuple[bool, Optional[AssertionErrorData]]: """Asserts that the given activity matches the baseline activity. :param activity: The activity to be tested. @@ -67,4 +80,4 @@ def check_activity_verbose(activity: Activity, baseline: Activity | dict) -> tup """ actual_activity = normalize_activity_data(activity) baseline = normalize_activity_data(baseline) - return _check(actual_activity, baseline, "activity") \ No newline at end of file + return _check(actual_activity, baseline, "activity") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index 2ae2ef0f..77c8938c 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -5,7 +5,8 @@ _OPERATIONS = { FieldAssertionType.EQUALS: lambda a, b: a == b or (a is UNSET_FIELD and b is None), - FieldAssertionType.NOT_EQUALS: lambda a, b: a != b or (a is UNSET_FIELD and b is not None), + FieldAssertionType.NOT_EQUALS: lambda a, b: a != b + or (a is UNSET_FIELD and b is not None), FieldAssertionType.GREATER_THAN: lambda a, b: a > b, FieldAssertionType.LESS_THAN: lambda a, b: a < b, FieldAssertionType.CONTAINS: lambda a, b: b in a, @@ -13,6 +14,7 @@ FieldAssertionType.RE_MATCH: lambda a, b: re.match(b, a) is not None, } + def _parse_assertion(field: Any) -> tuple[Any, Optional[FieldAssertionType]]: """Parses the assertion information and returns the assertion type and baseline value. @@ -52,11 +54,12 @@ def _parse_assertion(field: Any) -> tuple[Any, Optional[FieldAssertionType]]: return assertion, assertion_type + def check_field( actual_value: Any, assertion: Any, assertion_type: FieldAssertionType ) -> bool: """Checks if the actual value satisfies the given assertion based on the assertion type. - + :param actual_value: The value to be checked. :param assertion: The expected value or pattern to check against. :param assertion_type: The type of assertion to perform. @@ -68,11 +71,12 @@ def check_field( raise ValueError(f"Unsupported assertion type: {assertion_type}") return operation(actual_value, assertion) + def check_field_verbose( actual_value: Any, assertion: Any, assertion_type: FieldAssertionType ) -> tuple[bool, Optional[str]]: """Checks if the actual value satisfies the given assertion based on the assertion type. - + :param actual_value: The value to be checked. :param assertion: The expected value or pattern to check against. :param assertion_type: The type of assertion to perform. @@ -82,9 +86,12 @@ def check_field_verbose( operation = _OPERATIONS.get(assertion_type) if not operation: raise ValueError(f"Unsupported assertion type: {assertion_type}") - + result = operation(actual_value, assertion) if result: return True, None else: - return False, f"Assertion failed: actual value '{actual_value}' does not satisfy '{assertion_type.name}' with assertion '{assertion}'" \ No newline at end of file + return ( + False, + f"Assertion failed: actual value '{actual_value}' does not satisfy '{assertion_type.name}' with assertion '{assertion}'", + ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py index 24bec0ab..666d6e77 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py @@ -5,37 +5,36 @@ from .check_activity import check_activity from .type_defs import SelectorQuantifier + def select_activity( activities: list[Activity], selector: Activity, index: int = 0, ) -> Optional[Activity]: """Selects a single activity from a list based on the provided selector and index. - + :param activities: List of activities to select from. :param selector: Activity used as a selector. :param index: Index of the activity to select when multiple match. :return: The selected activity or None if no activity matches. """ - res = select_activities(activities, { - "selector": selector, - "quantifier": SelectorQuantifier.ONE, - "index": index - } + res = select_activities( + activities, + {"selector": selector, "quantifier": SelectorQuantifier.ONE, "index": index}, ) return res[index] if res else None + def select_activities( - activities: list[Activity], - selector_config: dict + activities: list[Activity], selector_config: dict ) -> list[Activity]: """Selects activities from a list based on the provided selector configuration. - + :param activities: List of activities to select from. :param selector_config: Configuration dict containing 'selector', 'quantifier', and optionally ' :return: List of selected activities. """ - + selector = selector_config.get("selector", {}) index = selector_config.get("index", None) @@ -47,13 +46,11 @@ def select_activities( if quantifier_name not in SelectorQuantifier.__members__: raise ValueError(f"Invalid quantifier: {quantifier_name}") - + quantifier = SelectorQuantifier(quantifier_name) - + if quantifier == SelectorQuantifier.ALL: - return list( - filter(lambda a: check_activity(a, selector), activities) - ) + return list(filter(lambda a: check_activity(a, selector), activities)) else: first = next(filter(lambda a: check_activity(a, selector), activities), None) - return [first] if first is not None else [] \ No newline at end of file + return [first] if first is not None else [] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py index fc1060bd..72b614d8 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from typing import Any + class UNSET_FIELD: """Singleton to represent an unset field in activity comparisons.""" @@ -13,6 +14,7 @@ def get(*args, **kwargs): """Returns the singleton instance.""" return UNSET_FIELD + class FieldAssertionType(str, Enum): """Defines the types of assertions that can be made on fields.""" @@ -26,12 +28,14 @@ class FieldAssertionType(str, Enum): NOT_IN = "NOT_IN" RE_MATCH = "RE_MATCH" + class SelectorQuantifier(str, Enum): """Defines quantifiers for selecting activities.""" - + ALL = "ALL" ONE = "ONE" + class AssertionQuantifier(str, Enum): """Defines quantifiers for assertions on activities.""" @@ -40,6 +44,7 @@ class AssertionQuantifier(str, Enum): ONE = "ONE" NONE = "NONE" + @dataclass class AssertionErrorData: """Data class to hold information about assertion errors.""" @@ -55,4 +60,4 @@ def __str__(self) -> str: f"actual value '{self.actual_value}' " f"does not satisfy assertion '{self.assertion}' " f"of type '{self.assertion_type}'." - ) \ No newline at end of file + ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py index fd6756ca..aab93742 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py @@ -1,7 +1,4 @@ from .data_driven_test import DataDrivenTest from .ddt import ddt -__all__ = [ - "DataDrivenTest", - "ddt" -] \ No newline at end of file +__all__ = ["DataDrivenTest", "ddt"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py index a50c2cd1..39637acc 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py @@ -7,7 +7,11 @@ from microsoft_agents.activity import Activity -from microsoft_agents.testing.assertions import assert_activity +from microsoft_agents.testing.assertions import ( + ActivityAssertion, + assert_activity, +) + class DataDrivenTest: @@ -17,7 +21,7 @@ def __init__(self, test_flow: dict) -> None: defaults = test_flow.get("defaults", {}) self._input_defaults = defaults.get("input", {}) self._assertion_defaults = defaults.get("assertion", {}) - self._sleep_defaults = defaults.get("sleep", {}) + self._sleep_defaults = defaults.get("sleep", {}) self._test = test_flow.get("test", []) @@ -25,12 +29,12 @@ def _load_input(self, input_data: dict) -> Activity: data = deepcopy(self._input_defaults) data.update(input_data) return Activity.model_validate(data) - + def _load_assertion(self, assertion_data: dict) -> dict: data = deepcopy(self._assertion_defaults) data.update(assertion_data) return data - + async def _sleep(self, sleep_data: dict) -> None: duration = sleep_data.get("duration") if duration is None: @@ -38,23 +42,22 @@ async def _sleep(self, sleep_data: dict) -> None: await asyncio.sleep(duration) async def run(self, agent_client, response_client) -> None: - + + responses = [] for step in self._test: step_type = step.get("type") if not step_type: raise ValueError("Each step must have a 'type' field.") - + if step_type == "input": input_activity = self._load_input(step) await agent_client.send_activity(input_activity) elif step_type == "assertion": - assertion = self._load_assertion(step) - responses = await response_client.pop() - selector = Selector(assertion.get("selector", {})) - selection = selector.select(responses) + responses.extend(await response_client.pop()) - assert_activity(selection, assertion) + activity_assertion = ActivityAssertion(step) + assert activity_assertion.check(responses) elif step_type == "sleep": - await self._sleep(step) \ No newline at end of file + await self._sleep(step) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py index af01e7db..f09f9bf5 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/ddt.py @@ -7,6 +7,7 @@ from microsoft_agents.testing.assertions import assert_activity + class DataDrivenTest: def __init__(self, test_flow: dict) -> None: @@ -15,7 +16,7 @@ def __init__(self, test_flow: dict) -> None: defaults = test_flow.get("defaults", {}) self._input_defaults = defaults.get("input", {}) self._assertion_defaults = defaults.get("assertion", {}) - self._sleep_defaults = defaults.get("sleep", {}) + self._sleep_defaults = defaults.get("sleep", {}) self._test = test_flow.get("test", []) @@ -23,12 +24,12 @@ def _load_input(self, input_data: dict) -> Activity: data = deepcopy(self._input_defaults) data.update(input_data) return Activity.model_validate(data) - + def _load_assertion(self, assertion_data: dict) -> dict: data = deepcopy(self._assertion_defaults) data.update(assertion_data) return data - + async def _sleep(self, sleep_data: dict) -> None: duration = sleep_data.get("duration") if duration is None: @@ -36,12 +37,12 @@ async def _sleep(self, sleep_data: dict) -> None: await asyncio.sleep(duration) async def run(self, agent_client, response_client) -> None: - + for step in self._test: step_type = step.get("type") if not step_type: raise ValueError("Each step must have a 'type' field.") - + if step_type == "input": input_activity = self._load_input(step) await agent_client.send_activity(input_activity) @@ -55,4 +56,4 @@ async def run(self, agent_client, response_client) -> None: assert_activity(selection, assertion) elif step_type == "sleep": - await self._sleep(step) \ No newline at end of file + await self._sleep(step) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py index 611a58d9..283e101e 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py @@ -13,7 +13,7 @@ class TestParseAssertion: params=[ FieldAssertionType.EQUALS, FieldAssertionType.NOT_EQUALS, - FieldAssertionType.GREATER_THAN + FieldAssertionType.GREATER_THAN, ] ) def assertion_type_str(self, request): @@ -40,11 +40,7 @@ def test_parse_assertion_list(self, assertion_value, assertion_type_str): @pytest.mark.parametrize( "field", - [ - "value", - 123, - 12.34 - ], + ["value", 123, 12.34], ) def test_parse_assertion_default(self, field): assertion, assertion_type = _parse_assertion(field) @@ -59,13 +55,14 @@ def test_parse_assertion_default(self, field): [FieldAssertionType.RE_MATCH], [], {"assertion_type": "invalid", "assertion": "test"}, - ] + ], ) def test_parse_assertion_none(self, field): assertion, assertion_type = _parse_assertion(field) assert assertion is None assert assertion_type is None + class TestAssertActivity: """Tests for assert_activity function.""" @@ -93,7 +90,7 @@ def test_assert_activity_with_partial_baseline(self): type="message", text="Hello", channel_id="test-channel", - conversation={"id": "conv123"} + conversation={"id": "conv123"}, ) baseline = {"type": "message", "text": "Hello"} assert_activity(activity, baseline) @@ -121,17 +118,14 @@ def test_assert_activity_with_dict_assertion_format(self): activity = Activity(type="message", text="Hello, World!") baseline = { "type": "message", - "text": {"assertion_type": "CONTAINS", "assertion": "Hello"} + "text": {"assertion_type": "CONTAINS", "assertion": "Hello"}, } assert_activity(activity, baseline) def test_assert_activity_with_list_assertion_format(self): """Test using list format for assertions.""" activity = Activity(type="message", text="Hello, World!") - baseline = { - "type": "message", - "text": ["CONTAINS", "World"] - } + baseline = {"type": "message", "text": ["CONTAINS", "World"]} assert_activity(activity, baseline) def test_assert_activity_with_not_equals_assertion(self): @@ -139,24 +133,20 @@ def test_assert_activity_with_not_equals_assertion(self): activity = Activity(type="message", text="Hello") baseline = { "type": "message", - "text": {"assertion_type": "NOT_EQUALS", "assertion": "Goodbye"} + "text": {"assertion_type": "NOT_EQUALS", "assertion": "Goodbye"}, } assert_activity(activity, baseline) def test_assert_activity_with_contains_assertion(self): """Test CONTAINS assertion type.""" activity = Activity(type="message", text="Hello, World!") - baseline = { - "text": {"assertion_type": "CONTAINS", "assertion": "World"} - } + baseline = {"text": {"assertion_type": "CONTAINS", "assertion": "World"}} assert_activity(activity, baseline) def test_assert_activity_with_not_contains_assertion(self): """Test NOT_CONTAINS assertion type.""" activity = Activity(type="message", text="Hello") - baseline = { - "text": {"assertion_type": "NOT_CONTAINS", "assertion": "Goodbye"} - } + baseline = {"text": {"assertion_type": "NOT_CONTAINS", "assertion": "Goodbye"}} assert_activity(activity, baseline) def test_assert_activity_with_regex_assertion(self): @@ -170,39 +160,26 @@ def test_assert_activity_with_regex_assertion(self): def test_assert_activity_with_multiple_fields_and_mixed_assertions(self): """Test multiple fields with different assertion types.""" activity = Activity( - type="message", - text="Hello, World!", - channel_id="test-channel" + type="message", text="Hello, World!", channel_id="test-channel" ) baseline = { "type": "message", "text": ["CONTAINS", "Hello"], - "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "prod-channel"} + "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "prod-channel"}, } assert_activity(activity, baseline) def test_assert_activity_fails_on_any_field_mismatch(self): """Test that activity check fails if any field doesn't match.""" - activity = Activity( - type="message", - text="Hello", - channel_id="test-channel" - ) - baseline = { - "type": "message", - "text": "Hello", - "channel_id": "prod-channel" - } + activity = Activity(type="message", text="Hello", channel_id="test-channel") + baseline = {"type": "message", "text": "Hello", "channel_id": "prod-channel"} assert_activity(activity, baseline) def test_assert_activity_with_numeric_fields(self): """Test with numeric field values.""" activity = Activity(type="message", locale="en-US") activity.channel_data = {"timestamp": 1234567890} - baseline = { - "type": "message", - "channel_data": {"timestamp": 1234567890} - } + baseline = {"type": "message", "channel_data": {"timestamp": 1234567890}} assert_activity(activity, baseline) def test_assert_activity_with_greater_than_assertion(self): @@ -210,9 +187,11 @@ def test_assert_activity_with_greater_than_assertion(self): activity = Activity(type="message") activity.channel_data = {"count": 100} baseline = { - "channel_data": {"count": {"assertion_type": "GREATER_THAN", "assertion": 50}} + "channel_data": { + "count": {"assertion_type": "GREATER_THAN", "assertion": 50} + } } - + # This test depends on how nested dicts are handled # If channel_data is compared as a whole dict, this might not work as expected # Keeping this test to illustrate the concept @@ -221,12 +200,11 @@ def test_assert_activity_with_greater_than_assertion(self): def test_assert_activity_with_complex_nested_structures(self): """Test with complex nested structures in baseline.""" activity = Activity( - type="message", - conversation={"id": "conv123", "name": "Test Conversation"} + type="message", conversation={"id": "conv123", "name": "Test Conversation"} ) baseline = { "type": "message", - "conversation": {"id": "conv123", "name": "Test Conversation"} + "conversation": {"id": "conv123", "name": "Test Conversation"}, } assert_activity(activity, baseline) @@ -234,9 +212,7 @@ def test_assert_activity_with_boolean_fields(self): """Test with boolean field values.""" activity = Activity(type="message") activity.channel_data = {"is_active": True} - baseline = { - "channel_data": {"is_active": True} - } + baseline = {"channel_data": {"is_active": True}} assert_activity(activity, baseline) def test_assert_activity_type_mismatch(self): @@ -244,17 +220,18 @@ def test_assert_activity_type_mismatch(self): activity = Activity(type="message", text="Hello") baseline = {"type": "event", "text": "Hello"} assert_activity(activity, baseline) - + def test_assert_activity_with_list_fields(self): """Test with list field values.""" activity = Activity(type="message") activity.attachments = [Attachment(content_type="text/plain", content="test")] baseline = { "type": "message", - "attachments": [{"content_type": "text/plain", "content": "test"}] + "attachments": [{"content_type": "text/plain", "content": "test"}], } assert_activity(activity, baseline) - + + class TestAssertActivityRealWorldScenarios: """Tests simulating real-world usage scenarios.""" @@ -263,12 +240,12 @@ def test_validate_bot_response_message(self): activity = Activity( type="message", text="I found 3 results for your query.", - from_property={"id": "bot123", "name": "HelpBot"} + from_property={"id": "bot123", "name": "HelpBot"}, ) baseline = { "type": "message", "text": ["RE_MATCH", r"I found \d+ results"], - "from_property": {"id": "bot123"} + "from_property": {"id": "bot123"}, } assert_activity(activity, baseline) @@ -277,26 +254,21 @@ def test_validate_user_message(self): activity = Activity( type="message", text="help me with something", - from_property={"id": "user456"} + from_property={"id": "user456"}, ) baseline = { "type": "message", - "text": {"assertion_type": "CONTAINS", "assertion": "help"} + "text": {"assertion_type": "CONTAINS", "assertion": "help"}, } assert_activity(activity, baseline) def test_validate_event_activity(self): """Test validating an event activity.""" activity = Activity( - type="event", - name="conversationUpdate", - value={"action": "add"} + type="event", name="conversationUpdate", value={"action": "add"} ) - baseline = { - "type": "event", - "name": "conversationUpdate" - } - + baseline = {"type": "event", "name": "conversationUpdate"} + assert assert_activity(activity, baseline) is True def test_partial_match_allows_extra_fields(self): @@ -307,64 +279,41 @@ def test_partial_match_allows_extra_fields(self): channel_id="teams", conversation={"id": "conv123"}, from_property={"id": "user123"}, - timestamp="2025-01-12T10:00:00Z" + timestamp="2025-01-12T10:00:00Z", ) - baseline = { - "type": "message", - "text": "Hello" - } + baseline = {"type": "message", "text": "Hello"} assert_activity(activity, baseline) def test_strict_match_with_multiple_fields(self): """Test strict matching with multiple fields specified.""" - activity = Activity( - type="message", - text="Hello", - channel_id="teams" - ) - baseline = { - "type": "message", - "text": "Hello", - "channel_id": "teams" - } + activity = Activity(type="message", text="Hello", channel_id="teams") + baseline = {"type": "message", "text": "Hello", "channel_id": "teams"} assert_activity(activity, baseline) def test_flexible_text_matching_with_regex(self): """Test flexible text matching using regex patterns.""" - activity = Activity( - type="message", - text="Order #12345 has been confirmed" - ) - baseline = { - "type": "message", - "text": ["RE_MATCH", r"Order #\d+ has been"] - } + activity = Activity(type="message", text="Order #12345 has been confirmed") + baseline = {"type": "message", "text": ["RE_MATCH", r"Order #\d+ has been"]} assert_activity(activity, baseline) def test_negative_assertions(self): """Test using negative assertions to ensure fields don't match.""" - activity = Activity( - type="message", - text="Success", - channel_id="teams" - ) + activity = Activity(type="message", text="Success", channel_id="teams") baseline = { "type": "message", "text": {"assertion_type": "NOT_CONTAINS", "assertion": "Error"}, - "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "slack"} + "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "slack"}, } assert_activity(activity, baseline) def test_combined_positive_and_negative_assertions(self): """Test combining positive and negative assertions.""" activity = Activity( - type="message", - text="Operation completed successfully", - channel_id="teams" + type="message", text="Operation completed successfully", channel_id="teams" ) baseline = { "type": "message", "text": ["CONTAINS", "completed"], - "channel_id": ["NOT_EQUALS", "slack"] + "channel_id": ["NOT_EQUALS", "slack"], } - assert_activity(activity, baseline) \ No newline at end of file + assert_activity(activity, baseline) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py index 99de0866..0884e69b 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py @@ -109,10 +109,15 @@ class TestCheckFieldNotContains: """Tests for NOT_CONTAINS assertion type.""" def test_not_contains_substring_not_in_string(self): - assert check_field("hello world", "foo", FieldAssertionType.NOT_CONTAINS) is True + assert ( + check_field("hello world", "foo", FieldAssertionType.NOT_CONTAINS) is True + ) def test_not_contains_substring_in_string(self): - assert check_field("hello world", "world", FieldAssertionType.NOT_CONTAINS) is False + assert ( + check_field("hello world", "world", FieldAssertionType.NOT_CONTAINS) + is False + ) def test_not_contains_element_not_in_list(self): assert check_field([1, 2, 3, 4], 5, FieldAssertionType.NOT_CONTAINS) is True @@ -132,12 +137,21 @@ def test_re_match_no_match(self): def test_re_match_email_pattern(self): pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" - assert check_field("test@example.com", pattern, FieldAssertionType.RE_MATCH) is True - assert check_field("invalid-email", pattern, FieldAssertionType.RE_MATCH) is False + assert ( + check_field("test@example.com", pattern, FieldAssertionType.RE_MATCH) + is True + ) + assert ( + check_field("invalid-email", pattern, FieldAssertionType.RE_MATCH) is False + ) def test_re_match_anchored_pattern(self): - assert check_field("hello world", r"^hello", FieldAssertionType.RE_MATCH) is True - assert check_field("hello world", r"^world", FieldAssertionType.RE_MATCH) is False + assert ( + check_field("hello world", r"^hello", FieldAssertionType.RE_MATCH) is True + ) + assert ( + check_field("hello world", r"^world", FieldAssertionType.RE_MATCH) is False + ) def test_re_match_full_string(self): assert check_field("abc", r"^abc$", FieldAssertionType.RE_MATCH) is True @@ -211,4 +225,7 @@ def test_validate_message_format(self): def test_validate_list_membership(self): allowed_roles = ["admin", "user", "guest"] assert check_field(allowed_roles, "admin", FieldAssertionType.CONTAINS) is True - assert check_field(allowed_roles, "superuser", FieldAssertionType.NOT_CONTAINS) is True \ No newline at end of file + assert ( + check_field(allowed_roles, "superuser", FieldAssertionType.NOT_CONTAINS) + is True + ) From e88997e55f9797c49505434999343cde216bc36e Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Thu, 13 Nov 2025 13:12:27 -0800 Subject: [PATCH 73/81] Populate function for dictionaries --- .../integration/data_driven/data_driven_test.py | 17 +++++++++++++++++ .../microsoft_agents/testing/utils/__init__.py | 4 ++-- .../utils/{populate_activity.py => populate.py} | 8 ++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) rename dev/microsoft-agents-testing/microsoft_agents/testing/utils/{populate_activity.py => populate.py} (62%) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py index 39637acc..61e3a759 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py @@ -3,6 +3,8 @@ import asyncio +import yaml + from copy import deepcopy from microsoft_agents.activity import Activity @@ -22,6 +24,18 @@ def __init__(self, test_flow: dict) -> None: self._input_defaults = defaults.get("input", {}) self._assertion_defaults = defaults.get("assertion", {}) self._sleep_defaults = defaults.get("sleep", {}) + + parent = test_flow.get("parent") + if parent: + with open(parent, "r", encoding="utf-8") as f: + parent_flow = yaml.safe_load(f) + input_defaults = parent_flow.get("defaults", {}).get("input", {}) + sleep_defaults = parent_flow.get("defaults", {}).get("sleep", {}) + assertion_defaults = parent_flow.get("defaults", {}).get("assertion", {}) + + self._input_defaults = {**input_defaults, **self._input_defaults} + self._sleep_defaults = {**sleep_defaults, **self._sleep_defaults} + self._assertion_defaults = {**assertion_defaults, **self._assertion_defaults} self._test = test_flow.get("test", []) @@ -51,11 +65,14 @@ async def run(self, agent_client, response_client) -> None: if step_type == "input": input_activity = self._load_input(step) + populate_activity(input_activity, self._input_defaults) await agent_client.send_activity(input_activity) elif step_type == "assertion": responses.extend(await response_client.pop()) + # populate defaults + activity_assertion = ActivityAssertion(step) assert activity_assertion.check(responses) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py index fd1102ff..ad221b21 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -1,11 +1,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from .populate_activity import populate_activity +from .populate import populate_activity from .misc import get_host_and_port, normalize_activity_data __all__ = [ - "populate_activity", + "populate", "get_host_and_port", "normalize_activity_data", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py similarity index 62% rename from dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py rename to dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py index 7d1edea8..13da5334 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py @@ -3,6 +3,14 @@ from microsoft_agents.activity import Activity +def populate(original: dict, defaults: dict) -> None: + """Populate a dictionary with default values.""" + + for key in defaults.keys(): + if key not in original: + original[key] = defaults[key] + elif isinstance(original[key], dict) and isinstance(defaults[key], dict): + populate(original[key], defaults[key]) def populate_activity(original: Activity, defaults: Activity | dict) -> Activity: """Populate an Activity object with default values.""" From 0a48730b1ff37383d058615842fa2cd4785b67b7 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Fri, 14 Nov 2025 06:49:20 -0800 Subject: [PATCH 74/81] Populate tests --- .../tests/utils/__init__.py | 0 .../tests/utils/test_populate.py | 350 ++++++++++++++++++ 2 files changed, 350 insertions(+) create mode 100644 dev/microsoft-agents-testing/tests/utils/__init__.py create mode 100644 dev/microsoft-agents-testing/tests/utils/test_populate.py diff --git a/dev/microsoft-agents-testing/tests/utils/__init__.py b/dev/microsoft-agents-testing/tests/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/utils/test_populate.py b/dev/microsoft-agents-testing/tests/utils/test_populate.py new file mode 100644 index 00000000..3889fb13 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/utils/test_populate.py @@ -0,0 +1,350 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +from microsoft_agents.activity import Activity, ChannelAccount, ConversationAccount + +from microsoft_agents.testing.utils.populate import default_update, populate_activity + + +class TestDefaultUpdate: + """Tests for the default_update function.""" + + def test_default_update_with_empty_original(self): + """Test that defaults are added to an empty dictionary.""" + original = {} + defaults = {"key1": "value1", "key2": "value2"} + default_update(original, defaults) + assert original == {"key1": "value1", "key2": "value2"} + + def test_default_update_with_empty_defaults(self): + """Test that original dictionary is unchanged when defaults is empty.""" + original = {"key1": "value1"} + defaults = {} + default_update(original, defaults) + assert original == {"key1": "value1"} + + def test_default_update_with_non_overlapping_keys(self): + """Test that defaults are added when keys don't overlap.""" + original = {"key1": "value1"} + defaults = {"key2": "value2", "key3": "value3"} + default_update(original, defaults) + assert original == {"key1": "value1", "key2": "value2", "key3": "value3"} + + def test_default_update_preserves_existing_values(self): + """Test that existing values in original are not overwritten.""" + original = {"key1": "original_value", "key2": "value2"} + defaults = {"key1": "default_value", "key3": "value3"} + default_update(original, defaults) + assert original == { + "key1": "original_value", + "key2": "value2", + "key3": "value3", + } + + def test_default_update_with_nested_dicts(self): + """Test that nested dictionaries are recursively updated.""" + original = {"nested": {"key1": "original"}} + defaults = {"nested": {"key1": "default", "key2": "value2"}} + default_update(original, defaults) + assert original == {"nested": {"key1": "original", "key2": "value2"}} + + def test_default_update_with_deeply_nested_dicts(self): + """Test recursive update with deeply nested structures.""" + original = {"level1": {"level2": {"key1": "original"}}} + defaults = { + "level1": { + "level2": {"key1": "default", "key2": "value2"}, + "level2b": {"key3": "value3"}, + } + } + default_update(original, defaults) + assert original == { + "level1": { + "level2": {"key1": "original", "key2": "value2"}, + "level2b": {"key3": "value3"}, + } + } + + def test_default_update_adds_nested_dict_when_missing(self): + """Test that nested dicts are added when they don't exist in original.""" + original = {"key1": "value1"} + defaults = {"nested": {"key2": "value2"}} + default_update(original, defaults) + assert original == {"key1": "value1", "nested": {"key2": "value2"}} + + def test_default_update_with_mixed_types(self): + """Test with various value types: strings, numbers, booleans, lists.""" + original = {"str": "text", "num": 42} + defaults = { + "str": "default_text", + "bool": True, + "list": [1, 2, 3], + "none": None, + } + default_update(original, defaults) + assert original == { + "str": "text", + "num": 42, + "bool": True, + "list": [1, 2, 3], + "none": None, + } + + def test_default_update_with_none_values(self): + """Test that None values in defaults are added.""" + original = {"key1": "value1"} + defaults = {"key2": None} + default_update(original, defaults) + assert original == {"key1": "value1", "key2": None} + + def test_default_update_preserves_none_in_original(self): + """Test that None values in original are preserved.""" + original = {"key1": None} + defaults = {"key1": "default_value"} + default_update(original, defaults) + assert original == {"key1": None} + + def test_default_update_with_list_values(self): + """Test that list values are not merged, only added if missing.""" + original = {"list1": [1, 2]} + defaults = {"list1": [3, 4], "list2": [5, 6]} + default_update(original, defaults) + assert original == {"list1": [1, 2], "list2": [5, 6]} + + def test_default_update_type_mismatch_original_wins(self): + """Test that when types differ, original value is preserved.""" + original = {"key1": "string_value"} + defaults = {"key1": {"nested": "dict"}} + default_update(original, defaults) + assert original == {"key1": "string_value"} + + def test_default_update_type_mismatch_defaults_dict(self): + """Test that when original is dict and default is not, original is preserved.""" + original = {"key1": {"nested": "dict"}} + defaults = {"key1": "string_value"} + default_update(original, defaults) + assert original == {"key1": {"nested": "dict"}} + + def test_default_update_modifies_in_place(self): + """Test that the function modifies the original dict in place.""" + original = {"key1": "value1"} + original_id = id(original) + defaults = {"key2": "value2"} + default_update(original, defaults) + assert id(original) == original_id + assert original == {"key1": "value1", "key2": "value2"} + + def test_default_update_with_complex_nested_structure(self): + """Test with complex real-world-like nested structure.""" + original = { + "user": {"name": "Alice", "settings": {"theme": "dark"}}, + "timestamp": "2025-01-01", + } + defaults = { + "user": { + "name": "DefaultName", + "settings": {"theme": "light", "language": "en"}, + "role": "user", + }, + "channel": "default-channel", + } + default_update(original, defaults) + assert original == { + "user": { + "name": "Alice", + "settings": {"theme": "dark", "language": "en"}, + "role": "user", + }, + "timestamp": "2025-01-01", + "channel": "default-channel", + } + + +class TestPopulateActivity: + """Tests for the populate_activity function.""" + + def test_populate_activity_with_none_values_filled(self): + """Test that None values in original are replaced with defaults.""" + original = Activity(type="message") + defaults = Activity(type="message", text="Default text") + result = populate_activity(original, defaults) + assert result.text == "Default text" + assert result.type == "message" + + def test_populate_activity_preserves_existing_values(self): + """Test that existing non-None values are preserved.""" + original = Activity(type="message", text="Original text") + defaults = Activity(type="event", text="Default text") + result = populate_activity(original, defaults) + assert result.text == "Original text" + assert result.type == "message" + + def test_populate_activity_returns_new_instance(self): + """Test that a new Activity instance is returned.""" + original = Activity(type="message", text="Original") + defaults = {"text": "Default text"} + result = populate_activity(original, defaults) + assert result is not original + assert id(result) != id(original) + + def test_populate_activity_original_unchanged(self): + """Test that the original Activity is not modified.""" + original = Activity(type="message") + defaults = Activity(type="message", text="Default text") + original_text = original.text + result = populate_activity(original, defaults) + assert original.text == original_text + assert result.text == "Default text" + + def test_populate_activity_with_dict_defaults(self): + """Test that defaults can be provided as a dictionary.""" + original = Activity(type="message") + original.channel_id = "channel" + defaults = {"text": "Default text", "channel_id": "default-channel"} + result = populate_activity(original, defaults) + assert result.text == "Default text" + assert result.channel_id == "channel" + + def test_populate_activity_with_activity_defaults(self): + """Test that defaults can be provided as an Activity object.""" + original = Activity(type="message") + defaults = Activity(type="event", text="Default text", channel_id="channel") + result = populate_activity(original, defaults) + assert result.text == "Default text" + + def test_populate_activity_with_empty_defaults(self): + """Test that original is unchanged when defaults is empty.""" + original = Activity(type="message", text="Original text") + defaults = {} + result = populate_activity(original, defaults) + assert result.text == "Original text" + assert result.type == "message" + + def test_populate_activity_with_multiple_fields(self): + """Test populating multiple None fields.""" + original = Activity( + type="message", + ) + defaults = { + "text": "Default text", + "channel_id": "default-channel", + "locale": "en-US", + } + result = populate_activity(original, defaults) + assert result.text == "Default text" + assert result.channel_id == "default-channel" + assert result.locale == "en-US" + + def test_populate_activity_with_complex_objects(self): + """Test populating with complex nested objects.""" + original = Activity(type="message") + defaults = Activity( + type="invoke", + from_property=ChannelAccount(id="bot123", name="Bot"), + conversation=ConversationAccount(id="conv123", name="Conversation"), + ) + result = populate_activity(original, defaults) + assert result.from_property is not None + assert result.from_property.id == "bot123" + assert result.conversation is not None + assert result.conversation.id == "conv123" + + def test_populate_activity_preserves_complex_objects(self): + """Test that existing complex objects are preserved.""" + original = Activity( + type="message", + from_property=ChannelAccount(id="user456", name="User"), + ) + defaults = Activity(type="invoke", from_property=ChannelAccount(id="bot123", name="Bot")) + result = populate_activity(original, defaults) + assert result.from_property.id == "user456" + + def test_populate_activity_partial_defaults(self): + """Test that only specified defaults are applied.""" + original = Activity(type="message") + defaults = {"text": "Default text"} + result = populate_activity(original, defaults) + assert result.text == "Default text" + assert result.channel_id is None + + def test_populate_activity_with_zero_and_empty_string(self): + """Test that zero and empty string are considered as set values.""" + original = Activity(type="message", text="") + defaults = {"text": "Default text", "locale": "en-US"} + result = populate_activity(original, defaults) + # Empty strings should be preserved as they are not None + assert result.text == "" + assert result.locale == "en-US" + + def test_populate_activity_with_false_boolean(self): + """Test that False boolean values are preserved.""" + original = Activity(type="message") + original.history_disclosed = False + defaults = {"history_disclosed": True} + result = populate_activity(original, defaults) + # False should be preserved as it's not None + assert result.history_disclosed is False + + def test_populate_activity_with_zero_numeric(self): + """Test that numeric zero values are preserved.""" + original = Activity(type="message") + # Assuming there's a numeric field we can test + original.channel_data = {"count": 0} + defaults = {"channel_data": {"count": 10}} + result = populate_activity(original, defaults) + # Zero should be preserved + assert result.channel_data == {"count": 0} + + def test_populate_activity_defaults_from_activity_excludes_unset(self): + """Test that only explicitly set fields from Activity defaults are used.""" + original = Activity(type="message") + # Create defaults with only type set explicitly + defaults = Activity(type="event") + result = populate_activity(original, defaults) + # Since defaults Activity didn't explicitly set text, it should remain None + assert result.text is None + + def test_populate_activity_with_empty_activity_defaults(self): + """Test with an Activity that has no fields set.""" + original = Activity(type="message") + defaults = {} + result = populate_activity(original, defaults) + assert result.type == "message" + assert result.text is None + + def test_populate_activity_real_world_scenario(self): + """Test a real-world scenario of populating a bot response.""" + original = Activity( + type="message", + text="User's query result", + from_property=ChannelAccount(id="bot123"), + ) + defaults = { + "conversation": ConversationAccount(id="default-conv"), + "channel_id": "teams", + "locale": "en-US", + } + result = populate_activity(original, defaults) + assert result.text == "User's query result" + assert result.from_property.id == "bot123" + assert result.conversation.id == "default-conv" + assert result.channel_id == "teams" + assert result.locale == "en-US" + + def test_populate_activity_with_list_fields(self): + """Test populating list fields like attachments or entities.""" + original = Activity(type="message") + defaults = {"attachments": [], "entities": []} + result = populate_activity(original, defaults) + assert result.attachments == [] + assert result.entities == [] + + def test_populate_activity_preserves_empty_lists(self): + """Test that empty lists in original are preserved.""" + original = Activity(type="message", attachments=[], entities=[]) + defaults = {"attachments": [{"type": "card"}], "entities": [{"type": "mention"}]} + result = populate_activity(original, defaults) + # Empty lists are not None, so they should be preserved + assert result.attachments == [] + assert result.entities == [] \ No newline at end of file From 2f96d88dd7c6ad87901f6985704489214586fcd8 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Fri, 14 Nov 2025 07:27:57 -0800 Subject: [PATCH 75/81] Selector class implementation --- .../manual_test => _manual_test}/__init__.py | 0 .../manual_test => _manual_test}/env.TEMPLATE | 0 .../manual_test => _manual_test}/main.py | 0 .../testing/assertions/__init__.py | 14 +- .../testing/assertions/activity_assertion.py | 53 ++++++- .../testing/assertions/assertions.py | 17 --- .../testing/assertions/check_activity.py | 9 +- .../testing/assertions/check_field.py | 1 + .../testing/assertions/select_activity.py | 56 -------- .../testing/assertions/selector.py | 133 ++++++++++++++++++ .../testing/assertions/type_defs.py | 21 ++- .../data_driven/data_driven_test.py | 11 +- .../testing/utils/__init__.py | 5 +- .../testing/utils/populate.py | 26 ++-- .../microsoft_agents/testing/utils/urls.py | 13 -- .../tests/assertions/_common.py | 3 + .../tests/assertions/test_assert_activity.py | 62 +------- .../tests/assertions/test_check_field.py | 66 ++++++++- .../test_integration_assertion.py} | 0 .../tests/integration/data_driven/__init__.py | 0 .../data_driven/test_activity_assertion.py | 0 .../data_driven/test_data_driven_tester.py | 0 .../tests/utils/test_populate.py | 11 +- 23 files changed, 320 insertions(+), 181 deletions(-) rename dev/microsoft-agents-testing/{tests/manual_test => _manual_test}/__init__.py (100%) rename dev/microsoft-agents-testing/{tests/manual_test => _manual_test}/env.TEMPLATE (100%) rename dev/microsoft-agents-testing/{tests/manual_test => _manual_test}/main.py (100%) delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py create mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py delete mode 100644 dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py rename dev/microsoft-agents-testing/tests/{integration/test_data_driven_tester.py => assertions/test_integration_assertion.py} (100%) create mode 100644 dev/microsoft-agents-testing/tests/integration/data_driven/__init__.py create mode 100644 dev/microsoft-agents-testing/tests/integration/data_driven/test_activity_assertion.py create mode 100644 dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_tester.py diff --git a/dev/microsoft-agents-testing/tests/manual_test/__init__.py b/dev/microsoft-agents-testing/_manual_test/__init__.py similarity index 100% rename from dev/microsoft-agents-testing/tests/manual_test/__init__.py rename to dev/microsoft-agents-testing/_manual_test/__init__.py diff --git a/dev/microsoft-agents-testing/tests/manual_test/env.TEMPLATE b/dev/microsoft-agents-testing/_manual_test/env.TEMPLATE similarity index 100% rename from dev/microsoft-agents-testing/tests/manual_test/env.TEMPLATE rename to dev/microsoft-agents-testing/_manual_test/env.TEMPLATE diff --git a/dev/microsoft-agents-testing/tests/manual_test/main.py b/dev/microsoft-agents-testing/_manual_test/main.py similarity index 100% rename from dev/microsoft-agents-testing/tests/manual_test/main.py rename to dev/microsoft-agents-testing/_manual_test/main.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py index 9e8695be..f2ecfe66 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py @@ -6,14 +6,22 @@ assert_activity, assert_field, ) -from .check_activity import check_activity -from .check_field import check_field -from .type_defs import FieldAssertionType, SelectorQuantifier, AssertionQuantifier +from .check_activity import check_activity, check_activity_verbose +from .check_field import check_field, check_field_verbose +from .type_defs import FieldAssertionType, AssertionQuantifier, UNSET_FIELD +from .selector import Selector, SelectorQuantifier __all__ = [ + "ActivityAssertion", "assert_activity", "assert_field", "check_activity", + "check_activity_verbose", "check_field", + "check_field_verbose", "FieldAssertionType", + "Selector", + "SelectorQuantifier", + "AssertionQuantifier", + "UNSET_FIELD", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py index 57299304..cf67f830 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py @@ -1,24 +1,38 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from __future__ import annotations + from typing import Optional from microsoft_agents.activity import Activity -from .select_activity import select_activities from .check_activity import check_activity_verbose +from .selector import Selector, SelectorQuantifier from .type_defs import AssertionQuantifier, AssertionErrorData class ActivityAssertion: + """Class for asserting activities based on a selector and assertion criteria.""" - def __init__(self, config: dict) -> None: + _selector: Selector + _quantifier: AssertionQuantifier + _assertion: dict | Activity + + def __init__( + self, + assertion: dict | Activity | None = None, + selector: Selector | None = None, + quantifier: AssertionQuantifier = AssertionQuantifier.ALL, + ) -> None: """Initializes the ActivityAssertion with the given configuration. :param config: The configuration dictionary containing quantifier, selector, and assertion. """ - quantifier_name = config.get("quantifier", AssertionQuantifier.ALL) - self._quantifier = AssertionQuantifier(quantifier_name) - self._selector = config.get("selector", {}) - self._assertion = config.get("assertion", {}) + self._assertion = assertion or {} + self._selector = selector or Selector(quantifier=SelectorQuantifier.ALL) + self._quantifier = quantifier @staticmethod def _combine_assertion_errors(errors: list[AssertionErrorData]) -> str: @@ -36,7 +50,7 @@ def check(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: :return: A tuple containing a boolean indicating if the assertion passed and an optional error message. """ - activities = select_activities(activities, self._selector) + activities = self._selector(activities) count = 0 for activity in activities: @@ -63,3 +77,28 @@ def check(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: ) return passes, None + + def __call__(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: + """Allows the ActivityAssertion instance to be called directly. + + :param activities: The list of activities to be tested. + :return: A tuple containing a boolean indicating if the assertion passed and an optional error message. + """ + return self.check(activities) + + @staticmethod + def from_config(config: dict) -> ActivityAssertion: + """Creates an ActivityAssertion instance from a configuration dictionary. + + :param config: The configuration dictionary containing quantifier, selector, and assertion. + :return: An ActivityAssertion instance. + """ + assertion = config.get("assertion", {}) + selector = Selector.from_config(config.get("selector", {})) + quantifier = AssertionQuantifier.from_config(config.get("quantifier", "all")) + + return ActivityAssertion( + assertion=assertion, + selector=selector, + quantifier=quantifier, + ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py index f778b191..3ca15fde 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -33,20 +33,3 @@ def assert_activity(activity: Activity, assertion: Activity | dict) -> None: """ res, assertion_error_data = check_activity_verbose(activity, assertion) assert res, str(assertion_error_data) - - -def assert_activities(activities: list[Activity], assertion_config: dict) -> None: - """Asserts that the given list of activities matches the baseline activities. - - :param activities: The list of activities to be tested. - :param assertion: The baseline dictionary representing the expected activities data. - """ - - quantifier = assertion_config.get( - "quantifier", - ) - selector = assertion_config.get("selector", {}) - - for activity in activities: - res, assertion_error_data = check_activity_verbose(activity, assertion) - assert res, str(assertion_error_data) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py index ba286c22..d1de3893 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_activity.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from typing import Any, TypeVar, Optional +from typing import Any, Optional from microsoft_agents.activity import Activity from microsoft_agents.testing.utils import normalize_activity_data @@ -13,6 +13,13 @@ def _check( actual: Any, baseline: Any, field_path: str = "" ) -> tuple[bool, Optional[AssertionErrorData]]: + """Recursively checks the actual data against the baseline data. + + :param actual: The actual data to be tested. + :param baseline: The baseline data to compare against. + :param field_path: The current field path being checked (for error reporting). + :return: A tuple containing a boolean indicating success and optional assertion error data. + """ assertion, assertion_type = _parse_assertion(baseline) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index 77c8938c..2eab22ef 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -3,6 +3,7 @@ from .type_defs import FieldAssertionType, UNSET_FIELD + _OPERATIONS = { FieldAssertionType.EQUALS: lambda a, b: a == b or (a is UNSET_FIELD and b is None), FieldAssertionType.NOT_EQUALS: lambda a, b: a != b diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py deleted file mode 100644 index 666d6e77..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/select_activity.py +++ /dev/null @@ -1,56 +0,0 @@ -from typing import Optional - -from microsoft_agents.activity import Activity - -from .check_activity import check_activity -from .type_defs import SelectorQuantifier - - -def select_activity( - activities: list[Activity], - selector: Activity, - index: int = 0, -) -> Optional[Activity]: - """Selects a single activity from a list based on the provided selector and index. - - :param activities: List of activities to select from. - :param selector: Activity used as a selector. - :param index: Index of the activity to select when multiple match. - :return: The selected activity or None if no activity matches. - """ - res = select_activities( - activities, - {"selector": selector, "quantifier": SelectorQuantifier.ONE, "index": index}, - ) - return res[index] if res else None - - -def select_activities( - activities: list[Activity], selector_config: dict -) -> list[Activity]: - """Selects activities from a list based on the provided selector configuration. - - :param activities: List of activities to select from. - :param selector_config: Configuration dict containing 'selector', 'quantifier', and optionally ' - :return: List of selected activities. - """ - - selector = selector_config.get("selector", {}) - - index = selector_config.get("index", None) - if index is not None: - quantifier_name = selector_config.get("quantifier", SelectorQuantifier.ONE) - else: - quantifier_name = selector_config.get("quantifier", SelectorQuantifier.ALL) - quantifier_name = quantifier_name.upper() - - if quantifier_name not in SelectorQuantifier.__members__: - raise ValueError(f"Invalid quantifier: {quantifier_name}") - - quantifier = SelectorQuantifier(quantifier_name) - - if quantifier == SelectorQuantifier.ALL: - return list(filter(lambda a: check_activity(a, selector), activities)) - else: - first = next(filter(lambda a: check_activity(a, selector), activities), None) - return [first] if first is not None else [] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py new file mode 100644 index 00000000..2c828ec0 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from enum import Enum + +from microsoft_agents.activity import Activity + +from .check_activity import check_activity + + +class SelectorQuantifier(str, Enum): + """Defines the types of selectors that can be used to select activities.""" + + ALL = "ALL" + ONE = "ONE" + + +class Selector: + """Class for selecting activities based on a selector and quantifier.""" + + _selector: dict + _quantifier: SelectorQuantifier + _index: int | None + + def __init__( + self, + selector: dict | Activity | None = None, + quantifier: SelectorQuantifier | str | None = None, + index: int | None = None, + ) -> None: + """Initializes the Selector with the given configuration. + + :param selector: The selector to use for selecting activities. + The selector is an object holding the activity fields to match. + :param quantifier: The quantifier to use for selecting activities. + :param index: The index of the activity to select when quantifier is ONE. + + When quantifier is ALL, index should be None. + When quantifier is ONE, index defaults to 0 if not provided. + """ + + if not quantifier and not index: + raise ValueError("Either quantifier or index must be provided.") + + if selector is None: + selector = {} + elif isinstance(selector, Activity): + selector = selector.model_dump(exclude_unset=True) + + # make sure quantifier is of type SelectorQuantifier + if quantifier and isinstance(quantifier, str): + quantifier_name = quantifier.upper() + if quantifier_name not in SelectorQuantifier.__members__: + raise ValueError(f"Invalid quantifier: {quantifier_name}") + quantifier = SelectorQuantifier(quantifier_name) + + # validate index and quantifier combination + if index is None: + if quantifier == SelectorQuantifier.ONE: + index = 0 + elif quantifier not in SelectorQuantifier: + raise ValueError(f"Invalid quantifier: {quantifier}") + else: + if quantifier == SelectorQuantifier.ALL: + raise ValueError("Index should not be set when quantifier is ALL.") + + assert isinstance(quantifier, SelectorQuantifier) # linter hint + + self._quantifier = quantifier + self._index = index + self._selector = selector + + def select_first(self, activities: list[Activity]) -> Activity | None: + """Selects the first activity from the list of activities. + + :param activities: The list of activities to select from. + :return: A list containing the first activity, or an empty list if none exist. + """ + res = self.select(activities) + if res: + return res[0] + return None + + def select(self, activities: list[Activity]) -> list[Activity]: + """Selects activities based on the selector configuration. + + :param activities: The list of activities to select from. + :return: A list of selected activities. + """ + if self._quantifier == SelectorQuantifier.ALL: + return list( + filter( + lambda activity: check_activity(activity, self._selector), + activities, + ) + ) + else: + filtered_list = [] + for activity in activities: + if check_activity(activity, self._selector): + filtered_list.append(activity) + + assert self._index is not None # linter hint + if self._index < 0 and abs(self._index) <= len(filtered_list): + return [filtered_list[self._index]] + elif self._index >= 0 and self._index < len(filtered_list): + return [filtered_list[self._index]] + else: + return [] + + def __call__(self, activities: list[Activity]) -> list[Activity]: + """Allows the Selector instance to be called as a function. + + :param activities: The list of activities to select from. + :return: A list of selected activities. + """ + return self.select(activities) + + @staticmethod + def from_config(config: dict) -> Selector: + """Creates a Selector instance from a configuration dictionary. + + :param config: The configuration dictionary containing selector, quantifier, and index. + :return: A Selector instance. + """ + selector = config.get("selector", {}) + quantifier = config.get("quantifier", None) + index = config.get("index", None) + + return Selector( + selector=selector, + quantifier=quantifier, + index=index, + ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py index 72b614d8..97c4be49 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +from __future__ import annotations + from enum import Enum from dataclasses import dataclass from typing import Any @@ -29,13 +31,6 @@ class FieldAssertionType(str, Enum): RE_MATCH = "RE_MATCH" -class SelectorQuantifier(str, Enum): - """Defines quantifiers for selecting activities.""" - - ALL = "ALL" - ONE = "ONE" - - class AssertionQuantifier(str, Enum): """Defines quantifiers for assertions on activities.""" @@ -44,6 +39,18 @@ class AssertionQuantifier(str, Enum): ONE = "ONE" NONE = "NONE" + @staticmethod + def from_config(value: str) -> AssertionQuantifier: + """Creates an AssertionQuantifier from a configuration string. + + :param value: The configuration string. + :return: The corresponding AssertionQuantifier. + """ + value = value.upper() + if value not in AssertionQuantifier: + raise ValueError(f"Invalid AssertionQuantifier value: {value}") + return AssertionQuantifier(value) + @dataclass class AssertionErrorData: diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py index 61e3a759..8b4594fe 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py @@ -24,18 +24,23 @@ def __init__(self, test_flow: dict) -> None: self._input_defaults = defaults.get("input", {}) self._assertion_defaults = defaults.get("assertion", {}) self._sleep_defaults = defaults.get("sleep", {}) - + parent = test_flow.get("parent") if parent: with open(parent, "r", encoding="utf-8") as f: parent_flow = yaml.safe_load(f) input_defaults = parent_flow.get("defaults", {}).get("input", {}) sleep_defaults = parent_flow.get("defaults", {}).get("sleep", {}) - assertion_defaults = parent_flow.get("defaults", {}).get("assertion", {}) + assertion_defaults = parent_flow.get("defaults", {}).get( + "assertion", {} + ) self._input_defaults = {**input_defaults, **self._input_defaults} self._sleep_defaults = {**sleep_defaults, **self._sleep_defaults} - self._assertion_defaults = {**assertion_defaults, **self._assertion_defaults} + self._assertion_defaults = { + **assertion_defaults, + **self._assertion_defaults, + } self._test = test_flow.get("test", []) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py index ad221b21..aae37c77 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -1,11 +1,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from .populate import populate_activity +from .populate import default_update, populate_activity from .misc import get_host_and_port, normalize_activity_data __all__ = [ - "populate", + "default_update", + "populate_activity", "get_host_and_port", "normalize_activity_data", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py index 13da5334..979050bb 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py @@ -3,25 +3,35 @@ from microsoft_agents.activity import Activity -def populate(original: dict, defaults: dict) -> None: - """Populate a dictionary with default values.""" + +def default_update(original: dict, defaults: dict) -> None: + """Populate a dictionary with default values. + + :param original: The original dictionary to populate. + :param defaults: The dictionary containing default values. + """ for key in defaults.keys(): if key not in original: original[key] = defaults[key] elif isinstance(original[key], dict) and isinstance(defaults[key], dict): - populate(original[key], defaults[key]) + default_update(original[key], defaults[key]) + def populate_activity(original: Activity, defaults: Activity | dict) -> Activity: - """Populate an Activity object with default values.""" + """Populate an Activity object with default values. + + :param original: The original Activity object to populate. + :param defaults: The Activity object or dictionary containing default values. + """ if isinstance(defaults, Activity): defaults = defaults.model_dump(exclude_unset=True) - new_activity = original.model_copy() + new_activity_dict = original.model_dump(exclude_unset=True) for key in defaults.keys(): - if getattr(new_activity, key) is None: - setattr(new_activity, key, defaults[key]) + if key not in new_activity_dict: + new_activity_dict[key] = defaults[key] - return new_activity + return Activity.model_validate(new_activity_dict) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py deleted file mode 100644 index 7cc8fe9e..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from urllib.parse import urlparse - - -def get_host_and_port(url: str) -> tuple[str, int]: - """Extract host and port from a URL.""" - - parsed_url = urlparse(url) - host = parsed_url.hostname or "localhost" - port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) - return host, port diff --git a/dev/microsoft-agents-testing/tests/assertions/_common.py b/dev/microsoft-agents-testing/tests/assertions/_common.py index 0a69960c..83e666e4 100644 --- a/dev/microsoft-agents-testing/tests/assertions/_common.py +++ b/dev/microsoft-agents-testing/tests/assertions/_common.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import pytest from microsoft_agents.activity import Activity diff --git a/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py index 283e101e..3abb9ef0 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py @@ -1,66 +1,8 @@ -import pytest +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. from microsoft_agents.activity import Activity, Attachment - -from microsoft_agents.testing.assertions.type_defs import FieldAssertionType from microsoft_agents.testing.assertions.assertions import assert_activity -from microsoft_agents.testing.assertions.check_field import _parse_assertion - - -class TestParseAssertion: - - @pytest.fixture( - params=[ - FieldAssertionType.EQUALS, - FieldAssertionType.NOT_EQUALS, - FieldAssertionType.GREATER_THAN, - ] - ) - def assertion_type_str(self, request): - return request.param - - @pytest.fixture(params=["simple_value", {"key": "value"}, 42]) - def assertion_value(self, request): - return request.param - - def test_parse_assertion_dict(self, assertion_value, assertion_type_str): - - assertion, assertion_type = _parse_assertion( - {"assertion_type": assertion_type_str, "assertion": assertion_value} - ) - assert assertion == assertion_value - assert assertion_type == FieldAssertionType(assertion_type_str) - - def test_parse_assertion_list(self, assertion_value, assertion_type_str): - assertion, assertion_type = _parse_assertion( - [assertion_type_str, assertion_value] - ) - assert assertion == assertion_value - assert assertion_type.value == assertion_type_str - - @pytest.mark.parametrize( - "field", - ["value", 123, 12.34], - ) - def test_parse_assertion_default(self, field): - assertion, assertion_type = _parse_assertion(field) - assert assertion == field - assert assertion_type == FieldAssertionType.EQUALS - - @pytest.mark.parametrize( - "field", - [ - {"assertion_type": FieldAssertionType.IN}, - {"assertion_type": FieldAssertionType.IN, "key": "value"}, - [FieldAssertionType.RE_MATCH], - [], - {"assertion_type": "invalid", "assertion": "test"}, - ], - ) - def test_parse_assertion_none(self, field): - assertion, assertion_type = _parse_assertion(field) - assert assertion is None - assert assertion_type is None class TestAssertActivity: diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py index 0884e69b..b1125f9a 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py @@ -1,7 +1,71 @@ -from microsoft_agents.testing.assertions.check_field import check_field +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest + +from microsoft_agents.testing.assertions.check_field import ( + check_field, + _parse_assertion, +) from microsoft_agents.testing.assertions.type_defs import FieldAssertionType +class TestParseAssertion: + + @pytest.fixture( + params=[ + FieldAssertionType.EQUALS, + FieldAssertionType.NOT_EQUALS, + FieldAssertionType.GREATER_THAN, + ] + ) + def assertion_type_str(self, request): + return request.param + + @pytest.fixture(params=["simple_value", {"key": "value"}, 42]) + def assertion_value(self, request): + return request.param + + def test_parse_assertion_dict(self, assertion_value, assertion_type_str): + + assertion, assertion_type = _parse_assertion( + {"assertion_type": assertion_type_str, "assertion": assertion_value} + ) + assert assertion == assertion_value + assert assertion_type == FieldAssertionType(assertion_type_str) + + def test_parse_assertion_list(self, assertion_value, assertion_type_str): + assertion, assertion_type = _parse_assertion( + [assertion_type_str, assertion_value] + ) + assert assertion == assertion_value + assert assertion_type.value == assertion_type_str + + @pytest.mark.parametrize( + "field", + ["value", 123, 12.34], + ) + def test_parse_assertion_default(self, field): + assertion, assertion_type = _parse_assertion(field) + assert assertion == field + assert assertion_type == FieldAssertionType.EQUALS + + @pytest.mark.parametrize( + "field", + [ + {"assertion_type": FieldAssertionType.IN}, + {"assertion_type": FieldAssertionType.IN, "key": "value"}, + [FieldAssertionType.RE_MATCH], + [], + {"assertion_type": "invalid", "assertion": "test"}, + ], + ) + def test_parse_assertion_none(self, field): + assertion, assertion_type = _parse_assertion(field) + assert assertion is None + assert assertion_type is None + + class TestCheckFieldEquals: """Tests for EQUALS assertion type.""" diff --git a/dev/microsoft-agents-testing/tests/integration/test_data_driven_tester.py b/dev/microsoft-agents-testing/tests/assertions/test_integration_assertion.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/test_data_driven_tester.py rename to dev/microsoft-agents-testing/tests/assertions/test_integration_assertion.py diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/__init__.py b/dev/microsoft-agents-testing/tests/integration/data_driven/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_activity_assertion.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_activity_assertion.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_tester.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_tester.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/utils/test_populate.py b/dev/microsoft-agents-testing/tests/utils/test_populate.py index 3889fb13..5140e772 100644 --- a/dev/microsoft-agents-testing/tests/utils/test_populate.py +++ b/dev/microsoft-agents-testing/tests/utils/test_populate.py @@ -256,7 +256,9 @@ def test_populate_activity_preserves_complex_objects(self): type="message", from_property=ChannelAccount(id="user456", name="User"), ) - defaults = Activity(type="invoke", from_property=ChannelAccount(id="bot123", name="Bot")) + defaults = Activity( + type="invoke", from_property=ChannelAccount(id="bot123", name="Bot") + ) result = populate_activity(original, defaults) assert result.from_property.id == "user456" @@ -343,8 +345,11 @@ def test_populate_activity_with_list_fields(self): def test_populate_activity_preserves_empty_lists(self): """Test that empty lists in original are preserved.""" original = Activity(type="message", attachments=[], entities=[]) - defaults = {"attachments": [{"type": "card"}], "entities": [{"type": "mention"}]} + defaults = { + "attachments": [{"type": "card"}], + "entities": [{"type": "mention"}], + } result = populate_activity(original, defaults) # Empty lists are not None, so they should be preserved assert result.attachments == [] - assert result.entities == [] \ No newline at end of file + assert result.entities == [] From 3a04fb54b7782c8b4ebd41c10cb72fe453c1fe1a Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Fri, 14 Nov 2025 07:36:23 -0800 Subject: [PATCH 76/81] Assertion tests completed for now --- .../testing/assertions/selector.py | 3 +- .../test_activity_assertion.py | 0 .../tests/assertions/test_assert_activity.py | 12 +- .../tests/assertions/test_check_field.py | 3 +- .../tests/assertions/test_select_activity.py | 0 .../tests/assertions/test_selector.py | 486 ++++++++++++++++++ 6 files changed, 496 insertions(+), 8 deletions(-) rename dev/microsoft-agents-testing/tests/{integration/data_driven => assertions}/test_activity_assertion.py (100%) delete mode 100644 dev/microsoft-agents-testing/tests/assertions/test_select_activity.py create mode 100644 dev/microsoft-agents-testing/tests/assertions/test_selector.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py index 2c828ec0..69533aca 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py @@ -38,7 +38,7 @@ def __init__( When quantifier is ONE, index defaults to 0 if not provided. """ - if not quantifier and not index: + if not quantifier and index is None: raise ValueError("Either quantifier or index must be provided.") if selector is None: @@ -62,6 +62,7 @@ def __init__( else: if quantifier == SelectorQuantifier.ALL: raise ValueError("Index should not be set when quantifier is ALL.") + quantifier = SelectorQuantifier.ONE assert isinstance(quantifier, SelectorQuantifier) # linter hint diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_activity_assertion.py b/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py similarity index 100% rename from dev/microsoft-agents-testing/tests/integration/data_driven/test_activity_assertion.py rename to dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py diff --git a/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py index 3abb9ef0..20dae390 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_assert_activity.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. from microsoft_agents.activity import Activity, Attachment -from microsoft_agents.testing.assertions.assertions import assert_activity +from microsoft_agents.testing.assertions import assert_activity, check_activity class TestAssertActivity: @@ -18,7 +18,7 @@ def test_assert_activity_with_non_matching_fields(self): """Test that activity doesn't match baseline with different field values.""" activity = Activity(type="message", text="Hello") baseline = {"type": "message", "text": "Goodbye"} - assert_activity(activity, baseline) + assert not check_activity(activity, baseline) def test_assert_activity_with_activity_baseline(self): """Test that baseline can be an Activity object.""" @@ -41,7 +41,7 @@ def test_assert_activity_with_missing_field(self): """Test that activity with missing field doesn't match baseline.""" activity = Activity(type="message") baseline = {"type": "message", "text": "Hello"} - assert_activity(activity, baseline) + assert not check_activity(activity, baseline) def test_assert_activity_with_none_values(self): """Test that None values are handled correctly.""" @@ -115,7 +115,7 @@ def test_assert_activity_fails_on_any_field_mismatch(self): """Test that activity check fails if any field doesn't match.""" activity = Activity(type="message", text="Hello", channel_id="test-channel") baseline = {"type": "message", "text": "Hello", "channel_id": "prod-channel"} - assert_activity(activity, baseline) + assert not check_activity(activity, baseline) def test_assert_activity_with_numeric_fields(self): """Test with numeric field values.""" @@ -161,7 +161,7 @@ def test_assert_activity_type_mismatch(self): """Test that different activity types don't match.""" activity = Activity(type="message", text="Hello") baseline = {"type": "event", "text": "Hello"} - assert_activity(activity, baseline) + assert not check_activity(activity, baseline) def test_assert_activity_with_list_fields(self): """Test with list field values.""" @@ -211,7 +211,7 @@ def test_validate_event_activity(self): ) baseline = {"type": "event", "name": "conversationUpdate"} - assert assert_activity(activity, baseline) is True + assert_activity(activity, baseline) def test_partial_match_allows_extra_fields(self): """Test that extra fields in activity don't cause failure.""" diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py index b1125f9a..e238312b 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py @@ -231,7 +231,8 @@ class TestCheckFieldEdgeCases: def test_invalid_assertion_type(self): # Passing an unsupported assertion type should return False - assert check_field("test", "test", "INVALID_TYPE") is False + with pytest.raises(ValueError): + assert check_field("test", "test", "INVALID_TYPE") def test_none_actual_value_with_equals(self): assert check_field(None, "test", FieldAssertionType.EQUALS) is False diff --git a/dev/microsoft-agents-testing/tests/assertions/test_select_activity.py b/dev/microsoft-agents-testing/tests/assertions/test_select_activity.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dev/microsoft-agents-testing/tests/assertions/test_selector.py b/dev/microsoft-agents-testing/tests/assertions/test_selector.py new file mode 100644 index 00000000..fc9ac25a --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_selector.py @@ -0,0 +1,486 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest + +from microsoft_agents.activity import Activity +from microsoft_agents.testing.assertions.selector import Selector, SelectorQuantifier + + +class TestSelectorInitialization: + """Tests for Selector initialization and validation.""" + + def test_init_with_quantifier_all(self): + """Test initialization with ALL quantifier.""" + selector = Selector(selector={}, quantifier=SelectorQuantifier.ALL) + assert selector._quantifier == SelectorQuantifier.ALL + assert selector._index is None + assert selector._selector == {} + + def test_init_with_quantifier_one_default_index(self): + """Test initialization with ONE quantifier defaults index to 0.""" + selector = Selector(selector={}, quantifier=SelectorQuantifier.ONE) + assert selector._quantifier == SelectorQuantifier.ONE + assert selector._index == 0 + + def test_init_with_quantifier_one_explicit_index(self): + """Test initialization with ONE quantifier and explicit index.""" + selector = Selector(selector={}, quantifier=SelectorQuantifier.ONE, index=2) + assert selector._quantifier == SelectorQuantifier.ONE + assert selector._index == 2 + + def test_init_with_index_only(self): + """Test initialization with index only (implies ONE quantifier).""" + selector = Selector(selector={}, index=1) + assert selector._quantifier == SelectorQuantifier.ONE + assert selector._index == 1 + + def test_init_with_string_quantifier(self): + """Test initialization with string quantifier.""" + selector = Selector(selector={}, quantifier="all") + assert selector._quantifier == SelectorQuantifier.ALL + + selector = Selector(selector={}, quantifier="ONE") + assert selector._quantifier == SelectorQuantifier.ONE + + def test_init_with_activity_selector(self): + """Test initialization with Activity object as selector.""" + activity = Activity(type="message", text="Hello") + selector = Selector(selector=activity, quantifier=SelectorQuantifier.ALL) + assert "type" in selector._selector + assert selector._selector["type"] == "message" + assert selector._selector["text"] == "Hello" + + def test_init_with_dict_selector(self): + """Test initialization with dictionary selector.""" + selector_dict = {"type": "message", "text": "Hello"} + selector = Selector(selector=selector_dict, quantifier=SelectorQuantifier.ALL) + assert selector._selector == selector_dict + + def test_init_with_none_selector(self): + """Test initialization with None selector defaults to empty dict.""" + selector = Selector(selector=None, quantifier=SelectorQuantifier.ALL) + assert selector._selector == {} + + def test_init_without_quantifier_or_index_raises_error(self): + """Test that initialization without quantifier or index raises ValueError.""" + with pytest.raises(ValueError, match="Either quantifier or index must be provided"): + Selector(selector={}) + + def test_init_with_invalid_quantifier_string_raises_error(self): + """Test that invalid quantifier string raises ValueError.""" + with pytest.raises(ValueError, match="Invalid quantifier"): + Selector(selector={}, quantifier="INVALID") + + def test_init_with_all_quantifier_and_index_raises_error(self): + """Test that ALL quantifier with index raises ValueError.""" + with pytest.raises(ValueError, match="Index should not be set when quantifier is ALL"): + Selector(selector={}, quantifier=SelectorQuantifier.ALL, index=0) + + def test_init_with_negative_index(self): + """Test initialization with negative index.""" + selector = Selector(selector={}, index=-1) + assert selector._index == -1 + assert selector._quantifier == SelectorQuantifier.ONE + + +class TestSelectorSelectWithQuantifierAll: + """Tests for select() method with ALL quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="message", text="World"), + Activity(type="event", name="test_event"), + Activity(type="message", text="Goodbye"), + ] + + def test_select_all_matching_type(self, activities): + """Test selecting all activities with matching type.""" + selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + result = selector.select(activities) + assert len(result) == 3 + assert all(a.type == "message" for a in result) + + def test_select_all_matching_multiple_fields(self, activities): + """Test selecting all activities matching multiple fields.""" + selector = Selector( + selector={"type": "message", "text": "Hello"}, + quantifier=SelectorQuantifier.ALL + ) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Hello" + + def test_select_all_no_matches(self, activities): + """Test selecting all with no matches returns empty list.""" + selector = Selector( + selector={"type": "nonexistent"}, + quantifier=SelectorQuantifier.ALL + ) + result = selector.select(activities) + assert len(result) == 0 + + def test_select_all_empty_selector(self, activities): + """Test selecting all with empty selector returns all activities.""" + selector = Selector(selector={}, quantifier=SelectorQuantifier.ALL) + result = selector.select(activities) + assert len(result) == len(activities) + + def test_select_all_from_empty_list(self): + """Test selecting from empty activity list.""" + selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + result = selector.select([]) + assert len(result) == 0 + + +class TestSelectorSelectWithQuantifierOne: + """Tests for select() method with ONE quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="First"), + Activity(type="message", text="Second"), + Activity(type="event", name="test_event"), + Activity(type="message", text="Third"), + ] + + def test_select_one_default_index(self, activities): + """Test selecting one activity with default index (0).""" + selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ONE) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "First" + + def test_select_one_explicit_index(self, activities): + """Test selecting one activity with explicit index.""" + selector = Selector( + selector={"type": "message"}, + quantifier=SelectorQuantifier.ONE, + index=1 + ) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Second" + + def test_select_one_last_index(self, activities): + """Test selecting one activity with last valid index.""" + selector = Selector( + selector={"type": "message"}, + quantifier=SelectorQuantifier.ONE, + index=2 + ) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Third" + + def test_select_one_negative_index(self, activities): + """Test selecting one activity with negative index.""" + selector = Selector(selector={"type": "message"}, index=-1) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Third" + + def test_select_one_negative_index_from_start(self, activities): + """Test selecting one activity with negative index from start.""" + selector = Selector(selector={"type": "message"}, index=-2) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Second" + + def test_select_one_index_out_of_range(self, activities): + """Test selecting with index out of range returns empty list.""" + selector = Selector( + selector={"type": "message"}, + quantifier=SelectorQuantifier.ONE, + index=10 + ) + result = selector.select(activities) + assert len(result) == 0 + + def test_select_one_negative_index_out_of_range(self, activities): + """Test selecting with negative index out of range returns empty list.""" + selector = Selector(selector={"type": "message"}, index=-10) + result = selector.select(activities) + assert len(result) == 0 + + def test_select_one_no_matches(self, activities): + """Test selecting one with no matches returns empty list.""" + selector = Selector( + selector={"type": "nonexistent"}, + quantifier=SelectorQuantifier.ONE + ) + result = selector.select(activities) + assert len(result) == 0 + + def test_select_one_from_empty_list(self): + """Test selecting one from empty list returns empty list.""" + selector = Selector(selector={"type": "message"}, index=0) + result = selector.select([]) + assert len(result) == 0 + + +class TestSelectorSelectFirst: + """Tests for select_first() method.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="First"), + Activity(type="message", text="Second"), + Activity(type="event", name="test_event"), + ] + + def test_select_first_with_matches(self, activities): + """Test select_first returns first matching activity.""" + selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + result = selector.select_first(activities) + assert result is not None + assert result.text == "First" + + def test_select_first_no_matches(self, activities): + """Test select_first with no matches returns None.""" + selector = Selector( + selector={"type": "nonexistent"}, + quantifier=SelectorQuantifier.ALL + ) + result = selector.select_first(activities) + assert result is None + + def test_select_first_empty_list(self): + """Test select_first on empty list returns None.""" + selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + result = selector.select_first([]) + assert result is None + + def test_select_first_with_one_quantifier(self, activities): + """Test select_first with ONE quantifier and specific index.""" + selector = Selector(selector={"type": "message"}, index=1) + result = selector.select_first(activities) + assert result is not None + assert result.text == "Second" + + +class TestSelectorCallable: + """Tests for __call__ method.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="event", name="test_event"), + ] + + def test_call_invokes_select(self, activities): + """Test that calling selector instance invokes select().""" + selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + result = selector(activities) + assert len(result) == 1 + assert result[0].text == "Hello" + + def test_call_returns_same_as_select(self, activities): + """Test that __call__ returns same result as select().""" + selector = Selector(selector={"type": "event"}, index=0) + call_result = selector(activities) + select_result = selector.select(activities) + assert call_result == select_result + + +class TestSelectorFromConfig: + """Tests for from_config static method.""" + + def test_from_config_with_all_quantifier(self): + """Test creating selector from config with ALL quantifier.""" + config = { + "selector": {"type": "message"}, + "quantifier": "ALL" + } + selector = Selector.from_config(config) + assert selector._quantifier == SelectorQuantifier.ALL + assert selector._selector == {"type": "message"} + assert selector._index is None + + def test_from_config_with_one_quantifier(self): + """Test creating selector from config with ONE quantifier.""" + config = { + "selector": {"type": "message"}, + "quantifier": "ONE" + } + selector = Selector.from_config(config) + assert selector._quantifier == SelectorQuantifier.ONE + assert selector._index == 0 + + def test_from_config_with_index(self): + """Test creating selector from config with index.""" + config = { + "selector": {"type": "message"}, + "index": 2 + } + selector = Selector.from_config(config) + assert selector._quantifier == SelectorQuantifier.ONE + assert selector._index == 2 + + def test_from_config_with_both_quantifier_and_index(self): + """Test creating selector from config with both quantifier and index.""" + config = { + "selector": {"type": "message"}, + "quantifier": "ONE", + "index": 1 + } + selector = Selector.from_config(config) + assert selector._quantifier == SelectorQuantifier.ONE + assert selector._index == 1 + + def test_from_config_missing_selector_defaults_to_empty(self): + """Test creating selector from config without selector field.""" + config = { + "quantifier": "ALL" + } + selector = Selector.from_config(config) + assert selector._selector == {} + + def test_from_config_empty_selector(self): + """Test creating selector from config with empty selector.""" + config = { + "selector": {}, + "quantifier": "ALL" + } + selector = Selector.from_config(config) + assert selector._selector == {} + + def test_from_config_complex_selector(self): + """Test creating selector from config with complex selector.""" + config = { + "selector": { + "type": "message", + "text": "Hello", + "channelData": {"key": "value"} + }, + "quantifier": "ALL" + } + selector = Selector.from_config(config) + assert selector._selector["type"] == "message" + assert selector._selector["text"] == "Hello" + assert selector._selector["channelData"]["key"] == "value" + + +class TestSelectorIntegration: + """Integration tests with realistic scenarios.""" + + @pytest.fixture + def conversation_activities(self): + """Create a realistic conversation flow.""" + return [ + Activity(type="conversationUpdate", name="add_member"), + Activity(type="message", text="Hello bot", from_property={"id": "user1"}), + Activity(type="message", text="Hi there!", from_property={"id": "bot"}), + Activity(type="message", text="How are you?", from_property={"id": "user1"}), + Activity(type="message", text="I'm doing well!", from_property={"id": "bot"}), + Activity(type="typing"), + Activity(type="message", text="Goodbye", from_property={"id": "user1"}), + ] + + def test_select_all_user_messages(self, conversation_activities): + """Test selecting all messages from a specific user.""" + selector = Selector( + selector={"type": "message", "from_property": {"id": "user1"}}, + quantifier=SelectorQuantifier.ALL + ) + result = selector.select(conversation_activities) + assert len(result) == 3 + + def test_select_first_bot_response(self, conversation_activities): + """Test selecting first bot response.""" + selector = Selector( + selector={"type": "message", "from_property": {"id": "bot"}}, + index=0 + ) + result = selector.select(conversation_activities) + assert len(result) == 1 + assert result[0].text == "Hi there!" + + def test_select_last_message_negative_index(self, conversation_activities): + """Test selecting last message using negative index.""" + selector = Selector( + selector={"type": "message"}, + index=-1 + ) + result = selector.select(conversation_activities) + assert len(result) == 1 + assert result[0].text == "Goodbye" + + def test_select_typing_indicator(self, conversation_activities): + """Test selecting typing indicator.""" + selector = Selector( + selector={"type": "typing"}, + quantifier=SelectorQuantifier.ALL + ) + result = selector.select(conversation_activities) + assert len(result) == 1 + + def test_select_conversation_update(self, conversation_activities): + """Test selecting conversation update events.""" + selector = Selector( + selector={"type": "conversationUpdate"}, + quantifier=SelectorQuantifier.ALL + ) + result = selector.select(conversation_activities) + assert len(result) == 1 + assert result[0].name == "add_member" + + +class TestSelectorEdgeCases: + """Tests for edge cases and boundary conditions.""" + + def test_select_with_partial_match(self): + """Test that partial matches work correctly.""" + activities = [ + Activity(type="message", text="Hello", channelData={"id": 1}), + Activity(type="message", text="World"), + ] + # Only matching on type, not text + selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + result = selector.select(activities) + assert len(result) == 2 + + def test_select_with_none_values(self): + """Test selecting activities with None values.""" + activities = [ + Activity(type="message"), + Activity(type="message", text="Hello"), + ] + selector = Selector( + selector={"type": "message", "text": None}, + quantifier=SelectorQuantifier.ALL + ) + result = selector.select(activities) + # This depends on how check_activity handles None + assert isinstance(result, list) + + def test_select_single_activity_list(self): + """Test selecting from list with single activity.""" + activities = [Activity(type="message", text="Only one")] + selector = Selector(selector={"type": "message"}, index=0) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Only one" + + def test_select_with_boundary_index_zero(self): + """Test selecting with index 0 on single item.""" + activities = [Activity(type="message", text="Single")] + selector = Selector(selector={"type": "message"}, index=0) + result = selector.select(activities) + assert len(result) == 1 + + def test_select_with_boundary_negative_one(self): + """Test selecting with index -1 on single item.""" + activities = [Activity(type="message", text="Single")] + selector = Selector(selector={"type": "message"}, index=-1) + result = selector.select(activities) + assert len(result) == 1 \ No newline at end of file From 31b2b994f9331841b195d0fec517ba91d1e8f788 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Fri, 14 Nov 2025 08:11:57 -0800 Subject: [PATCH 77/81] Adding ActivityAssertion tests --- .../microsoft_agents/testing/__init__.py | 22 + .../testing/assertions/__init__.py | 3 +- .../testing/assertions/activity_assertion.py | 7 +- .../testing/assertions/selector.py | 40 +- .../data_driven/data_driven_test.py | 7 +- .../assertions/test_activity_assertion.py | 659 ++++++++++++++++++ .../tests/assertions/test_check_field.py | 2 +- .../tests/assertions/test_selector.py | 221 +----- 8 files changed, 714 insertions(+), 247 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py index b85d37e9..824f8e5e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py @@ -3,6 +3,18 @@ from .sdk_config import SDKConfig +from .assertions import ( + ActivityAssertion, + Selector, + AssertionQuantifier, + assert_activity, + assert_field, + check_activity, + check_activity_verbose, + check_field, + check_field_verbose, + FieldAssertionType, +) from .auth import generate_token, generate_token_from_config from .utils import populate_activity, get_host_and_port @@ -30,4 +42,14 @@ "Integration", "populate_activity", "get_host_and_port", + "ActivityAssertion", + "Selector", + "AssertionQuantifier", + "assert_activity", + "assert_field", + "check_activity", + "check_activity_verbose", + "check_field", + "check_field_verbose", + "FieldAssertionType", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py index f2ecfe66..664b17c6 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py @@ -9,7 +9,7 @@ from .check_activity import check_activity, check_activity_verbose from .check_field import check_field, check_field_verbose from .type_defs import FieldAssertionType, AssertionQuantifier, UNSET_FIELD -from .selector import Selector, SelectorQuantifier +from .selector import Selector __all__ = [ "ActivityAssertion", @@ -21,7 +21,6 @@ "check_field_verbose", "FieldAssertionType", "Selector", - "SelectorQuantifier", "AssertionQuantifier", "UNSET_FIELD", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py index cf67f830..a2702fac 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py @@ -8,7 +8,7 @@ from microsoft_agents.activity import Activity from .check_activity import check_activity_verbose -from .selector import Selector, SelectorQuantifier +from .selector import Selector from .type_defs import AssertionQuantifier, AssertionErrorData @@ -31,7 +31,7 @@ def __init__( """ self._assertion = assertion or {} - self._selector = selector or Selector(quantifier=SelectorQuantifier.ALL) + self._selector = selector or Selector() self._quantifier = quantifier @staticmethod @@ -67,7 +67,8 @@ def check(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: False, f"Activity matched the assertion when none were expected: {activity}", ) - count += 1 + if res: + count += 1 passes = True if self._quantifier == AssertionQuantifier.ONE and count != 1: diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py index 69533aca..d8bf2cb6 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py @@ -7,24 +7,15 @@ from .check_activity import check_activity -class SelectorQuantifier(str, Enum): - """Defines the types of selectors that can be used to select activities.""" - - ALL = "ALL" - ONE = "ONE" - - class Selector: """Class for selecting activities based on a selector and quantifier.""" _selector: dict - _quantifier: SelectorQuantifier _index: int | None def __init__( self, selector: dict | Activity | None = None, - quantifier: SelectorQuantifier | str | None = None, index: int | None = None, ) -> None: """Initializes the Selector with the given configuration. @@ -38,37 +29,13 @@ def __init__( When quantifier is ONE, index defaults to 0 if not provided. """ - if not quantifier and index is None: - raise ValueError("Either quantifier or index must be provided.") - if selector is None: selector = {} elif isinstance(selector, Activity): selector = selector.model_dump(exclude_unset=True) - # make sure quantifier is of type SelectorQuantifier - if quantifier and isinstance(quantifier, str): - quantifier_name = quantifier.upper() - if quantifier_name not in SelectorQuantifier.__members__: - raise ValueError(f"Invalid quantifier: {quantifier_name}") - quantifier = SelectorQuantifier(quantifier_name) - - # validate index and quantifier combination - if index is None: - if quantifier == SelectorQuantifier.ONE: - index = 0 - elif quantifier not in SelectorQuantifier: - raise ValueError(f"Invalid quantifier: {quantifier}") - else: - if quantifier == SelectorQuantifier.ALL: - raise ValueError("Index should not be set when quantifier is ALL.") - quantifier = SelectorQuantifier.ONE - - assert isinstance(quantifier, SelectorQuantifier) # linter hint - - self._quantifier = quantifier - self._index = index self._selector = selector + self._index = index def select_first(self, activities: list[Activity]) -> Activity | None: """Selects the first activity from the list of activities. @@ -87,7 +54,7 @@ def select(self, activities: list[Activity]) -> list[Activity]: :param activities: The list of activities to select from. :return: A list of selected activities. """ - if self._quantifier == SelectorQuantifier.ALL: + if self._index is None: return list( filter( lambda activity: check_activity(activity, self._selector), @@ -100,7 +67,6 @@ def select(self, activities: list[Activity]) -> list[Activity]: if check_activity(activity, self._selector): filtered_list.append(activity) - assert self._index is not None # linter hint if self._index < 0 and abs(self._index) <= len(filtered_list): return [filtered_list[self._index]] elif self._index >= 0 and self._index < len(filtered_list): @@ -124,11 +90,9 @@ def from_config(config: dict) -> Selector: :return: A Selector instance. """ selector = config.get("selector", {}) - quantifier = config.get("quantifier", None) index = config.get("index", None) return Selector( selector=selector, - quantifier=quantifier, index=index, ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py index 8b4594fe..c9bf5258 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py @@ -74,12 +74,11 @@ async def run(self, agent_client, response_client) -> None: await agent_client.send_activity(input_activity) elif step_type == "assertion": - responses.extend(await response_client.pop()) + activity_assertion = ActivityAssertion.from_config(step) - # populate defaults + responses.extend(await response_client.pop()) - activity_assertion = ActivityAssertion(step) - assert activity_assertion.check(responses) + assert activity_assertion(responses) elif step_type == "sleep": await self._sleep(step) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py b/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py index e69de29b..d596157a 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py @@ -0,0 +1,659 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest + +from microsoft_agents.activity import Activity +from microsoft_agents.testing import ( + ActivityAssertion, + Selector, + AssertionQuantifier, + FieldAssertionType, +) + + +class TestActivityAssertionCheckWithQuantifierAll: + """Tests for check() method with ALL quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="message", text="World"), + Activity(type="event", name="test_event"), + Activity(type="message", text="Goodbye"), + ] + + def test_check_all_matching_activities(self, activities): + """Test that all matching activities pass the assertion.""" + assertion = ActivityAssertion( + assertion={"type": "message"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert passes is True + assert error is None + + def test_check_all_with_one_failing_activity(self, activities): + """Test that one failing activity causes ALL assertion to fail.""" + assertion = ActivityAssertion( + assertion={"text": "Hello"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert passes is False + assert error is not None + assert "Activity did not match the assertion" in error + + def test_check_all_with_empty_selector(self, activities): + """Test ALL quantifier with empty selector (matches all activities).""" + assertion = ActivityAssertion( + assertion={"type": "message"}, + selector=Selector(selector={}), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + # Should fail because not all activities are messages + assert passes is False + + def test_check_all_with_empty_activities(self): + """Test ALL quantifier with empty activities list.""" + assertion = ActivityAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check([]) + assert passes is True + assert error is None + + def test_check_all_with_complex_assertion(self, activities): + """Test ALL quantifier with complex nested assertion.""" + complex_activities = [ + Activity(type="message", text="Hello", channelData={"id": 1}), + Activity(type="message", text="World", channelData={"id": 2}), + ] + assertion = ActivityAssertion( + assertion={"type": "message"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(complex_activities) + assert passes is True + + +class TestActivityAssertionCheckWithQuantifierNone: + """Tests for check() method with NONE quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="message", text="World"), + Activity(type="event", name="test_event"), + ] + + def test_check_none_with_no_matches(self, activities): + """Test NONE quantifier when no activities match.""" + assertion = ActivityAssertion( + assertion={"text": "Nonexistent"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.NONE, + ) + passes, error = assertion.check(activities) + assert passes is True + assert error is None + + def test_check_none_with_one_match(self, activities): + """Test NONE quantifier fails when one activity matches.""" + assertion = ActivityAssertion( + assertion={"text": "Hello"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.NONE, + ) + passes, error = assertion.check(activities) + assert passes is False + assert error is not None + assert "Activity matched the assertion when none were expected" in error + + def test_check_none_with_all_matching(self, activities): + """Test NONE quantifier fails when all activities match.""" + assertion = ActivityAssertion( + assertion={"type": "message"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.NONE, + ) + passes, error = assertion.check(activities) + assert passes is False + + def test_check_none_with_empty_activities(self): + """Test NONE quantifier with empty activities list.""" + assertion = ActivityAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.NONE + ) + passes, error = assertion.check([]) + assert passes is True + assert error is None + + +class TestActivityAssertionCheckWithQuantifierOne: + """Tests for check() method with ONE quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="First"), + Activity(type="message", text="Second"), + Activity(type="event", name="test_event"), + Activity(type="message", text="Third"), + ] + + def test_check_one_with_exactly_one_match(self, activities): + """Test ONE quantifier passes when exactly one activity matches.""" + assertion = ActivityAssertion( + assertion={"text": "First"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = assertion.check(activities) + assert passes is True + assert error is None + + def test_check_one_with_no_matches(self, activities): + """Test ONE quantifier fails when no activities match.""" + assertion = ActivityAssertion( + assertion={"text": "Nonexistent"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = assertion.check(activities) + assert passes is False + assert error is not None + assert "Expected exactly one activity" in error + assert "found 0" in error + + def test_check_one_with_multiple_matches(self, activities): + """Test ONE quantifier fails when multiple activities match.""" + assertion = ActivityAssertion( + assertion={"type": "message"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = assertion.check(activities) + assert passes is False + assert error is not None + assert "Expected exactly one activity" in error + assert "found 3" in error + + def test_check_one_with_empty_activities(self): + """Test ONE quantifier with empty activities list.""" + assertion = ActivityAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ONE + ) + passes, error = assertion.check([]) + assert passes is False + assert "found 0" in error + + +class TestActivityAssertionCheckWithQuantifierAny: + """Tests for check() method with ANY quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="message", text="World"), + Activity(type="event", name="test_event"), + ] + + def test_check_any_basic_functionality(self, activities): + """Test that ANY quantifier exists and can be used.""" + # ANY quantifier doesn't have special logic in the current implementation + # but should not cause errors + assertion = ActivityAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ANY + ) + passes, error = assertion.check(activities) + # Based on the implementation, ANY behaves like checking if count > 0 + assert passes is True + assert error is None + + +class TestActivityAssertionCallable: + """Tests for __call__ method.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="event", name="test_event"), + ] + + def test_call_invokes_check(self, activities): + """Test that calling assertion instance invokes check().""" + assertion = ActivityAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ALL + ) + result = assertion(activities) + assert isinstance(result, tuple) + assert len(result) == 2 + assert isinstance(result[0], bool) + + def test_call_returns_same_as_check(self, activities): + """Test that __call__ returns same result as check().""" + assertion = ActivityAssertion( + assertion={"type": "message"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ALL, + ) + call_result = assertion(activities) + check_result = assertion.check(activities) + assert call_result == check_result + + +class TestActivityAssertionFromConfig: + """Tests for from_config static method.""" + + def test_from_config_minimal(self): + """Test creating assertion from minimal config.""" + config = {} + assertion = ActivityAssertion.from_config(config) + assert assertion._assertion == {} + assert assertion._quantifier == AssertionQuantifier.ALL + + def test_from_config_with_assertion(self): + """Test creating assertion from config with assertion field.""" + config = {"assertion": {"type": "message", "text": "Hello"}} + assertion = ActivityAssertion.from_config(config) + assert assertion._assertion == config["assertion"] + + def test_from_config_with_selector(self): + """Test creating assertion from config with selector field.""" + config = {"selector": {"selector": {"type": "message"}, "quantifier": "ALL"}} + assertion = ActivityAssertion.from_config(config) + assert assertion._selector is not None + + def test_from_config_with_quantifier(self): + """Test creating assertion from config with quantifier field.""" + config = {"quantifier": "one"} + assertion = ActivityAssertion.from_config(config) + assert assertion._quantifier == AssertionQuantifier.ONE + + def test_from_config_with_all_fields(self): + """Test creating assertion from config with all fields.""" + config = { + "assertion": {"type": "message"}, + "selector": { + "selector": {"text": "Hello"}, + "quantifier": "ONE", + "index": 0, + }, + "quantifier": "all", + } + assertion = ActivityAssertion.from_config(config) + assert assertion._assertion == {"type": "message"} + assert assertion._quantifier == AssertionQuantifier.ALL + + def test_from_config_with_case_insensitive_quantifier(self): + """Test from_config handles case-insensitive quantifier strings.""" + for quantifier_str in ["all", "ALL", "All", "ONE", "one", "NONE", "none"]: + config = {"quantifier": quantifier_str} + assertion = ActivityAssertion.from_config(config) + assert isinstance(assertion._quantifier, AssertionQuantifier) + + def test_from_config_with_complex_assertion(self): + """Test creating assertion from config with complex nested assertion.""" + config = { + "assertion": {"type": "message", "channelData": {"nested": {"value": 123}}}, + "quantifier": "all", + } + assertion = ActivityAssertion.from_config(config) + assert assertion._assertion["type"] == "message" + assert assertion._assertion["channelData"]["nested"]["value"] == 123 + + +class TestActivityAssertionCombineErrors: + """Tests for _combine_assertion_errors static method.""" + + def test_combine_empty_errors(self): + """Test combining empty error list.""" + result = ActivityAssertion._combine_assertion_errors([]) + assert result == "" + + def test_combine_single_error(self): + """Test combining single error.""" + from microsoft_agents.testing.assertions.type_defs import ( + AssertionErrorData, + FieldAssertionType, + ) + + error = AssertionErrorData( + field_path="activity.text", + actual_value="Hello", + assertion="World", + assertion_type=FieldAssertionType.EQUALS, + ) + result = ActivityAssertion._combine_assertion_errors([error]) + assert "activity.text" in result + assert "Hello" in result + + def test_combine_multiple_errors(self): + """Test combining multiple errors.""" + from microsoft_agents.testing.assertions.type_defs import ( + AssertionErrorData, + FieldAssertionType, + ) + + errors = [ + AssertionErrorData( + field_path="activity.text", + actual_value="Hello", + assertion="World", + assertion_type=FieldAssertionType.EQUALS, + ), + AssertionErrorData( + field_path="activity.type", + actual_value="message", + assertion="event", + assertion_type=FieldAssertionType.EQUALS, + ), + ] + result = ActivityAssertion._combine_assertion_errors(errors) + assert "activity.text" in result + assert "activity.type" in result + assert "\n" in result + + +class TestActivityAssertionIntegration: + """Integration tests with realistic scenarios.""" + + @pytest.fixture + def conversation_activities(self): + """Create a realistic conversation flow.""" + return [ + Activity(type="conversationUpdate", name="add_member"), + Activity(type="message", text="Hello bot", from_property={"id": "user1"}), + Activity(type="message", text="Hi there!", from_property={"id": "bot"}), + Activity( + type="message", text="How are you?", from_property={"id": "user1"} + ), + Activity( + type="message", text="I'm doing well!", from_property={"id": "bot"} + ), + Activity(type="typing"), + Activity(type="message", text="Goodbye", from_property={"id": "user1"}), + ] + + def test_assert_all_user_messages_have_from_property(self, conversation_activities): + """Test that all user messages have a from_property.""" + assertion = ActivityAssertion( + assertion={"from_property": {"id": "user1"}}, + selector=Selector( + selector={"type": "message", "from_property": {"id": "user1"}}, + ), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(conversation_activities) + assert passes is True + + def test_assert_no_error_messages(self, conversation_activities): + """Test that there are no error messages in the conversation.""" + assertion = ActivityAssertion( + assertion={"type": "error"}, + selector=Selector(selector={}), + quantifier=AssertionQuantifier.NONE, + ) + passes, error = assertion.check(conversation_activities) + assert passes is True + + def test_assert_exactly_one_conversation_update(self, conversation_activities): + """Test that there's exactly one conversation update.""" + assertion = ActivityAssertion( + assertion={"type": "conversationUpdate"}, + selector=Selector(selector={"type": "conversationUpdate"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = assertion.check(conversation_activities) + assert passes is True + + def test_assert_first_message_is_greeting(self, conversation_activities): + """Test that the first message contains a greeting.""" + assertion = ActivityAssertion( + assertion={"text": {"assertion_type": "CONTAINS", "assertion": "Hello"}}, + selector=Selector(selector={"type": "message"}, index=0), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(conversation_activities) + assert passes is True + + def test_complex_multi_field_assertion(self, conversation_activities): + """Test complex assertion with multiple fields.""" + assertion = ActivityAssertion( + assertion={"type": "message", "from_property": {"id": "bot"}}, + selector=Selector( + selector={"type": "message", "from_property": {"id": "bot"}}, + ), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(conversation_activities) + assert passes is True + + +class TestActivityAssertionEdgeCases: + """Tests for edge cases and boundary conditions.""" + + def test_empty_assertion_matches_all(self): + """Test that empty assertion matches all activities.""" + activities = [ + Activity(type="message", text="Hello"), + Activity(type="event", name="test"), + ] + assertion = ActivityAssertion(assertion={}, quantifier=AssertionQuantifier.ALL) + passes, error = assertion.check(activities) + assert passes is True + + def test_assertion_with_none_values(self): + """Test assertion with None values.""" + activities = [Activity(type="message")] + assertion = ActivityAssertion( + assertion={"text": None}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check(activities) + # This behavior depends on check_activity implementation + assert isinstance(passes, bool) + + def test_selector_filters_before_assertion(self): + """Test that selector filters activities before assertion check.""" + activities = [ + Activity(type="message", text="Hello"), + Activity(type="event", name="test"), + Activity(type="message", text="World"), + ] + # Selector gets only messages, assertion checks for specific text + assertion = ActivityAssertion( + assertion={"text": "Hello"}, + selector=Selector(selector={"type": "message"}, index=0), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert passes is True + + def test_assertion_error_message_format(self): + """Test that error messages are properly formatted.""" + activities = [Activity(type="message", text="Wrong")] + assertion = ActivityAssertion( + assertion={"text": "Expected"}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check(activities) + assert passes is False + assert error is not None + assert "Activity did not match the assertion" in error + assert "Error:" in error + + def test_multiple_activities_same_content(self): + """Test handling multiple activities with identical content.""" + activities = [ + Activity(type="message", text="Hello"), + Activity(type="message", text="Hello"), + Activity(type="message", text="Hello"), + ] + assertion = ActivityAssertion( + assertion={"text": "Hello"}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check(activities) + assert passes is True + + def test_assertion_with_unset_fields(self): + """Test assertion against activities with unset fields.""" + activities = [ + Activity(type="message"), # No text field set + ] + assertion = ActivityAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check(activities) + assert passes is True + + +class TestActivityAssertionErrorMessages: + """Tests specifically for error message content and formatting.""" + + def test_all_quantifier_error_includes_activity(self): + """Test that ALL quantifier error includes the failing activity.""" + activities = [Activity(type="message", text="Wrong")] + assertion = ActivityAssertion( + assertion={"text": "Expected"}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check(activities) + assert passes is False + assert "Activity did not match the assertion" in error + + def test_none_quantifier_error_includes_activity(self): + """Test that NONE quantifier error includes the matching activity.""" + activities = [Activity(type="message", text="Unexpected")] + assertion = ActivityAssertion( + assertion={"text": "Unexpected"}, quantifier=AssertionQuantifier.NONE + ) + passes, error = assertion.check(activities) + assert passes is False + assert "Activity matched the assertion when none were expected" in error + + def test_one_quantifier_error_includes_count(self): + """Test that ONE quantifier error includes the actual count.""" + activities = [ + Activity(type="message"), + Activity(type="message"), + ] + assertion = ActivityAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ONE + ) + passes, error = assertion.check(activities) + assert passes is False + assert "Expected exactly one activity" in error + assert "2" in error + + +class TestActivityAssertionRealWorldScenarios: + """Tests simulating real-world bot testing scenarios.""" + + def test_validate_welcome_message_sent(self): + """Test that a welcome message is sent when user joins.""" + activities = [ + Activity(type="conversationUpdate", name="add_member"), + Activity(type="message", text="Welcome to our bot!"), + ] + assertion = ActivityAssertion( + assertion={ + "type": "message", + "text": {"assertion_type": "CONTAINS", "assertion": "Welcome"}, + }, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert passes is True + + def test_validate_no_duplicate_responses(self): + """Test that bot doesn't send duplicate responses.""" + activities = [ + Activity(type="message", text="Response 1"), + Activity(type="message", text="Response 2"), + Activity(type="message", text="Response 3"), + ] + # Check that exactly one of each unique response exists + for response_text in ["Response 1", "Response 2", "Response 3"]: + assertion = ActivityAssertion( + assertion={"text": response_text}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = assertion.check(activities) + assert passes is True + + def test_validate_error_handling_response(self): + """Test that bot responds appropriately to errors.""" + activities = [ + Activity(type="message", text="invalid command"), + Activity(type="message", text="I'm sorry, I didn't understand that."), + ] + assertion = ActivityAssertion( + assertion={ + "text": { + "assertion_type": "RE_MATCH", + "assertion": "sorry|understand|help", + } + }, + selector=Selector(selector={"type": "message"}, index=-1), # Last message + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert not passes + assert "sorry" in error and "understand" in error and "help" in error + assert FieldAssertionType.RE_MATCH.name in error + + def test_validate_typing_indicator_before_response(self): + """Test that typing indicator is sent before response.""" + activities = [ + Activity(type="message", text="User question"), + Activity(type="typing"), + Activity(type="message", text="Bot response"), + ] + # Verify typing indicator exists + typing_assertion = ActivityAssertion( + assertion={"type": "typing"}, + selector=Selector(selector={"type": "typing"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = typing_assertion.check(activities) + assert passes is True + + def test_validate_conversation_flow_order(self): + """Test that conversation follows expected flow.""" + activities = [ + Activity(type="conversationUpdate"), + Activity(type="message", text="User: Hello"), + Activity(type="typing"), + Activity(type="message", text="Bot: Hi!"), + ] + + # Test each step individually + steps = [ + ({"type": "conversationUpdate"}, 0), + ({"type": "message"}, 1), + ({"type": "typing"}, 2), + ({"type": "message"}, 3), + ] + + for assertion_dict, expected_index in steps: + assertion = ActivityAssertion( + assertion=assertion_dict, + selector=Selector(selector={}, index=expected_index), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert passes is True, f"Failed at index {expected_index}: {error}" diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py index e238312b..cafc556d 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py @@ -232,7 +232,7 @@ class TestCheckFieldEdgeCases: def test_invalid_assertion_type(self): # Passing an unsupported assertion type should return False with pytest.raises(ValueError): - assert check_field("test", "test", "INVALID_TYPE") + assert check_field("test", "test", "INVALID_TYPE") def test_none_actual_value_with_equals(self): assert check_field(None, "test", FieldAssertionType.EQUALS) is False diff --git a/dev/microsoft-agents-testing/tests/assertions/test_selector.py b/dev/microsoft-agents-testing/tests/assertions/test_selector.py index fc9ac25a..5360cb6b 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_selector.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_selector.py @@ -4,84 +4,7 @@ import pytest from microsoft_agents.activity import Activity -from microsoft_agents.testing.assertions.selector import Selector, SelectorQuantifier - - -class TestSelectorInitialization: - """Tests for Selector initialization and validation.""" - - def test_init_with_quantifier_all(self): - """Test initialization with ALL quantifier.""" - selector = Selector(selector={}, quantifier=SelectorQuantifier.ALL) - assert selector._quantifier == SelectorQuantifier.ALL - assert selector._index is None - assert selector._selector == {} - - def test_init_with_quantifier_one_default_index(self): - """Test initialization with ONE quantifier defaults index to 0.""" - selector = Selector(selector={}, quantifier=SelectorQuantifier.ONE) - assert selector._quantifier == SelectorQuantifier.ONE - assert selector._index == 0 - - def test_init_with_quantifier_one_explicit_index(self): - """Test initialization with ONE quantifier and explicit index.""" - selector = Selector(selector={}, quantifier=SelectorQuantifier.ONE, index=2) - assert selector._quantifier == SelectorQuantifier.ONE - assert selector._index == 2 - - def test_init_with_index_only(self): - """Test initialization with index only (implies ONE quantifier).""" - selector = Selector(selector={}, index=1) - assert selector._quantifier == SelectorQuantifier.ONE - assert selector._index == 1 - - def test_init_with_string_quantifier(self): - """Test initialization with string quantifier.""" - selector = Selector(selector={}, quantifier="all") - assert selector._quantifier == SelectorQuantifier.ALL - - selector = Selector(selector={}, quantifier="ONE") - assert selector._quantifier == SelectorQuantifier.ONE - - def test_init_with_activity_selector(self): - """Test initialization with Activity object as selector.""" - activity = Activity(type="message", text="Hello") - selector = Selector(selector=activity, quantifier=SelectorQuantifier.ALL) - assert "type" in selector._selector - assert selector._selector["type"] == "message" - assert selector._selector["text"] == "Hello" - - def test_init_with_dict_selector(self): - """Test initialization with dictionary selector.""" - selector_dict = {"type": "message", "text": "Hello"} - selector = Selector(selector=selector_dict, quantifier=SelectorQuantifier.ALL) - assert selector._selector == selector_dict - - def test_init_with_none_selector(self): - """Test initialization with None selector defaults to empty dict.""" - selector = Selector(selector=None, quantifier=SelectorQuantifier.ALL) - assert selector._selector == {} - - def test_init_without_quantifier_or_index_raises_error(self): - """Test that initialization without quantifier or index raises ValueError.""" - with pytest.raises(ValueError, match="Either quantifier or index must be provided"): - Selector(selector={}) - - def test_init_with_invalid_quantifier_string_raises_error(self): - """Test that invalid quantifier string raises ValueError.""" - with pytest.raises(ValueError, match="Invalid quantifier"): - Selector(selector={}, quantifier="INVALID") - - def test_init_with_all_quantifier_and_index_raises_error(self): - """Test that ALL quantifier with index raises ValueError.""" - with pytest.raises(ValueError, match="Index should not be set when quantifier is ALL"): - Selector(selector={}, quantifier=SelectorQuantifier.ALL, index=0) - - def test_init_with_negative_index(self): - """Test initialization with negative index.""" - selector = Selector(selector={}, index=-1) - assert selector._index == -1 - assert selector._quantifier == SelectorQuantifier.ONE +from microsoft_agents.testing.assertions.selector import Selector class TestSelectorSelectWithQuantifierAll: @@ -99,7 +22,7 @@ def activities(self): def test_select_all_matching_type(self, activities): """Test selecting all activities with matching type.""" - selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + selector = Selector(selector={"type": "message"}) result = selector.select(activities) assert len(result) == 3 assert all(a.type == "message" for a in result) @@ -108,7 +31,6 @@ def test_select_all_matching_multiple_fields(self, activities): """Test selecting all activities matching multiple fields.""" selector = Selector( selector={"type": "message", "text": "Hello"}, - quantifier=SelectorQuantifier.ALL ) result = selector.select(activities) assert len(result) == 1 @@ -118,20 +40,19 @@ def test_select_all_no_matches(self, activities): """Test selecting all with no matches returns empty list.""" selector = Selector( selector={"type": "nonexistent"}, - quantifier=SelectorQuantifier.ALL ) result = selector.select(activities) assert len(result) == 0 def test_select_all_empty_selector(self, activities): """Test selecting all with empty selector returns all activities.""" - selector = Selector(selector={}, quantifier=SelectorQuantifier.ALL) + selector = Selector(selector={}) result = selector.select(activities) assert len(result) == len(activities) def test_select_all_from_empty_list(self): """Test selecting from empty activity list.""" - selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + selector = Selector(selector={"type": "message"}) result = selector.select([]) assert len(result) == 0 @@ -151,29 +72,21 @@ def activities(self): def test_select_one_default_index(self, activities): """Test selecting one activity with default index (0).""" - selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ONE) + selector = Selector(selector={"type": "message"}, index=0) result = selector.select(activities) assert len(result) == 1 assert result[0].text == "First" def test_select_one_explicit_index(self, activities): """Test selecting one activity with explicit index.""" - selector = Selector( - selector={"type": "message"}, - quantifier=SelectorQuantifier.ONE, - index=1 - ) + selector = Selector(selector={"type": "message"}, index=1) result = selector.select(activities) assert len(result) == 1 assert result[0].text == "Second" def test_select_one_last_index(self, activities): """Test selecting one activity with last valid index.""" - selector = Selector( - selector={"type": "message"}, - quantifier=SelectorQuantifier.ONE, - index=2 - ) + selector = Selector(selector={"type": "message"}, index=2) result = selector.select(activities) assert len(result) == 1 assert result[0].text == "Third" @@ -194,11 +107,7 @@ def test_select_one_negative_index_from_start(self, activities): def test_select_one_index_out_of_range(self, activities): """Test selecting with index out of range returns empty list.""" - selector = Selector( - selector={"type": "message"}, - quantifier=SelectorQuantifier.ONE, - index=10 - ) + selector = Selector(selector={"type": "message"}, index=10) result = selector.select(activities) assert len(result) == 0 @@ -210,10 +119,7 @@ def test_select_one_negative_index_out_of_range(self, activities): def test_select_one_no_matches(self, activities): """Test selecting one with no matches returns empty list.""" - selector = Selector( - selector={"type": "nonexistent"}, - quantifier=SelectorQuantifier.ONE - ) + selector = Selector(selector={"type": "nonexistent"}, index=0) result = selector.select(activities) assert len(result) == 0 @@ -238,7 +144,7 @@ def activities(self): def test_select_first_with_matches(self, activities): """Test select_first returns first matching activity.""" - selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + selector = Selector(selector={"type": "message"}) result = selector.select_first(activities) assert result is not None assert result.text == "First" @@ -247,14 +153,13 @@ def test_select_first_no_matches(self, activities): """Test select_first with no matches returns None.""" selector = Selector( selector={"type": "nonexistent"}, - quantifier=SelectorQuantifier.ALL ) result = selector.select_first(activities) assert result is None def test_select_first_empty_list(self): """Test select_first on empty list returns None.""" - selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + selector = Selector(selector={"type": "message"}) result = selector.select_first([]) assert result is None @@ -279,7 +184,7 @@ def activities(self): def test_call_invokes_select(self, activities): """Test that calling selector instance invokes select().""" - selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + selector = Selector(selector={"type": "message"}) result = selector(activities) assert len(result) == 1 assert result[0].text == "Hello" @@ -292,84 +197,6 @@ def test_call_returns_same_as_select(self, activities): assert call_result == select_result -class TestSelectorFromConfig: - """Tests for from_config static method.""" - - def test_from_config_with_all_quantifier(self): - """Test creating selector from config with ALL quantifier.""" - config = { - "selector": {"type": "message"}, - "quantifier": "ALL" - } - selector = Selector.from_config(config) - assert selector._quantifier == SelectorQuantifier.ALL - assert selector._selector == {"type": "message"} - assert selector._index is None - - def test_from_config_with_one_quantifier(self): - """Test creating selector from config with ONE quantifier.""" - config = { - "selector": {"type": "message"}, - "quantifier": "ONE" - } - selector = Selector.from_config(config) - assert selector._quantifier == SelectorQuantifier.ONE - assert selector._index == 0 - - def test_from_config_with_index(self): - """Test creating selector from config with index.""" - config = { - "selector": {"type": "message"}, - "index": 2 - } - selector = Selector.from_config(config) - assert selector._quantifier == SelectorQuantifier.ONE - assert selector._index == 2 - - def test_from_config_with_both_quantifier_and_index(self): - """Test creating selector from config with both quantifier and index.""" - config = { - "selector": {"type": "message"}, - "quantifier": "ONE", - "index": 1 - } - selector = Selector.from_config(config) - assert selector._quantifier == SelectorQuantifier.ONE - assert selector._index == 1 - - def test_from_config_missing_selector_defaults_to_empty(self): - """Test creating selector from config without selector field.""" - config = { - "quantifier": "ALL" - } - selector = Selector.from_config(config) - assert selector._selector == {} - - def test_from_config_empty_selector(self): - """Test creating selector from config with empty selector.""" - config = { - "selector": {}, - "quantifier": "ALL" - } - selector = Selector.from_config(config) - assert selector._selector == {} - - def test_from_config_complex_selector(self): - """Test creating selector from config with complex selector.""" - config = { - "selector": { - "type": "message", - "text": "Hello", - "channelData": {"key": "value"} - }, - "quantifier": "ALL" - } - selector = Selector.from_config(config) - assert selector._selector["type"] == "message" - assert selector._selector["text"] == "Hello" - assert selector._selector["channelData"]["key"] == "value" - - class TestSelectorIntegration: """Integration tests with realistic scenarios.""" @@ -380,8 +207,12 @@ def conversation_activities(self): Activity(type="conversationUpdate", name="add_member"), Activity(type="message", text="Hello bot", from_property={"id": "user1"}), Activity(type="message", text="Hi there!", from_property={"id": "bot"}), - Activity(type="message", text="How are you?", from_property={"id": "user1"}), - Activity(type="message", text="I'm doing well!", from_property={"id": "bot"}), + Activity( + type="message", text="How are you?", from_property={"id": "user1"} + ), + Activity( + type="message", text="I'm doing well!", from_property={"id": "bot"} + ), Activity(type="typing"), Activity(type="message", text="Goodbye", from_property={"id": "user1"}), ] @@ -390,7 +221,6 @@ def test_select_all_user_messages(self, conversation_activities): """Test selecting all messages from a specific user.""" selector = Selector( selector={"type": "message", "from_property": {"id": "user1"}}, - quantifier=SelectorQuantifier.ALL ) result = selector.select(conversation_activities) assert len(result) == 3 @@ -398,8 +228,7 @@ def test_select_all_user_messages(self, conversation_activities): def test_select_first_bot_response(self, conversation_activities): """Test selecting first bot response.""" selector = Selector( - selector={"type": "message", "from_property": {"id": "bot"}}, - index=0 + selector={"type": "message", "from_property": {"id": "bot"}}, index=0 ) result = selector.select(conversation_activities) assert len(result) == 1 @@ -407,10 +236,7 @@ def test_select_first_bot_response(self, conversation_activities): def test_select_last_message_negative_index(self, conversation_activities): """Test selecting last message using negative index.""" - selector = Selector( - selector={"type": "message"}, - index=-1 - ) + selector = Selector(selector={"type": "message"}, index=-1) result = selector.select(conversation_activities) assert len(result) == 1 assert result[0].text == "Goodbye" @@ -419,7 +245,6 @@ def test_select_typing_indicator(self, conversation_activities): """Test selecting typing indicator.""" selector = Selector( selector={"type": "typing"}, - quantifier=SelectorQuantifier.ALL ) result = selector.select(conversation_activities) assert len(result) == 1 @@ -428,7 +253,6 @@ def test_select_conversation_update(self, conversation_activities): """Test selecting conversation update events.""" selector = Selector( selector={"type": "conversationUpdate"}, - quantifier=SelectorQuantifier.ALL ) result = selector.select(conversation_activities) assert len(result) == 1 @@ -445,7 +269,7 @@ def test_select_with_partial_match(self): Activity(type="message", text="World"), ] # Only matching on type, not text - selector = Selector(selector={"type": "message"}, quantifier=SelectorQuantifier.ALL) + selector = Selector(selector={"type": "message"}) result = selector.select(activities) assert len(result) == 2 @@ -457,7 +281,6 @@ def test_select_with_none_values(self): ] selector = Selector( selector={"type": "message", "text": None}, - quantifier=SelectorQuantifier.ALL ) result = selector.select(activities) # This depends on how check_activity handles None @@ -483,4 +306,4 @@ def test_select_with_boundary_negative_one(self): activities = [Activity(type="message", text="Single")] selector = Selector(selector={"type": "message"}, index=-1) result = selector.select(activities) - assert len(result) == 1 \ No newline at end of file + assert len(result) == 1 From 4df4788779e12f3ec36f119427a8131cd3f95133 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Fri, 14 Nov 2025 08:50:28 -0800 Subject: [PATCH 78/81] DataDrivenTest implementation connected with default config --- .../testing/assertions/check_field.py | 3 + .../testing/assertions/selector.py | 5 +- .../data_driven/data_driven_test.py | 20 ++++-- .../testing/utils/__init__.py | 4 +- .../testing/utils/populate.py | 4 +- .../tests/utils/test_populate.py | 66 +++++++++---------- 6 files changed, 58 insertions(+), 44 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py index 2eab22ef..6693f706 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import re from typing import Any, Optional diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py index d8bf2cb6..c469c65b 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py @@ -1,6 +1,7 @@ -from __future__ import annotations +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. -from enum import Enum +from __future__ import annotations from microsoft_agents.activity import Activity diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py index c9bf5258..d7e283b3 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py @@ -9,13 +9,16 @@ from microsoft_agents.activity import Activity -from microsoft_agents.testing.assertions import ( - ActivityAssertion, - assert_activity, +from microsoft_agents.testing.assertions import ActivityAssertion +from microsoft_agents.testing.utils import ( + populate_activity, + update_with_defaults, ) +from ..core import AgentClient, ResponseClient class DataDrivenTest: + """Data driven test runner.""" def __init__(self, test_flow: dict) -> None: self._description = test_flow.get("description", "") @@ -60,7 +63,11 @@ async def _sleep(self, sleep_data: dict) -> None: duration = self._sleep_defaults.get("duration", 0) await asyncio.sleep(duration) - async def run(self, agent_client, response_client) -> None: + async def run(self, agent_client: AgentClient, response_client: ResponseClient) -> None: + """Run the data driven test. + + :param agent_client: The agent client to send activities to. + """ responses = [] for step in self._test: @@ -74,7 +81,10 @@ async def run(self, agent_client, response_client) -> None: await agent_client.send_activity(input_activity) elif step_type == "assertion": - activity_assertion = ActivityAssertion.from_config(step) + assertion = self._load_assertion(step) + update_with_defaults(assertion, self._assertion_defaults) + + activity_assertion = ActivityAssertion.from_config(assertion) responses.extend(await response_client.pop()) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py index aae37c77..85ccc432 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -1,11 +1,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from .populate import default_update, populate_activity +from .populate import update_with_defaults, populate_activity from .misc import get_host_and_port, normalize_activity_data __all__ = [ - "default_update", + "update_with_defaults", "populate_activity", "get_host_and_port", "normalize_activity_data", diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py index 979050bb..acec37a9 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py @@ -4,7 +4,7 @@ from microsoft_agents.activity import Activity -def default_update(original: dict, defaults: dict) -> None: +def update_with_defaults(original: dict, defaults: dict) -> None: """Populate a dictionary with default values. :param original: The original dictionary to populate. @@ -15,7 +15,7 @@ def default_update(original: dict, defaults: dict) -> None: if key not in original: original[key] = defaults[key] elif isinstance(original[key], dict) and isinstance(defaults[key], dict): - default_update(original[key], defaults[key]) + update_with_defaults(original[key], defaults[key]) def populate_activity(original: Activity, defaults: Activity | dict) -> Activity: diff --git a/dev/microsoft-agents-testing/tests/utils/test_populate.py b/dev/microsoft-agents-testing/tests/utils/test_populate.py index 5140e772..ff464eef 100644 --- a/dev/microsoft-agents-testing/tests/utils/test_populate.py +++ b/dev/microsoft-agents-testing/tests/utils/test_populate.py @@ -4,52 +4,52 @@ import pytest from microsoft_agents.activity import Activity, ChannelAccount, ConversationAccount -from microsoft_agents.testing.utils.populate import default_update, populate_activity +from microsoft_agents.testing.utils.populate import update_with_defaults, populate_activity -class TestDefaultUpdate: - """Tests for the default_update function.""" +class TestUpdateWithDefaults: + """Tests for the update_with_defaults function.""" - def test_default_update_with_empty_original(self): + def test_update_with_defaults_with_empty_original(self): """Test that defaults are added to an empty dictionary.""" original = {} defaults = {"key1": "value1", "key2": "value2"} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == {"key1": "value1", "key2": "value2"} - def test_default_update_with_empty_defaults(self): + def test_update_with_defaults_with_empty_defaults(self): """Test that original dictionary is unchanged when defaults is empty.""" original = {"key1": "value1"} defaults = {} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == {"key1": "value1"} - def test_default_update_with_non_overlapping_keys(self): + def test_update_with_defaults_with_non_overlapping_keys(self): """Test that defaults are added when keys don't overlap.""" original = {"key1": "value1"} defaults = {"key2": "value2", "key3": "value3"} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == {"key1": "value1", "key2": "value2", "key3": "value3"} - def test_default_update_preserves_existing_values(self): + def test_update_with_defaults_preserves_existing_values(self): """Test that existing values in original are not overwritten.""" original = {"key1": "original_value", "key2": "value2"} defaults = {"key1": "default_value", "key3": "value3"} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == { "key1": "original_value", "key2": "value2", "key3": "value3", } - def test_default_update_with_nested_dicts(self): + def test_update_with_defaults_with_nested_dicts(self): """Test that nested dictionaries are recursively updated.""" original = {"nested": {"key1": "original"}} defaults = {"nested": {"key1": "default", "key2": "value2"}} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == {"nested": {"key1": "original", "key2": "value2"}} - def test_default_update_with_deeply_nested_dicts(self): + def test_update_with_defaults_with_deeply_nested_dicts(self): """Test recursive update with deeply nested structures.""" original = {"level1": {"level2": {"key1": "original"}}} defaults = { @@ -58,7 +58,7 @@ def test_default_update_with_deeply_nested_dicts(self): "level2b": {"key3": "value3"}, } } - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == { "level1": { "level2": {"key1": "original", "key2": "value2"}, @@ -66,14 +66,14 @@ def test_default_update_with_deeply_nested_dicts(self): } } - def test_default_update_adds_nested_dict_when_missing(self): + def test_update_with_defaults_adds_nested_dict_when_missing(self): """Test that nested dicts are added when they don't exist in original.""" original = {"key1": "value1"} defaults = {"nested": {"key2": "value2"}} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == {"key1": "value1", "nested": {"key2": "value2"}} - def test_default_update_with_mixed_types(self): + def test_update_with_defaults_with_mixed_types(self): """Test with various value types: strings, numbers, booleans, lists.""" original = {"str": "text", "num": 42} defaults = { @@ -82,7 +82,7 @@ def test_default_update_with_mixed_types(self): "list": [1, 2, 3], "none": None, } - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == { "str": "text", "num": 42, @@ -91,51 +91,51 @@ def test_default_update_with_mixed_types(self): "none": None, } - def test_default_update_with_none_values(self): + def test_update_with_defaults_with_none_values(self): """Test that None values in defaults are added.""" original = {"key1": "value1"} defaults = {"key2": None} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == {"key1": "value1", "key2": None} - def test_default_update_preserves_none_in_original(self): + def test_update_with_defaults_preserves_none_in_original(self): """Test that None values in original are preserved.""" original = {"key1": None} defaults = {"key1": "default_value"} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == {"key1": None} - def test_default_update_with_list_values(self): + def test_update_with_defaults_with_list_values(self): """Test that list values are not merged, only added if missing.""" original = {"list1": [1, 2]} defaults = {"list1": [3, 4], "list2": [5, 6]} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == {"list1": [1, 2], "list2": [5, 6]} - def test_default_update_type_mismatch_original_wins(self): + def test_update_with_defaults_type_mismatch_original_wins(self): """Test that when types differ, original value is preserved.""" original = {"key1": "string_value"} defaults = {"key1": {"nested": "dict"}} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == {"key1": "string_value"} - def test_default_update_type_mismatch_defaults_dict(self): + def test_update_with_defaults_type_mismatch_defaults_dict(self): """Test that when original is dict and default is not, original is preserved.""" original = {"key1": {"nested": "dict"}} defaults = {"key1": "string_value"} - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == {"key1": {"nested": "dict"}} - def test_default_update_modifies_in_place(self): + def test_update_with_defaults_modifies_in_place(self): """Test that the function modifies the original dict in place.""" original = {"key1": "value1"} original_id = id(original) defaults = {"key2": "value2"} - default_update(original, defaults) + update_with_defaults(original, defaults) assert id(original) == original_id assert original == {"key1": "value1", "key2": "value2"} - def test_default_update_with_complex_nested_structure(self): + def test_update_with_defaults_with_complex_nested_structure(self): """Test with complex real-world-like nested structure.""" original = { "user": {"name": "Alice", "settings": {"theme": "dark"}}, @@ -149,7 +149,7 @@ def test_default_update_with_complex_nested_structure(self): }, "channel": "default-channel", } - default_update(original, defaults) + update_with_defaults(original, defaults) assert original == { "user": { "name": "Alice", From 0c195b144e67b12f035d64c19a0dfa01bcc403ec Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Fri, 14 Nov 2025 08:54:38 -0800 Subject: [PATCH 79/81] More test cases and formatting --- .../integration/data_driven/__init__.py | 3 +- .../testing/integration/data_driven/ddt.py | 52 +- .../data_driven/test_data_driven_test.py | 513 ++++++++++++++++++ .../data_driven/test_data_driven_tester.py | 0 4 files changed, 540 insertions(+), 28 deletions(-) create mode 100644 dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py delete mode 100644 dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_tester.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py index aab93742..0d6b013c 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py @@ -1,4 +1,3 @@ from .data_driven_test import DataDrivenTest -from .ddt import ddt -__all__ = ["DataDrivenTest", "ddt"] +__all__ = ["DataDrivenTest"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py index a627e7e3..d47e3627 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py @@ -1,41 +1,41 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. +# # Copyright (c) Microsoft Corporation. All rights reserved. +# # Licensed under the MIT License. -from typing import Callable +# from typing import Callable -import pytest +# import pytest -from microsoft_agents.testing.integration.core import Integration +# from microsoft_agents.testing.integration.core import Integration -from .data_driven_test import DataDrivenTest +# from .data_driven_test import DataDrivenTest -def _add_test_method(test_cls: Integration, test_path: str, base_dir: str) -> None: +# def _add_test_method(test_cls: Integration, test_path: str, base_dir: str) -> None: - test_case_name = f"test_data_driven__{test_path.replace('/', '_').replace('.', '_')}" +# test_case_name = f"test_data_driven__{test_path.replace('/', '_').replace('.', '_')}" - @pytest.mark.asyncio - async def _func(self, agent_client, response_client) -> None: - ddt = DataDrivenTest(f"{base_dir}/{test_path}") - await ddt.run(agent_client, response_client) +# @pytest.mark.asyncio +# async def _func(self, agent_client, response_client) -> None: +# ddt = DataDrivenTest(f"{base_dir}/{test_path}") +# await ddt.run(agent_client, response_client) - setattr(test_cls, test_case_name, func) +# setattr(test_cls, test_case_name, func) -def ddt(test_path: str) -> Callable[[Integration], Integration]: +# def ddt(test_path: str) -> Callable[[Integration], Integration]: - def decorator(test_cls: Integration) -> Integration: +# def decorator(test_cls: Integration) -> Integration: - test_case_name = f"test_data_driven__{test_path.replace('/', '_').replace('.', '_')}" +# test_case_name = f"test_data_driven__{test_path.replace('/', '_').replace('.', '_')}" - async def func(self, agent_client, response_client) -> None: - ddt = DataDrivenTest(test_path) +# async def func(self, agent_client, response_client) -> None: +# ddt = DataDrivenTest(test_path) - responses = [] +# responses = [] - await for step in ddt: - if isinstance(step, Activity): - await agent_client.send_activity(step) - elif isinstance(step, dict): - # assertion - responses.extend(await response_client.pop()) +# await for step in ddt: +# if isinstance(step, Activity): +# await agent_client.send_activity(step) +# elif isinstance(step, dict): +# # assertion +# responses.extend(await response_client.pop()) - return decorator \ No newline at end of file +# return decorator \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py new file mode 100644 index 00000000..82584406 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py @@ -0,0 +1,513 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +import tempfile +import os +from unittest.mock import AsyncMock, Mock, patch, MagicMock + +from microsoft_agents.activity import Activity, ActivityTypes +from microsoft_agents.testing.integration.data_driven import DataDrivenTest +from microsoft_agents.testing.integration.core import AgentClient, ResponseClient + + +class TestDataDrivenTestInit: + """Tests for DataDrivenTest initialization.""" + + def test_init_minimal_config(self): + """Test initialization with minimal configuration.""" + test_flow = {"test": []} + ddt = DataDrivenTest(test_flow) + + assert ddt._description == "" + assert ddt._input_defaults == {} + assert ddt._assertion_defaults == {} + assert ddt._sleep_defaults == {} + assert ddt._test == [] + + def test_init_with_description(self): + """Test initialization with description.""" + test_flow = { + "description": "Test description", + "test": [] + } + ddt = DataDrivenTest(test_flow) + + assert ddt._description == "Test description" + + def test_init_with_defaults(self): + """Test initialization with defaults.""" + test_flow = { + "defaults": { + "input": {"type": "message", "text": "default text"}, + "assertion": {"type": "message"}, + "sleep": {"duration": 1.5} + }, + "test": [] + } + ddt = DataDrivenTest(test_flow) + + assert ddt._input_defaults == {"type": "message", "text": "default text"} + assert ddt._assertion_defaults == {"type": "message"} + assert ddt._sleep_defaults == {"duration": 1.5} + + def test_init_with_parent(self): + """Test initialization with parent file.""" + parent_content = """ +defaults: + input: + type: message + locale: en-US + assertion: + type: message + sleep: + duration: 2.0 +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + f.write(parent_content) + parent_file = f.name + + try: + test_flow = { + "parent": parent_file, + "defaults": { + "input": {"text": "child text"}, + "assertion": {"text": "child assertion"} + }, + "test": [] + } + ddt = DataDrivenTest(test_flow) + + # Parent defaults should be merged with child defaults + assert ddt._input_defaults["type"] == "message" + assert ddt._input_defaults["locale"] == "en-US" + assert ddt._input_defaults["text"] == "child text" + assert ddt._assertion_defaults["type"] == "message" + assert ddt._assertion_defaults["text"] == "child assertion" + assert ddt._sleep_defaults["duration"] == 2.0 + finally: + os.unlink(parent_file) + + def test_init_with_test_steps(self): + """Test initialization with test steps.""" + test_flow = { + "test": [ + {"type": "input", "text": "Hello"}, + {"type": "assertion", "text": "Response"}, + {"type": "sleep", "duration": 1.0} + ] + } + ddt = DataDrivenTest(test_flow) + + assert len(ddt._test) == 3 + assert ddt._test[0]["type"] == "input" + assert ddt._test[1]["type"] == "assertion" + assert ddt._test[2]["type"] == "sleep" + + +class TestDataDrivenTestLoadInput: + """Tests for _load_input method.""" + + def test_load_input_minimal(self): + """Test loading input with minimal data.""" + test_flow = {"test": []} + ddt = DataDrivenTest(test_flow) + + input_data = {"type": "message", "text": "Hello"} + activity = ddt._load_input(input_data) + + assert isinstance(activity, Activity) + assert activity.type == "message" + assert activity.text == "Hello" + + def test_load_input_with_defaults(self): + """Test loading input merges with defaults.""" + test_flow = { + "defaults": { + "input": { + "type": "message", + "locale": "en-US", + "text": "default text" + } + }, + "test": [] + } + ddt = DataDrivenTest(test_flow) + + input_data = {"text": "Hello"} + activity = ddt._load_input(input_data) + + assert activity.type == "message" + assert activity.text == "Hello" + assert activity.locale == "en-US" + + def test_load_input_overrides_defaults(self): + """Test that input data overrides defaults.""" + test_flow = { + "defaults": { + "input": {"type": "message", "text": "default"} + }, + "test": [] + } + ddt = DataDrivenTest(test_flow) + + input_data = {"type": "event", "text": "override"} + activity = ddt._load_input(input_data) + + assert activity.type == "event" + assert activity.text == "override" + + +class TestDataDrivenTestLoadAssertion: + """Tests for _load_assertion method.""" + + def test_load_assertion_minimal(self): + """Test loading assertion with minimal data.""" + test_flow = {"test": []} + ddt = DataDrivenTest(test_flow) + + assertion_data = {"type": "message"} + result = ddt._load_assertion(assertion_data) + + assert result == {"type": "message"} + + def test_load_assertion_with_defaults(self): + """Test loading assertion merges with defaults.""" + test_flow = { + "defaults": { + "assertion": { + "type": "message", + "quantifier": "all" + } + }, + "test": [] + } + ddt = DataDrivenTest(test_flow) + + assertion_data = {"text": "Hello"} + result = ddt._load_assertion(assertion_data) + + assert result["type"] == "message" + assert result["quantifier"] == "all" + assert result["text"] == "Hello" + + def test_load_assertion_overrides_defaults(self): + """Test that assertion data overrides defaults.""" + test_flow = { + "defaults": { + "assertion": {"type": "message", "quantifier": "all"} + }, + "test": [] + } + ddt = DataDrivenTest(test_flow) + + assertion_data = {"type": "event", "quantifier": "one"} + result = ddt._load_assertion(assertion_data) + + assert result["type"] == "event" + assert result["quantifier"] == "one" + + +class TestDataDrivenTestSleep: + """Tests for _sleep method.""" + + @pytest.mark.asyncio + async def test_sleep_with_explicit_duration(self): + """Test sleep with explicit duration.""" + test_flow = {"test": []} + ddt = DataDrivenTest(test_flow) + + with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep: + await ddt._sleep({"duration": 2.5}) + mock_sleep.assert_called_once_with(2.5) + + @pytest.mark.asyncio + async def test_sleep_with_default_duration(self): + """Test sleep uses default duration when not specified.""" + test_flow = { + "defaults": { + "sleep": {"duration": 3.0} + }, + "test": [] + } + ddt = DataDrivenTest(test_flow) + + with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep: + await ddt._sleep({}) + mock_sleep.assert_called_once_with(3.0) + + @pytest.mark.asyncio + async def test_sleep_without_duration_defaults_to_zero(self): + """Test sleep defaults to 0 when no duration specified.""" + test_flow = {"test": []} + ddt = DataDrivenTest(test_flow) + + with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep: + await ddt._sleep({}) + mock_sleep.assert_called_once_with(0) + + +class TestDataDrivenTestRun: + """Tests for run method.""" + + @pytest.mark.asyncio + async def test_run_empty_test(self): + """Test running an empty test.""" + test_flow = {"test": []} + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[]) + + await ddt.run(agent_client, response_client) + + agent_client.send_activity.assert_not_called() + response_client.pop.assert_not_called() + + @pytest.mark.asyncio + async def test_run_single_input_step(self): + """Test running a test with single input step.""" + test_flow = { + "defaults": { + "input": {"type": "message"} + }, + "test": [ + {"type": "input", "text": "Hello"} + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: + mock_populate.side_effect = lambda act, defaults: act + await ddt.run(agent_client, response_client) + + agent_client.send_activity.assert_called_once() + call_args = agent_client.send_activity.call_args[0][0] + assert isinstance(call_args, Activity) + assert call_args.text == "Hello" + + @pytest.mark.asyncio + async def test_run_input_and_assertion(self): + """Test running a test with input and assertion steps.""" + test_flow = { + "test": [ + {"type": "input", "type": "message", "text": "Hello"}, + {"type": "assertion", "assertion": {"type": "message"}} + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_activity = Activity(type="message", text="Response") + response_client.pop = AsyncMock(return_value=[response_activity]) + + mock_assertion = MagicMock(return_value=(True, None)) + + with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: + mock_populate.side_effect = lambda act, defaults: act + with patch('microsoft_agents.testing.assertions.ActivityAssertion.from_config', return_value=mock_assertion): + await ddt.run(agent_client, response_client) + + agent_client.send_activity.assert_called_once() + response_client.pop.assert_called_once() + mock_assertion.assert_called_once_with([response_activity]) + + @pytest.mark.asyncio + async def test_run_assertion_accumulates_responses(self): + """Test that assertions accumulate responses from previous steps.""" + test_flow = { + "test": [ + {"type": "input", "type": "message", "text": "Hello"}, + {"type": "assertion", "assertion": {"type": "message"}} + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + # Response client returns activities on first call, empty on second + response_activity1 = Activity(type="message", text="Response 1") + response_activity2 = Activity(type="message", text="Response 2") + response_client.pop = AsyncMock(return_value=[response_activity1, response_activity2]) + + mock_assertion = MagicMock(return_value=(True, None)) + + with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: + mock_populate.side_effect = lambda act, defaults: act + with patch('microsoft_agents.testing.assertions.ActivityAssertion.from_config', return_value=mock_assertion): + await ddt.run(agent_client, response_client) + + # Check that assertion was called with accumulated responses + mock_assertion.assert_called_once() + call_args = mock_assertion.call_args[0][0] + assert len(call_args) == 2 + assert call_args[0].text == "Response 1" + assert call_args[1].text == "Response 2" + + @pytest.mark.asyncio + async def test_run_with_sleep_step(self): + """Test running a test with sleep step.""" + test_flow = { + "test": [ + {"type": "sleep", "duration": 1.5} + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep: + await ddt.run(agent_client, response_client) + mock_sleep.assert_called_once_with(1.5) + + @pytest.mark.asyncio + async def test_run_multiple_steps(self): + """Test running a test with multiple steps.""" + test_flow = { + "test": [ + {"type": "input", "type": "message", "text": "Hello"}, + {"type": "sleep", "duration": 0.5}, + {"type": "input", "type": "message", "text": "World"}, + {"type": "assertion", "assertion": {"type": "message"}} + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[Activity(type="message", text="Response")]) + + mock_assertion = MagicMock(return_value=(True, None)) + + with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: + mock_populate.side_effect = lambda act, defaults: act + with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep: + with patch('microsoft_agents.testing.assertions.ActivityAssertion.from_config', return_value=mock_assertion): + await ddt.run(agent_client, response_client) + + assert agent_client.send_activity.call_count == 2 + mock_sleep.assert_called_once_with(0.5) + response_client.pop.assert_called_once() + + @pytest.mark.asyncio + async def test_run_step_without_type_raises_error(self): + """Test that a step without type raises ValueError.""" + test_flow = { + "test": [ + {"text": "Hello"} # Missing 'type' field + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + with pytest.raises(ValueError, match="Each step must have a 'type' field"): + await ddt.run(agent_client, response_client) + + @pytest.mark.asyncio + async def test_run_uses_update_with_defaults_for_assertions(self): + """Test that run uses update_with_defaults for assertion configuration.""" + test_flow = { + "defaults": { + "assertion": {"quantifier": "all"} + }, + "test": [ + {"type": "input", "type": "message", "text": "Hello"}, + {"type": "assertion", "assertion": {"type": "message"}} + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[]) + + mock_assertion = MagicMock(return_value=(True, None)) + + with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: + mock_populate.side_effect = lambda act, defaults: act + with patch('microsoft_agents.testing.utils.update_with_defaults') as mock_update: + with patch('microsoft_agents.testing.assertions.ActivityAssertion.from_config', return_value=mock_assertion): + await ddt.run(agent_client, response_client) + + # Verify update_with_defaults was called + mock_update.assert_called_once() + call_args = mock_update.call_args[0] + assert "assertion" in call_args[0] or "type" in call_args[0] + assert call_args[1] == {"quantifier": "all"} + + @pytest.mark.asyncio + async def test_run_populates_activity_with_defaults(self): + """Test that run calls populate_activity with input defaults.""" + test_flow = { + "defaults": { + "input": {"locale": "en-US"} + }, + "test": [ + {"type": "input", "type": "message", "text": "Hello"} + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: + mock_populate.side_effect = lambda act, defaults: act + await ddt.run(agent_client, response_client) + + mock_populate.assert_called_once() + call_args = mock_populate.call_args[0] + assert isinstance(call_args[0], Activity) + assert call_args[1] == {"locale": "en-US"} + + +class TestDataDrivenTestIntegration: + """Integration tests for DataDrivenTest.""" + + @pytest.mark.asyncio + async def test_full_test_flow(self): + """Test a complete test flow with all step types.""" + test_flow = { + "description": "Complete integration test", + "defaults": { + "input": {"type": "message", "locale": "en-US"}, + "assertion": {"quantifier": "all"}, + "sleep": {"duration": 0.1} + }, + "test": [ + {"type": "input", "text": "Hello"}, + {"type": "sleep"}, + {"type": "input", "text": "World"}, + {"type": "assertion", "assertion": {"type": "message"}}, + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[ + Activity(type="message", text="Response 1"), + Activity(type="message", text="Response 2") + ]) + + mock_assertion = MagicMock(return_value=(True, None)) + + with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: + mock_populate.side_effect = lambda act, defaults: act + with patch('asyncio.sleep', new_callable=AsyncMock): + with patch('microsoft_agents.testing.assertions.ActivityAssertion.from_config', return_value=mock_assertion): + await ddt.run(agent_client, response_client) + + assert agent_client.send_activity.call_count == 2 + assert response_client.pop.call_count == 1 + assert mock_assertion.call_count == 1 \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_tester.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_tester.py deleted file mode 100644 index e69de29b..00000000 From 7a79cc9812045cfe0dbe07bc8f750cd53d032b32 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Fri, 14 Nov 2025 11:40:02 -0800 Subject: [PATCH 80/81] Data driven testing single test setup support --- .../testing/assertions/activity_assertion.py | 7 +- .../testing/assertions/selector.py | 2 +- .../data_driven/data_driven_test.py | 33 +- .../testing/integration/data_driven/ddt.py | 4 +- dev/microsoft-agents-testing/pytest.ini | 1 + .../assertions/test_activity_assertion.py | 42 +- .../data_driven/test_data_driven_test.py | 567 ++++++++---------- .../tests/utils/test_populate.py | 5 +- 8 files changed, 290 insertions(+), 371 deletions(-) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py index a2702fac..2fb161a5 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/activity_assertion.py @@ -79,13 +79,14 @@ def check(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: return passes, None - def __call__(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: + def __call__(self, activities: list[Activity]) -> None: """Allows the ActivityAssertion instance to be called directly. :param activities: The list of activities to be tested. :return: A tuple containing a boolean indicating if the assertion passed and an optional error message. """ - return self.check(activities) + passes, error = self.check(activities) + assert passes, error @staticmethod def from_config(config: dict) -> ActivityAssertion: @@ -94,7 +95,7 @@ def from_config(config: dict) -> ActivityAssertion: :param config: The configuration dictionary containing quantifier, selector, and assertion. :return: An ActivityAssertion instance. """ - assertion = config.get("assertion", {}) + assertion = config.get("activity", {}) selector = Selector.from_config(config.get("selector", {})) quantifier = AssertionQuantifier.from_config(config.get("quantifier", "all")) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py index c469c65b..f588216c 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/selector.py @@ -90,7 +90,7 @@ def from_config(config: dict) -> Selector: :param config: The configuration dictionary containing selector, quantifier, and index. :return: A Selector instance. """ - selector = config.get("selector", {}) + selector = config.get("activity", {}) index = config.get("index", None) return Selector( diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py index d7e283b3..631be08d 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py @@ -17,6 +17,7 @@ from ..core import AgentClient, ResponseClient + class DataDrivenTest: """Data driven test runner.""" @@ -48,14 +49,14 @@ def __init__(self, test_flow: dict) -> None: self._test = test_flow.get("test", []) def _load_input(self, input_data: dict) -> Activity: - data = deepcopy(self._input_defaults) - data.update(input_data) - return Activity.model_validate(data) + defaults = deepcopy(self._input_defaults) + update_with_defaults(input_data, defaults) + return Activity.model_validate(input_data.get("activity", {})) - def _load_assertion(self, assertion_data: dict) -> dict: - data = deepcopy(self._assertion_defaults) - data.update(assertion_data) - return data + def _load_assertion(self, assertion_data: dict) -> ActivityAssertion: + defaults = deepcopy(self._assertion_defaults) + update_with_defaults(assertion_data, defaults) + return ActivityAssertion.from_config(assertion_data) async def _sleep(self, sleep_data: dict) -> None: duration = sleep_data.get("duration") @@ -63,9 +64,11 @@ async def _sleep(self, sleep_data: dict) -> None: duration = self._sleep_defaults.get("duration", 0) await asyncio.sleep(duration) - async def run(self, agent_client: AgentClient, response_client: ResponseClient) -> None: + async def run( + self, agent_client: AgentClient, response_client: ResponseClient + ) -> None: """Run the data driven test. - + :param agent_client: The agent client to send activities to. """ @@ -77,18 +80,12 @@ async def run(self, agent_client: AgentClient, response_client: ResponseClient) if step_type == "input": input_activity = self._load_input(step) - populate_activity(input_activity, self._input_defaults) await agent_client.send_activity(input_activity) - elif step_type == "assertion": - - assertion = self._load_assertion(step) - update_with_defaults(assertion, self._assertion_defaults) - - activity_assertion = ActivityAssertion.from_config(assertion) + elif step_type == "assertion": + activity_assertion = self._load_assertion(step) responses.extend(await response_client.pop()) - - assert activity_assertion(responses) + activity_assertion(responses) elif step_type == "sleep": await self._sleep(step) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py index d47e3627..0a72166b 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py @@ -21,7 +21,7 @@ # setattr(test_cls, test_case_name, func) # def ddt(test_path: str) -> Callable[[Integration], Integration]: - + # def decorator(test_cls: Integration) -> Integration: # test_case_name = f"test_data_driven__{test_path.replace('/', '_').replace('.', '_')}" @@ -38,4 +38,4 @@ # # assertion # responses.extend(await response_client.pop()) -# return decorator \ No newline at end of file +# return decorator diff --git a/dev/microsoft-agents-testing/pytest.ini b/dev/microsoft-agents-testing/pytest.ini index 479894a8..fee2ab83 100644 --- a/dev/microsoft-agents-testing/pytest.ini +++ b/dev/microsoft-agents-testing/pytest.ini @@ -18,6 +18,7 @@ testpaths = tests python_files = test_*.py *_test.py python_classes = Test* python_functions = test_* +asyncio_mode=auto # Output configuration addopts = diff --git a/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py b/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py index d596157a..fc25761f 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py @@ -222,40 +222,6 @@ def test_check_any_basic_functionality(self, activities): assert passes is True assert error is None - -class TestActivityAssertionCallable: - """Tests for __call__ method.""" - - @pytest.fixture - def activities(self): - """Create a list of test activities.""" - return [ - Activity(type="message", text="Hello"), - Activity(type="event", name="test_event"), - ] - - def test_call_invokes_check(self, activities): - """Test that calling assertion instance invokes check().""" - assertion = ActivityAssertion( - assertion={"type": "message"}, quantifier=AssertionQuantifier.ALL - ) - result = assertion(activities) - assert isinstance(result, tuple) - assert len(result) == 2 - assert isinstance(result[0], bool) - - def test_call_returns_same_as_check(self, activities): - """Test that __call__ returns same result as check().""" - assertion = ActivityAssertion( - assertion={"type": "message"}, - selector=Selector(selector={"type": "message"}), - quantifier=AssertionQuantifier.ALL, - ) - call_result = assertion(activities) - check_result = assertion.check(activities) - assert call_result == check_result - - class TestActivityAssertionFromConfig: """Tests for from_config static method.""" @@ -268,9 +234,9 @@ def test_from_config_minimal(self): def test_from_config_with_assertion(self): """Test creating assertion from config with assertion field.""" - config = {"assertion": {"type": "message", "text": "Hello"}} + config = {"activity": {"type": "message", "text": "Hello"}} assertion = ActivityAssertion.from_config(config) - assert assertion._assertion == config["assertion"] + assert assertion._assertion == config["activity"] def test_from_config_with_selector(self): """Test creating assertion from config with selector field.""" @@ -287,7 +253,7 @@ def test_from_config_with_quantifier(self): def test_from_config_with_all_fields(self): """Test creating assertion from config with all fields.""" config = { - "assertion": {"type": "message"}, + "activity": {"type": "message"}, "selector": { "selector": {"text": "Hello"}, "quantifier": "ONE", @@ -309,7 +275,7 @@ def test_from_config_with_case_insensitive_quantifier(self): def test_from_config_with_complex_assertion(self): """Test creating assertion from config with complex nested assertion.""" config = { - "assertion": {"type": "message", "channelData": {"nested": {"value": 123}}}, + "activity": {"type": "message", "channelData": {"nested": {"value": 123}}}, "quantifier": "all", } assertion = ActivityAssertion.from_config(config) diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py index 82584406..e1d627c5 100644 --- a/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py +++ b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py @@ -4,248 +4,79 @@ import pytest import tempfile import os -from unittest.mock import AsyncMock, Mock, patch, MagicMock +from unittest.mock import AsyncMock from microsoft_agents.activity import Activity, ActivityTypes from microsoft_agents.testing.integration.data_driven import DataDrivenTest from microsoft_agents.testing.integration.core import AgentClient, ResponseClient +class TestDataDrivenTestSleep: + """Tests for _sleep method.""" -class TestDataDrivenTestInit: - """Tests for DataDrivenTest initialization.""" - - def test_init_minimal_config(self): - """Test initialization with minimal configuration.""" - test_flow = {"test": []} - ddt = DataDrivenTest(test_flow) - - assert ddt._description == "" - assert ddt._input_defaults == {} - assert ddt._assertion_defaults == {} - assert ddt._sleep_defaults == {} - assert ddt._test == [] - - def test_init_with_description(self): - """Test initialization with description.""" - test_flow = { - "description": "Test description", - "test": [] - } - ddt = DataDrivenTest(test_flow) - - assert ddt._description == "Test description" - - def test_init_with_defaults(self): - """Test initialization with defaults.""" - test_flow = { - "defaults": { - "input": {"type": "message", "text": "default text"}, - "assertion": {"type": "message"}, - "sleep": {"duration": 1.5} - }, - "test": [] - } - ddt = DataDrivenTest(test_flow) - - assert ddt._input_defaults == {"type": "message", "text": "default text"} - assert ddt._assertion_defaults == {"type": "message"} - assert ddt._sleep_defaults == {"duration": 1.5} - - def test_init_with_parent(self): - """Test initialization with parent file.""" - parent_content = """ -defaults: - input: - type: message - locale: en-US - assertion: - type: message - sleep: - duration: 2.0 -""" - - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: - f.write(parent_content) - parent_file = f.name - - try: - test_flow = { - "parent": parent_file, - "defaults": { - "input": {"text": "child text"}, - "assertion": {"text": "child assertion"} - }, - "test": [] - } - ddt = DataDrivenTest(test_flow) - - # Parent defaults should be merged with child defaults - assert ddt._input_defaults["type"] == "message" - assert ddt._input_defaults["locale"] == "en-US" - assert ddt._input_defaults["text"] == "child text" - assert ddt._assertion_defaults["type"] == "message" - assert ddt._assertion_defaults["text"] == "child assertion" - assert ddt._sleep_defaults["duration"] == 2.0 - finally: - os.unlink(parent_file) - - def test_init_with_test_steps(self): - """Test initialization with test steps.""" - test_flow = { - "test": [ - {"type": "input", "text": "Hello"}, - {"type": "assertion", "text": "Response"}, - {"type": "sleep", "duration": 1.0} - ] - } - ddt = DataDrivenTest(test_flow) - - assert len(ddt._test) == 3 - assert ddt._test[0]["type"] == "input" - assert ddt._test[1]["type"] == "assertion" - assert ddt._test[2]["type"] == "sleep" - - -class TestDataDrivenTestLoadInput: - """Tests for _load_input method.""" - - def test_load_input_minimal(self): - """Test loading input with minimal data.""" - test_flow = {"test": []} - ddt = DataDrivenTest(test_flow) - - input_data = {"type": "message", "text": "Hello"} - activity = ddt._load_input(input_data) - - assert isinstance(activity, Activity) - assert activity.type == "message" - assert activity.text == "Hello" - - def test_load_input_with_defaults(self): - """Test loading input merges with defaults.""" - test_flow = { - "defaults": { - "input": { - "type": "message", - "locale": "en-US", - "text": "default text" - } - }, - "test": [] - } - ddt = DataDrivenTest(test_flow) - - input_data = {"text": "Hello"} - activity = ddt._load_input(input_data) - - assert activity.type == "message" - assert activity.text == "Hello" - assert activity.locale == "en-US" - - def test_load_input_overrides_defaults(self): - """Test that input data overrides defaults.""" - test_flow = { - "defaults": { - "input": {"type": "message", "text": "default"} - }, - "test": [] - } - ddt = DataDrivenTest(test_flow) - - input_data = {"type": "event", "text": "override"} - activity = ddt._load_input(input_data) - - assert activity.type == "event" - assert activity.text == "override" - - -class TestDataDrivenTestLoadAssertion: - """Tests for _load_assertion method.""" - - def test_load_assertion_minimal(self): - """Test loading assertion with minimal data.""" + @pytest.mark.asyncio + async def test_sleep_with_explicit_duration(self): + """Test sleep with explicit duration.""" test_flow = {"test": []} ddt = DataDrivenTest(test_flow) - assertion_data = {"type": "message"} - result = ddt._load_assertion(assertion_data) - - assert result == {"type": "message"} - - def test_load_assertion_with_defaults(self): - """Test loading assertion merges with defaults.""" - test_flow = { - "defaults": { - "assertion": { - "type": "message", - "quantifier": "all" - } - }, - "test": [] - } - ddt = DataDrivenTest(test_flow) - - assertion_data = {"text": "Hello"} - result = ddt._load_assertion(assertion_data) + import time + start = time.time() + await ddt._sleep({"duration": 0.1}) + elapsed = time.time() - start - assert result["type"] == "message" - assert result["quantifier"] == "all" - assert result["text"] == "Hello" + assert elapsed >= 0.1 + assert elapsed < 0.2 # Allow some margin - def test_load_assertion_overrides_defaults(self): - """Test that assertion data overrides defaults.""" + @pytest.mark.asyncio + async def test_sleep_with_default_duration(self): + """Test sleep uses default duration when not specified.""" test_flow = { "defaults": { - "assertion": {"type": "message", "quantifier": "all"} + "sleep": {"duration": 0.1} }, "test": [] } ddt = DataDrivenTest(test_flow) - assertion_data = {"type": "event", "quantifier": "one"} - result = ddt._load_assertion(assertion_data) + import time + start = time.time() + await ddt._sleep({}) + elapsed = time.time() - start - assert result["type"] == "event" - assert result["quantifier"] == "one" - - -class TestDataDrivenTestSleep: - """Tests for _sleep method.""" + assert elapsed >= 0.1 + assert elapsed < 0.2 @pytest.mark.asyncio - async def test_sleep_with_explicit_duration(self): - """Test sleep with explicit duration.""" + async def test_sleep_without_duration_defaults_to_zero(self): + """Test sleep defaults to 0 when no duration specified.""" test_flow = {"test": []} ddt = DataDrivenTest(test_flow) - with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep: - await ddt._sleep({"duration": 2.5}) - mock_sleep.assert_called_once_with(2.5) + import time + start = time.time() + await ddt._sleep({}) + elapsed = time.time() - start + + assert elapsed < 0.1 # Should be nearly instant @pytest.mark.asyncio - async def test_sleep_with_default_duration(self): - """Test sleep uses default duration when not specified.""" + async def test_sleep_overrides_default_duration(self): + """Test that explicit duration overrides default.""" test_flow = { "defaults": { - "sleep": {"duration": 3.0} + "sleep": {"duration": 5.0} }, "test": [] } ddt = DataDrivenTest(test_flow) - with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep: - await ddt._sleep({}) - mock_sleep.assert_called_once_with(3.0) - - @pytest.mark.asyncio - async def test_sleep_without_duration_defaults_to_zero(self): - """Test sleep defaults to 0 when no duration specified.""" - test_flow = {"test": []} - ddt = DataDrivenTest(test_flow) + import time + start = time.time() + await ddt._sleep({"duration": 0.1}) + elapsed = time.time() - start - with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep: - await ddt._sleep({}) - mock_sleep.assert_called_once_with(0) + assert elapsed >= 0.1 + assert elapsed < 0.2 # Should use explicit duration, not default class TestDataDrivenTestRun: @@ -271,10 +102,10 @@ async def test_run_single_input_step(self): """Test running a test with single input step.""" test_flow = { "defaults": { - "input": {"type": "message"} + "input": {"activity": {"type": "message"} } }, "test": [ - {"type": "input", "text": "Hello"} + {"type": "input", "activity": {"text": "Hello"}} ] } ddt = DataDrivenTest(test_flow) @@ -282,22 +113,21 @@ async def test_run_single_input_step(self): agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: - mock_populate.side_effect = lambda act, defaults: act - await ddt.run(agent_client, response_client) + await ddt.run(agent_client, response_client) agent_client.send_activity.assert_called_once() call_args = agent_client.send_activity.call_args[0][0] assert isinstance(call_args, Activity) assert call_args.text == "Hello" + assert call_args.type == "message" @pytest.mark.asyncio - async def test_run_input_and_assertion(self): - """Test running a test with input and assertion steps.""" + async def test_run_input_and_assertion_passing(self): + """Test running a test with input and passing assertion.""" test_flow = { "test": [ - {"type": "input", "type": "message", "text": "Hello"}, - {"type": "assertion", "assertion": {"type": "message"}} + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + {"type": "assertion", "activity": {"type": "message"}} ] } ddt = DataDrivenTest(test_flow) @@ -307,24 +137,39 @@ async def test_run_input_and_assertion(self): response_activity = Activity(type="message", text="Response") response_client.pop = AsyncMock(return_value=[response_activity]) - mock_assertion = MagicMock(return_value=(True, None)) - - with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: - mock_populate.side_effect = lambda act, defaults: act - with patch('microsoft_agents.testing.assertions.ActivityAssertion.from_config', return_value=mock_assertion): - await ddt.run(agent_client, response_client) + # Should not raise any assertion error + await ddt.run(agent_client, response_client) agent_client.send_activity.assert_called_once() response_client.pop.assert_called_once() - mock_assertion.assert_called_once_with([response_activity]) + + @pytest.mark.asyncio + async def test_run_input_and_assertion_failing(self): + """Test running a test with input and failing assertion.""" + test_flow = { + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + {"type": "assertion", "activity": {"type": "event"}} # Will fail + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_activity = Activity(type="message", text="Response") + response_client.pop = AsyncMock(return_value=[response_activity]) + + # Should raise assertion error + with pytest.raises(AssertionError): + await ddt.run(agent_client, response_client) @pytest.mark.asyncio async def test_run_assertion_accumulates_responses(self): - """Test that assertions accumulate responses from previous steps.""" + """Test that assertions accumulate responses from response_client.""" test_flow = { "test": [ - {"type": "input", "type": "message", "text": "Hello"}, - {"type": "assertion", "assertion": {"type": "message"}} + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + {"type": "assertion", "activity": {"type": "message"}} ] } ddt = DataDrivenTest(test_flow) @@ -332,31 +177,51 @@ async def test_run_assertion_accumulates_responses(self): agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - # Response client returns activities on first call, empty on second + # Response client returns activities response_activity1 = Activity(type="message", text="Response 1") response_activity2 = Activity(type="message", text="Response 2") response_client.pop = AsyncMock(return_value=[response_activity1, response_activity2]) - mock_assertion = MagicMock(return_value=(True, None)) + # Should not raise - both are message type + await ddt.run(agent_client, response_client) + + response_client.pop.assert_called_once() + + @pytest.mark.asyncio + async def test_run_multiple_assertions_accumulate(self): + """Test that multiple assertions accumulate responses.""" + test_flow = { + "test": [ + {"type": "input", "activity": {"type": "message", "text": "First"}}, + {"type": "assertion", "activity": {"type": "message"}, "quantifier": "one"}, + {"type": "input", "activity": {"type": "message", "text": "Second"}}, + {"type": "assertion", "activity": {"type": "message"}, "quantifier": "all"} + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + # First pop returns one activity, second returns two + response_client.pop = AsyncMock( + side_effect=[ + [Activity(type="message", text="Response 1")], + [Activity(type="message", text="Response 2")] + ] + ) - with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: - mock_populate.side_effect = lambda act, defaults: act - with patch('microsoft_agents.testing.assertions.ActivityAssertion.from_config', return_value=mock_assertion): - await ddt.run(agent_client, response_client) + await ddt.run(agent_client, response_client) - # Check that assertion was called with accumulated responses - mock_assertion.assert_called_once() - call_args = mock_assertion.call_args[0][0] - assert len(call_args) == 2 - assert call_args[0].text == "Response 1" - assert call_args[1].text == "Response 2" + assert agent_client.send_activity.call_count == 2 + assert response_client.pop.call_count == 2 @pytest.mark.asyncio async def test_run_with_sleep_step(self): """Test running a test with sleep step.""" test_flow = { "test": [ - {"type": "sleep", "duration": 1.5} + {"type": "sleep", "duration": 0.1} ] } ddt = DataDrivenTest(test_flow) @@ -364,19 +229,23 @@ async def test_run_with_sleep_step(self): agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep: - await ddt.run(agent_client, response_client) - mock_sleep.assert_called_once_with(1.5) + import time + start = time.time() + await ddt.run(agent_client, response_client) + elapsed = time.time() - start + + assert elapsed >= 0.1 + assert elapsed < 0.2 @pytest.mark.asyncio async def test_run_multiple_steps(self): """Test running a test with multiple steps.""" test_flow = { "test": [ - {"type": "input", "type": "message", "text": "Hello"}, - {"type": "sleep", "duration": 0.5}, - {"type": "input", "type": "message", "text": "World"}, - {"type": "assertion", "assertion": {"type": "message"}} + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + {"type": "sleep", "duration": 0.05}, + {"type": "input", "activity": {"type": "message", "text": "World"}}, + {"type": "assertion", "activity": {"type": "message"}} ] } ddt = DataDrivenTest(test_flow) @@ -385,16 +254,13 @@ async def test_run_multiple_steps(self): response_client = AsyncMock(spec=ResponseClient) response_client.pop = AsyncMock(return_value=[Activity(type="message", text="Response")]) - mock_assertion = MagicMock(return_value=(True, None)) - - with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: - mock_populate.side_effect = lambda act, defaults: act - with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep: - with patch('microsoft_agents.testing.assertions.ActivityAssertion.from_config', return_value=mock_assertion): - await ddt.run(agent_client, response_client) + import time + start = time.time() + await ddt.run(agent_client, response_client) + elapsed = time.time() - start assert agent_client.send_activity.call_count == 2 - mock_sleep.assert_called_once_with(0.5) + assert elapsed >= 0.05 response_client.pop.assert_called_once() @pytest.mark.asyncio @@ -414,46 +280,97 @@ async def test_run_step_without_type_raises_error(self): await ddt.run(agent_client, response_client) @pytest.mark.asyncio - async def test_run_uses_update_with_defaults_for_assertions(self): - """Test that run uses update_with_defaults for assertion configuration.""" + async def test_run_with_assertion_quantifier_all(self): + """Test assertion with quantifier 'all'.""" test_flow = { - "defaults": { - "assertion": {"quantifier": "all"} - }, "test": [ - {"type": "input", "type": "message", "text": "Hello"}, - {"type": "assertion", "assertion": {"type": "message"}} + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + { + "type": "assertion", + "quantifier": "all", + "activity": {"type": "message"} + } ] } ddt = DataDrivenTest(test_flow) agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[]) + response_client.pop = AsyncMock(return_value=[ + Activity(type="message", text="Response 1"), + Activity(type="message", text="Response 2") + ]) + + # Should pass - all are message type + await ddt.run(agent_client, response_client) + + @pytest.mark.asyncio + async def test_run_with_assertion_quantifier_one(self): + """Test assertion with quantifier 'one'.""" + test_flow = { + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + { + "type": "assertion", + "quantifier": "one", + "activity": {"type": "event"} + } + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[ + Activity(type="message", text="Response 1"), + Activity(type="event", name="test_event"), + Activity(type="message", text="Response 2") + ]) - mock_assertion = MagicMock(return_value=(True, None)) + # Should pass - exactly one event type + await ddt.run(agent_client, response_client) + + @pytest.mark.asyncio + async def test_run_with_assertion_selector(self): + """Test assertion with selector.""" + test_flow = { + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + { + "type": "assertion", + "selector": {"activity": {"type": "message"}}, + "activity": {"text": "Response"} + } + ] + } + ddt = DataDrivenTest(test_flow) - with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: - mock_populate.side_effect = lambda act, defaults: act - with patch('microsoft_agents.testing.utils.update_with_defaults') as mock_update: - with patch('microsoft_agents.testing.assertions.ActivityAssertion.from_config', return_value=mock_assertion): - await ddt.run(agent_client, response_client) + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[ + Activity(type="event", name="test"), + Activity(type="message", text="Response"), + Activity(type="typing") + ]) - # Verify update_with_defaults was called - mock_update.assert_called_once() - call_args = mock_update.call_args[0] - assert "assertion" in call_args[0] or "type" in call_args[0] - assert call_args[1] == {"quantifier": "all"} + # Should pass - the message activity matches + await ddt.run(agent_client, response_client) @pytest.mark.asyncio - async def test_run_populates_activity_with_defaults(self): - """Test that run calls populate_activity with input defaults.""" + async def test_run_populate_activity_with_defaults(self): + """Test that input activities are populated with defaults.""" test_flow = { "defaults": { - "input": {"locale": "en-US"} + "input": { + "activity": { + "type": "message", + "locale": "en-US", + "channelId": "test-channel" + } + } }, "test": [ - {"type": "input", "type": "message", "text": "Hello"} + {"type": "input", "activity": {"text": "Hello"}} ] } ddt = DataDrivenTest(test_flow) @@ -461,53 +378,87 @@ async def test_run_populates_activity_with_defaults(self): agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: - mock_populate.side_effect = lambda act, defaults: act - await ddt.run(agent_client, response_client) + await ddt.run(agent_client, response_client) - mock_populate.assert_called_once() - call_args = mock_populate.call_args[0] - assert isinstance(call_args[0], Activity) - assert call_args[1] == {"locale": "en-US"} + call_args = agent_client.send_activity.call_args[0][0] + assert call_args.text == "Hello" + assert call_args.type == "message" + assert call_args.locale == "en-US" + assert call_args.channel_id == "test-channel" class TestDataDrivenTestIntegration: - """Integration tests for DataDrivenTest.""" + """Integration tests for DataDrivenTest with real scenarios.""" @pytest.mark.asyncio - async def test_full_test_flow(self): - """Test a complete test flow with all step types.""" + async def test_full_conversation_flow(self): + """Test a complete conversation flow.""" test_flow = { - "description": "Complete integration test", + "description": "Complete conversation test", "defaults": { - "input": {"type": "message", "locale": "en-US"}, + "input": {"activity": {"type": "message", "locale": "en-US"}}, "assertion": {"quantifier": "all"}, - "sleep": {"duration": 0.1} }, "test": [ - {"type": "input", "text": "Hello"}, - {"type": "sleep"}, - {"type": "input", "text": "World"}, - {"type": "assertion", "assertion": {"type": "message"}}, + {"type": "input", "activity": {"text": "Hello"}}, + {"type": "assertion", "activity": {"type": "message"}}, + {"type": "sleep", "duration": 0.05}, + {"type": "input", "activity": {"text": "How are you?"}}, + {"type": "assertion", "activity": {"type": "message"}}, ] } ddt = DataDrivenTest(test_flow) agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[ - Activity(type="message", text="Response 1"), - Activity(type="message", text="Response 2") - ]) - - mock_assertion = MagicMock(return_value=(True, None)) + response_client.pop = AsyncMock( + side_effect=[ + [Activity(type="message", text="Hi there!")], + [Activity(type="message", text="I'm doing well!")] + ] + ) - with patch('microsoft_agents.testing.utils.populate_activity') as mock_populate: - mock_populate.side_effect = lambda act, defaults: act - with patch('asyncio.sleep', new_callable=AsyncMock): - with patch('microsoft_agents.testing.assertions.ActivityAssertion.from_config', return_value=mock_assertion): - await ddt.run(agent_client, response_client) + await ddt.run(agent_client, response_client) assert agent_client.send_activity.call_count == 2 - assert response_client.pop.call_count == 1 - assert mock_assertion.call_count == 1 \ No newline at end of file + assert response_client.pop.call_count == 2 + + @pytest.mark.asyncio + @pytest.mark.skip(reason="TODO") + async def test_with_parent_file_integration(self): + """Test with parent file providing defaults.""" + parent_content = """defaults: + input: + type: message + locale: en-US + assertion: + quantifier: all +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + f.write(parent_content) + parent_file = f.name + + try: + test_flow = { + "parent": parent_file, + "test": [ + {"type": "input", "activity": {"text": "Hello"}}, + {"type": "assertion", "activity": {"type": "message"}} + ] + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[ + Activity(type="message", text="Response") + ]) + + await ddt.run(agent_client, response_client) + + call_args = agent_client.send_activity.call_args[0][0] + assert call_args.locale == "en-US" + assert call_args.type == "message" + finally: + os.unlink(parent_file) \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/utils/test_populate.py b/dev/microsoft-agents-testing/tests/utils/test_populate.py index ff464eef..07b99eab 100644 --- a/dev/microsoft-agents-testing/tests/utils/test_populate.py +++ b/dev/microsoft-agents-testing/tests/utils/test_populate.py @@ -4,7 +4,10 @@ import pytest from microsoft_agents.activity import Activity, ChannelAccount, ConversationAccount -from microsoft_agents.testing.utils.populate import update_with_defaults, populate_activity +from microsoft_agents.testing.utils.populate import ( + update_with_defaults, + populate_activity, +) class TestUpdateWithDefaults: From f8c9166629c7afe810a8fdd7ab135b5e6fdccdc7 Mon Sep 17 00:00:00 2001 From: Rodrigo Brandao Date: Fri, 14 Nov 2025 11:40:23 -0800 Subject: [PATCH 81/81] Formatting --- .../assertions/test_activity_assertion.py | 1 + .../data_driven/test_data_driven_test.py | 245 +++++++++--------- 2 files changed, 124 insertions(+), 122 deletions(-) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py b/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py index fc25761f..b459eab7 100644 --- a/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py +++ b/dev/microsoft-agents-testing/tests/assertions/test_activity_assertion.py @@ -222,6 +222,7 @@ def test_check_any_basic_functionality(self, activities): assert passes is True assert error is None + class TestActivityAssertionFromConfig: """Tests for from_config static method.""" diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py index e1d627c5..32186e1e 100644 --- a/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py +++ b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py @@ -10,6 +10,7 @@ from microsoft_agents.testing.integration.data_driven import DataDrivenTest from microsoft_agents.testing.integration.core import AgentClient, ResponseClient + class TestDataDrivenTestSleep: """Tests for _sleep method.""" @@ -18,31 +19,28 @@ async def test_sleep_with_explicit_duration(self): """Test sleep with explicit duration.""" test_flow = {"test": []} ddt = DataDrivenTest(test_flow) - + import time + start = time.time() await ddt._sleep({"duration": 0.1}) elapsed = time.time() - start - + assert elapsed >= 0.1 assert elapsed < 0.2 # Allow some margin @pytest.mark.asyncio async def test_sleep_with_default_duration(self): """Test sleep uses default duration when not specified.""" - test_flow = { - "defaults": { - "sleep": {"duration": 0.1} - }, - "test": [] - } + test_flow = {"defaults": {"sleep": {"duration": 0.1}}, "test": []} ddt = DataDrivenTest(test_flow) - + import time + start = time.time() await ddt._sleep({}) elapsed = time.time() - start - + assert elapsed >= 0.1 assert elapsed < 0.2 @@ -51,30 +49,27 @@ async def test_sleep_without_duration_defaults_to_zero(self): """Test sleep defaults to 0 when no duration specified.""" test_flow = {"test": []} ddt = DataDrivenTest(test_flow) - + import time + start = time.time() await ddt._sleep({}) elapsed = time.time() - start - + assert elapsed < 0.1 # Should be nearly instant @pytest.mark.asyncio async def test_sleep_overrides_default_duration(self): """Test that explicit duration overrides default.""" - test_flow = { - "defaults": { - "sleep": {"duration": 5.0} - }, - "test": [] - } + test_flow = {"defaults": {"sleep": {"duration": 5.0}}, "test": []} ddt = DataDrivenTest(test_flow) - + import time + start = time.time() await ddt._sleep({"duration": 0.1}) elapsed = time.time() - start - + assert elapsed >= 0.1 assert elapsed < 0.2 # Should use explicit duration, not default @@ -87,13 +82,13 @@ async def test_run_empty_test(self): """Test running an empty test.""" test_flow = {"test": []} ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) response_client.pop = AsyncMock(return_value=[]) - + await ddt.run(agent_client, response_client) - + agent_client.send_activity.assert_not_called() response_client.pop.assert_not_called() @@ -101,20 +96,16 @@ async def test_run_empty_test(self): async def test_run_single_input_step(self): """Test running a test with single input step.""" test_flow = { - "defaults": { - "input": {"activity": {"type": "message"} } - }, - "test": [ - {"type": "input", "activity": {"text": "Hello"}} - ] + "defaults": {"input": {"activity": {"type": "message"}}}, + "test": [{"type": "input", "activity": {"text": "Hello"}}], } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - + await ddt.run(agent_client, response_client) - + agent_client.send_activity.assert_called_once() call_args = agent_client.send_activity.call_args[0][0] assert isinstance(call_args, Activity) @@ -127,19 +118,19 @@ async def test_run_input_and_assertion_passing(self): test_flow = { "test": [ {"type": "input", "activity": {"type": "message", "text": "Hello"}}, - {"type": "assertion", "activity": {"type": "message"}} + {"type": "assertion", "activity": {"type": "message"}}, ] } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) response_activity = Activity(type="message", text="Response") response_client.pop = AsyncMock(return_value=[response_activity]) - + # Should not raise any assertion error await ddt.run(agent_client, response_client) - + agent_client.send_activity.assert_called_once() response_client.pop.assert_called_once() @@ -149,16 +140,16 @@ async def test_run_input_and_assertion_failing(self): test_flow = { "test": [ {"type": "input", "activity": {"type": "message", "text": "Hello"}}, - {"type": "assertion", "activity": {"type": "event"}} # Will fail + {"type": "assertion", "activity": {"type": "event"}}, # Will fail ] } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) response_activity = Activity(type="message", text="Response") response_client.pop = AsyncMock(return_value=[response_activity]) - + # Should raise assertion error with pytest.raises(AssertionError): await ddt.run(agent_client, response_client) @@ -169,22 +160,24 @@ async def test_run_assertion_accumulates_responses(self): test_flow = { "test": [ {"type": "input", "activity": {"type": "message", "text": "Hello"}}, - {"type": "assertion", "activity": {"type": "message"}} + {"type": "assertion", "activity": {"type": "message"}}, ] } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - + # Response client returns activities response_activity1 = Activity(type="message", text="Response 1") response_activity2 = Activity(type="message", text="Response 2") - response_client.pop = AsyncMock(return_value=[response_activity1, response_activity2]) - + response_client.pop = AsyncMock( + return_value=[response_activity1, response_activity2] + ) + # Should not raise - both are message type await ddt.run(agent_client, response_client) - + response_client.pop.assert_called_once() @pytest.mark.asyncio @@ -193,47 +186,52 @@ async def test_run_multiple_assertions_accumulate(self): test_flow = { "test": [ {"type": "input", "activity": {"type": "message", "text": "First"}}, - {"type": "assertion", "activity": {"type": "message"}, "quantifier": "one"}, + { + "type": "assertion", + "activity": {"type": "message"}, + "quantifier": "one", + }, {"type": "input", "activity": {"type": "message", "text": "Second"}}, - {"type": "assertion", "activity": {"type": "message"}, "quantifier": "all"} + { + "type": "assertion", + "activity": {"type": "message"}, + "quantifier": "all", + }, ] } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - + # First pop returns one activity, second returns two response_client.pop = AsyncMock( side_effect=[ [Activity(type="message", text="Response 1")], - [Activity(type="message", text="Response 2")] + [Activity(type="message", text="Response 2")], ] ) - + await ddt.run(agent_client, response_client) - + assert agent_client.send_activity.call_count == 2 assert response_client.pop.call_count == 2 @pytest.mark.asyncio async def test_run_with_sleep_step(self): """Test running a test with sleep step.""" - test_flow = { - "test": [ - {"type": "sleep", "duration": 0.1} - ] - } + test_flow = {"test": [{"type": "sleep", "duration": 0.1}]} ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - + import time + start = time.time() await ddt.run(agent_client, response_client) elapsed = time.time() - start - + assert elapsed >= 0.1 assert elapsed < 0.2 @@ -245,20 +243,23 @@ async def test_run_multiple_steps(self): {"type": "input", "activity": {"type": "message", "text": "Hello"}}, {"type": "sleep", "duration": 0.05}, {"type": "input", "activity": {"type": "message", "text": "World"}}, - {"type": "assertion", "activity": {"type": "message"}} + {"type": "assertion", "activity": {"type": "message"}}, ] } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[Activity(type="message", text="Response")]) - + response_client.pop = AsyncMock( + return_value=[Activity(type="message", text="Response")] + ) + import time + start = time.time() await ddt.run(agent_client, response_client) elapsed = time.time() - start - + assert agent_client.send_activity.call_count == 2 assert elapsed >= 0.05 response_client.pop.assert_called_once() @@ -266,16 +267,12 @@ async def test_run_multiple_steps(self): @pytest.mark.asyncio async def test_run_step_without_type_raises_error(self): """Test that a step without type raises ValueError.""" - test_flow = { - "test": [ - {"text": "Hello"} # Missing 'type' field - ] - } + test_flow = {"test": [{"text": "Hello"}]} # Missing 'type' field ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - + with pytest.raises(ValueError, match="Each step must have a 'type' field"): await ddt.run(agent_client, response_client) @@ -288,19 +285,21 @@ async def test_run_with_assertion_quantifier_all(self): { "type": "assertion", "quantifier": "all", - "activity": {"type": "message"} - } + "activity": {"type": "message"}, + }, ] } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[ - Activity(type="message", text="Response 1"), - Activity(type="message", text="Response 2") - ]) - + response_client.pop = AsyncMock( + return_value=[ + Activity(type="message", text="Response 1"), + Activity(type="message", text="Response 2"), + ] + ) + # Should pass - all are message type await ddt.run(agent_client, response_client) @@ -313,20 +312,22 @@ async def test_run_with_assertion_quantifier_one(self): { "type": "assertion", "quantifier": "one", - "activity": {"type": "event"} - } + "activity": {"type": "event"}, + }, ] } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[ - Activity(type="message", text="Response 1"), - Activity(type="event", name="test_event"), - Activity(type="message", text="Response 2") - ]) - + response_client.pop = AsyncMock( + return_value=[ + Activity(type="message", text="Response 1"), + Activity(type="event", name="test_event"), + Activity(type="message", text="Response 2"), + ] + ) + # Should pass - exactly one event type await ddt.run(agent_client, response_client) @@ -339,20 +340,22 @@ async def test_run_with_assertion_selector(self): { "type": "assertion", "selector": {"activity": {"type": "message"}}, - "activity": {"text": "Response"} - } + "activity": {"text": "Response"}, + }, ] } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[ - Activity(type="event", name="test"), - Activity(type="message", text="Response"), - Activity(type="typing") - ]) - + response_client.pop = AsyncMock( + return_value=[ + Activity(type="event", name="test"), + Activity(type="message", text="Response"), + Activity(type="typing"), + ] + ) + # Should pass - the message activity matches await ddt.run(agent_client, response_client) @@ -365,21 +368,19 @@ async def test_run_populate_activity_with_defaults(self): "activity": { "type": "message", "locale": "en-US", - "channelId": "test-channel" + "channelId": "test-channel", } } }, - "test": [ - {"type": "input", "activity": {"text": "Hello"}} - ] + "test": [{"type": "input", "activity": {"text": "Hello"}}], } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - + await ddt.run(agent_client, response_client) - + call_args = agent_client.send_activity.call_args[0][0] assert call_args.text == "Hello" assert call_args.type == "message" @@ -405,21 +406,21 @@ async def test_full_conversation_flow(self): {"type": "sleep", "duration": 0.05}, {"type": "input", "activity": {"text": "How are you?"}}, {"type": "assertion", "activity": {"type": "message"}}, - ] + ], } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) response_client.pop = AsyncMock( side_effect=[ [Activity(type="message", text="Hi there!")], - [Activity(type="message", text="I'm doing well!")] + [Activity(type="message", text="I'm doing well!")], ] ) - + await ddt.run(agent_client, response_client) - + assert agent_client.send_activity.call_count == 2 assert response_client.pop.call_count == 2 @@ -434,31 +435,31 @@ async def test_with_parent_file_integration(self): assertion: quantifier: all """ - - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write(parent_content) parent_file = f.name - + try: test_flow = { "parent": parent_file, "test": [ {"type": "input", "activity": {"text": "Hello"}}, - {"type": "assertion", "activity": {"type": "message"}} - ] + {"type": "assertion", "activity": {"type": "message"}}, + ], } ddt = DataDrivenTest(test_flow) - + agent_client = AsyncMock(spec=AgentClient) response_client = AsyncMock(spec=ResponseClient) - response_client.pop = AsyncMock(return_value=[ - Activity(type="message", text="Response") - ]) - + response_client.pop = AsyncMock( + return_value=[Activity(type="message", text="Response")] + ) + await ddt.run(agent_client, response_client) - + call_args = agent_client.send_activity.call_args[0][0] assert call_args.locale == "en-US" assert call_args.type == "message" finally: - os.unlink(parent_file) \ No newline at end of file + os.unlink(parent_file)