diff --git a/dev/README.md b/dev/README.md index 11a470b9..1b302d78 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,5 +1,7 @@ # Development Tools +DISCLAIMER: the content of this directory is experimental and not meant for production use. + Development utilities for the Microsoft Agents for Python project. ## Contents @@ -11,7 +13,7 @@ Development utilities for the Microsoft Agents for Python project. ## Quick Setup ```bash -./install.sh +pip install -e ./microsoft-agents-testing/ --config-settings editable_mode=compat ``` ## Benchmarking diff --git a/dev/install.sh b/dev/install.sh deleted file mode 100644 index 512d1d88..00000000 --- a/dev/install.sh +++ /dev/null @@ -1 +0,0 @@ -pip install -e ./microsoft-agents-testing/ --config-settings editable_mode=compat \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/manual_test/__init__.py b/dev/integration/__init__.py similarity index 100% rename from dev/microsoft-agents-testing/tests/manual_test/__init__.py rename to dev/integration/__init__.py diff --git a/dev/integration/agents/__init__.py b/dev/integration/agents/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/agents/basic_agent/__init__.py b/dev/integration/agents/basic_agent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/agents/basic_agent/python/README.md b/dev/integration/agents/basic_agent/python/README.md new file mode 100644 index 00000000..11278cde --- /dev/null +++ b/dev/integration/agents/basic_agent/python/README.md @@ -0,0 +1,82 @@ +# 🤖 Agents SDK Test Framework's Python Bot + +This Python bot is part of the Agents SDK Test Framework. It exercises agent behaviors, validates responses, and helps iterate on integrations with LLMs and tools. + +## Highlights ✨ +- ⚙ī¸ Test-runner for validating agent flows and tool/function calling +- 🧠 Integrates with LLM providers (Azure OpenAI, Semantic Kernel) +- đŸ–Ĩī¸ Uses Microsoft Agents SDK packages for hosting and activity management + +## 🚀 Getting Started + +### 🛠ī¸ Prerequisites +- Python 3.9+ +- `pip` (Python package manager) + +### đŸ“Ļ Installation +1. Install dependencies: + ```powershell + pip install --pre --no-deps -r pre_requirements.txt + pip install -r requirements.txt + ``` +#### ℹī¸ Why are there two installation steps? + +**Dependency installation is split into two steps to ensure reliability and avoid conflicts:** + +- **Step 1:** `pre_requirements.txt` — Installs core Microsoft Agents SDK packages. These may require pre-release flags or special handling, and installing them first (without dependency resolution) helps prevent version clashes. +- **Step 2:** `requirements.txt` — Installs the rest of the project dependencies, after the core packages are in place, to ensure compatibility and a smooth setup. + +This approach helps avoid dependency issues and guarantees all required packages are installed in the correct order. + +### ⚙ī¸ Set up Environment Variables +Copy or rename `.envLocal` to `.env` and fill in the required values (keys, endpoints, etc.). + +> 💡 Tip: The repo often uses Azure resources (Azure OpenAI / Bot Service) in examples. + +### â–ļī¸ Running the Agent +Start the agent locally: +```powershell +python app.py +``` + +## 📁 Project Layout +``` +Agent/python/ + agent.py + app.py + config.py + requirements.txt + requirements2.txt + pre_requirements.txt + .env + .envLocal + weather/ + agents/ + weather_forecast_agent.py + weather_forecast_agent_response.py + plugins/ + adaptive_card_plugin.py + date_time_plugin.py + weather_forecast_plugin.py + weather_forecast.py +``` + +This launches the process that hosts the agent and exposes the `/api/messages` endpoint. + +## 📚 Key Dependencies +- `microsoft-agents-hosting-core`, `microsoft-agents-hosting-aiohttp`, `microsoft-agents-activity`, `microsoft-agents-authentication-msal` — Microsoft Agents SDK packages +- `semantic-kernel` — LLM orchestration +- `openai` — Azure OpenAI integration + +## Health & Messaging Endpoints +- Health check: (if exposed) `GET /` should return 200 +- Messaging / activity endpoint: `POST /api/messages` (see `app.py`) + +## Agent Flow 🔁 +1. The test runner accepts scenario inputs (natural language user messages). +2. It forwards activity payloads to the agent runtime. +3. The agent may call functions/tools (e.g., weather, date/time). +4. The runner validates the agent's JSON / Adaptive Card outputs and records results. + +## Contributing +- Open a PR with changes and add a short description of the test scenarios you added or modified. diff --git a/dev/integration/agents/basic_agent/python/__init__.py b/dev/integration/agents/basic_agent/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/agents/basic_agent/python/env.TEMPLATE b/dev/integration/agents/basic_agent/python/env.TEMPLATE new file mode 100644 index 00000000..df8f217e --- /dev/null +++ b/dev/integration/agents/basic_agent/python/env.TEMPLATE @@ -0,0 +1,8 @@ +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID= +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET= +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID= + +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_API_VERSION= +AZURE_OPENAI_DEPLOYMENT_NAME= \ No newline at end of file diff --git a/dev/integration/agents/basic_agent/python/pre_requirements.txt b/dev/integration/agents/basic_agent/python/pre_requirements.txt new file mode 100644 index 00000000..13de107e --- /dev/null +++ b/dev/integration/agents/basic_agent/python/pre_requirements.txt @@ -0,0 +1,8 @@ +microsoft-agents-hosting-core +microsoft-agents-hosting-aiohttp +microsoft-agents-authentication-msal +microsoft-agents-activity +microsoft-agents-hosting-teams +microsoft-agents-copilotstudio-client +microsoft-agents-storage-blob +microsoft-agents-storage-cosmos \ No newline at end of file diff --git a/dev/integration/agents/basic_agent/python/requirements.txt b/dev/integration/agents/basic_agent/python/requirements.txt new file mode 100644 index 00000000..ddf4b785 --- /dev/null +++ b/dev/integration/agents/basic_agent/python/requirements.txt @@ -0,0 +1,9 @@ +openai +openai-agents +semantic-kernel +microsoft-agents-hosting-aiohttp +microsoft-agents-authentication-msal +microsoft-agents-hosting-teams +microsoft-agents-copilotstudio-client +microsoft-agents-storage-blob +microsoft-agents-storage-cosmos \ No newline at end of file diff --git a/dev/integration/agents/basic_agent/python/src/__init__.py b/dev/integration/agents/basic_agent/python/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/agents/basic_agent/python/src/agent.py b/dev/integration/agents/basic_agent/python/src/agent.py new file mode 100644 index 00000000..5e1c76d8 --- /dev/null +++ b/dev/integration/agents/basic_agent/python/src/agent.py @@ -0,0 +1,313 @@ +from __future__ import annotations +import json +import re + +from microsoft_agents.hosting.core import ( + AgentApplication, + TurnState, + TurnContext, + MessageFactory, +) +from microsoft_agents.activity import ( + ActivityTypes, + InvokeResponse, + Activity, + ConversationUpdateTypes, + Attachment, + EndOfConversationCodes, + DeliveryModes, +) + +from microsoft_agents.hosting.teams import TeamsActivityHandler + + +from semantic_kernel.contents import ChatHistory +from .weather.agents.weather_forecast_agent import WeatherForecastAgent + +from openai import AsyncAzureOpenAI +import asyncio + + +class Agent: + def __init__(self, client: AsyncAzureOpenAI): + self.client = client + self.multiple_message_pattern = re.compile(r"(\w+)\s+(\d+)") + self.weather_message_pattern = re.compile(r"^w: .*") + + def register_handlers(self, agent_app: AgentApplication[TurnState]): + """Register all handlers with the agent application""" + agent_app.conversation_update(ConversationUpdateTypes.MEMBERS_ADDED)( + self.on_members_added + ) + agent_app.message(self.weather_message_pattern)(self.on_weather_message) + agent_app.message(self.multiple_message_pattern)(self.on_multiple_message) + agent_app.message(re.compile(r"^poem$"))(self.on_poem_message) + agent_app.message(re.compile(r"^end$"))(self.on_end_message) + agent_app.message(re.compile(r"^stream$"))(self.on_stream_message) + agent_app.activity(ActivityTypes.message)(self.on_message) + agent_app.activity(ActivityTypes.invoke)(self.on_invoke) + agent_app.message_reaction("reactionsAdded")(self.on_reaction_added) + agent_app.message_reaction("reactionsRemoved")(self.on_reaction_removed) + agent_app.activity(ActivityTypes.message_update)(self.on_message_edit) + agent_app.activity(ActivityTypes.event)(self.on_event) + + async def on_members_added(self, context: TurnContext, _state: TurnState): + await context.send_activity(MessageFactory.text("Hello and Welcome!")) + + async def on_stream_message(self, context: TurnContext, state: TurnState): + if context.activity.delivery_mode == DeliveryModes.stream: + for x in range(1, 5): + await asyncio.sleep(1) + await context.send_activity("Stream response " + str(x)) + else: + await context.send_activity( + "Activity is not set to stream for delivery mode" + ) + + async def on_weather_message(self, context: TurnContext, state: TurnState): + + context.streaming_response.queue_informative_update( + "Working on a response for you" + ) + + chat_history = state.get_value( + "ConversationState.chatHistory", + ChatHistory, + target_cls=ChatHistory, + ) + + weather_agent = WeatherForecastAgent() + + forecast_response = await weather_agent.invoke_agent( + context.activity.text, chat_history + ) + if forecast_response is None: + context.streaming_response.queue_text_chunk( + "Sorry, I couldn't get the weather forecast at the moment." + ) + await context.streaming_response.end_stream() + return + + if forecast_response.contentType == "AdaptiveCard": + context.streaming_response.set_attachments( + [ + Attachment( + content_type="application/vnd.microsoft.card.adaptive", + content=forecast_response.content, + ) + ] + ) + else: + context.streaming_response.queue_text_chunk(forecast_response.content) + + await context.streaming_response.end_stream() + + async def on_multiple_message(self, context: TurnContext, state: TurnState): + counter = state.get_value( + "ConversationState.counter", + default_value_factory=(lambda: 0), + target_cls=int, + ) + + match = self.multiple_message_pattern.match(context.activity.text) + if not match: + return + word = match.group(1) + count = int(match.group(2)) + for _ in range(count): + await context.send_activity(f"[{counter}] You said: {word}") + counter += 1 + + state.set_value("ConversationState.counter", counter) + await state.save(context) + + async def on_poem_message(self, context: TurnContext, state: TurnState): + try: + context.streaming_response.queue_informative_update( + "Hold on for an awesome poem about Apollo..." + ) + + stream = await self.client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": """ + You are a creative assistant who has deeply studied Greek and Roman Gods, You also know all of the Percy Jackson Series + You write poems about the Greek Gods as they are depicted in the Percy Jackson books. + You format the poems in a way that is easy to read and understand + You break your poems into stanzas + You format your poems in Markdown using double lines to separate stanzas + """, + }, + { + "role": "user", + "content": "Write a poem about the Greek God Apollo as depicted in the Percy Jackson books", + }, + ], + stream=True, + max_tokens=1000, + ) + + async for update in stream: + if len(update.choices) > 0: + delta = update.choices[0].delta + if delta.content: + context.streaming_response.queue_text_chunk(delta.content) + finally: + await context.streaming_response.end_stream() + + async def on_end_message(self, context: TurnContext, state: TurnState): + await context.send_activity("Ending conversation...") + + endOfConversation = Activity.create_end_of_conversation_activity() + endOfConversation.code = EndOfConversationCodes.completed_successfully + await context.send_activity(endOfConversation) + + # Simulate a message handler for Action.Submit + # Waiting for Teams Extension to support Action.Submit + async def on_action_submit(self, context: TurnContext, state: TurnState): + user_text = context.activity.value.get("usertext", "") + if not user_text: + await context.send_activity("No user text provided in the action submit.") + return + await context.send_activity( + "doStuff action submitted " + json.dumps(context.activity.value) + ) + + async def on_action_execute(self, context: TurnContext, state: TurnState): + action = context.activity.value.get("action", {}) + data = action.get("data", {}) + user_text = data.get("usertext", "") + + if not user_text: + await context.send_activity("No user text provided in the action execute.") + return + + invoke_response = InvokeResponse( + status=200, + body={ + "statusCode": 200, + "type": "application/vnd.microsoft.card.adaptive", + "value": {"usertext": user_text}, + }, + ) + + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + + async def on_reaction_added(self, context: TurnContext, state: TurnState): + await context.send_activity( + "Message Reaction Added: " + context.activity.reactions_added[0].type + ) + + async def on_reaction_removed(self, context: TurnContext, state: TurnState): + await context.send_activity( + "Message Reaction Removed: " + context.activity.reactions_removed[0].type + ) + + async def on_message(self, context: TurnContext, state: TurnState): + + if context.activity.value and context.activity.value.get("verb") == "doStuff": + await self.on_action_submit(context, state) + return + + counter = state.get_value( + "ConversationState.counter", + default_value_factory=(lambda: 0), + target_cls=int, + ) + await context.send_activity(f"[{counter}] You said: {context.activity.text}") + counter += 1 + state.set_value("ConversationState.counter", counter) + + await state.save(context) + + async def on_invoke(self, context: TurnContext, state: TurnState): + + # Simulate Teams extensions until implemented + if context.activity.name == "adaptiveCard/action": + await self.on_action_execute(context, state) + elif context.activity.name == "composeExtension/query": + invoke_response = InvokeResponse( + status=200, + body={ + "composeExtension": { + "type": "result", + "attachmentLayout": "list", + "attachments": [ + {"contentType": "test", "contentUrl": "example.com"} + ], + } + }, + ) + + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + elif context.activity.name == "composeExtension/queryLink": + invoke_response = InvokeResponse( + status=200, + body={ + "channelId": "msteams", + "composeExtension": { + "type": "result", + "text": "On Query Link", + }, + }, + ) + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + elif context.activity.name == "composeExtension/selectItem": + value = context.activity.value + invoke_response = InvokeResponse( + status=200, + body={ + "channelId": "msteams", + "composeExtension": { + "type": "result", + "attachmentLayout": "list", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.thumbnail", + "content": { + "title": f"{value['id']}, {value['version']}" + }, + } + ], + }, + }, + ) + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + else: + invoke_response = InvokeResponse( + status=200, + body={"message": "Invoke received.", "data": context.activity.value}, + ) + + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + + async def on_message_edit(self, context: TurnContext, state: TurnState): + await context.send_activity(f"Message Edited: {context.activity.id}") + + async def on_event(self, context: TurnContext, state: TurnState): + if context.activity.name == "application/vnd.microsoft.meetingStart": + await context.send_activity( + f"Meeting started with ID: {context.activity.value['id']}" + ) + elif context.activity.name == "application/vnd.microsoft.meetingEnd": + await context.send_activity( + f"Meeting ended with ID: {context.activity.value['id']}" + ) + elif ( + context.activity.name == "application/vnd.microsoft.meetingParticipantJoin" + ): + await context.send_activity("Welcome to the meeting!") + else: + await context.send_activity("Received an event: " + context.activity.name) diff --git a/dev/integration/agents/basic_agent/python/src/app.py b/dev/integration/agents/basic_agent/python/src/app.py new file mode 100644 index 00000000..22e19416 --- /dev/null +++ b/dev/integration/agents/basic_agent/python/src/app.py @@ -0,0 +1,98 @@ +from __future__ import annotations +import logging +from aiohttp.web import Application, Request, Response, run_app +from dotenv import load_dotenv +from os import environ, path + +from semantic_kernel import Kernel +from semantic_kernel.utils.logging import setup_logging +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from openai import AsyncAzureOpenAI + +from microsoft_agents.hosting.aiohttp import ( + CloudAdapter, + jwt_authorization_middleware, + start_agent_process, +) +from microsoft_agents.hosting.core import ( + Authorization, + AgentApplication, + TurnState, + MemoryStorage, +) +from microsoft_agents.authentication.msal import MsalConnectionManager +from microsoft_agents.activity import ( + load_configuration_from_env, + ConversationUpdateTypes, + ActivityTypes, +) +import re + +from .agent import Agent + +# Load environment variables +load_dotenv() + +# Load configuration +agents_sdk_config = load_configuration_from_env(environ) + +# Initialize storage and connection manager +STORAGE = MemoryStorage() +CONNECTION_MANAGER = MsalConnectionManager(**agents_sdk_config) +ADAPTER = CloudAdapter(connection_manager=CONNECTION_MANAGER) +AUTHORIZATION = Authorization(STORAGE, CONNECTION_MANAGER, **agents_sdk_config) + +# Initialize Semantic Kernel +kernel = Kernel() + +chat_completion = AzureChatCompletion( + deployment_name=environ.get("AZURE_OPENAI_DEPLOYMENT_NAME", "gpt-4o"), + base_url=environ.get("AZURE_OPENAI_ENDPOINT"), + api_key=environ.get("AZURE_OPENAI_API_KEY"), + service_id="adaptive_card_service", +) + +kernel.add_service(chat_completion) + +# Initialize Azure OpenAI client +client = AsyncAzureOpenAI( + api_version=environ.get("AZURE_OPENAI_API_VERSION"), + azure_endpoint=environ.get("AZURE_OPENAI_ENDPOINT"), + api_key=environ.get("AZURE_OPENAI_API_KEY"), +) + +# Initialize Agent Application +AGENT_APP_INSTANCE = AgentApplication[TurnState]( + storage=STORAGE, adapter=ADAPTER, authorization=AUTHORIZATION, **agents_sdk_config +) + +logger = logging.getLogger(__name__) + +# Create and configure the AgentBot +AGENT = Agent(client) +AGENT.register_handlers(AGENT_APP_INSTANCE) + + +# Listen for incoming requests on /api/messages +async def messages(req: Request) -> Response: + agent: AgentApplication = req.app["agent_app"] + adapter: CloudAdapter = req.app["adapter"] + return await start_agent_process( + req, + agent, + adapter, + ) + + +# Create the application +APP = Application(middlewares=[jwt_authorization_middleware]) +APP.router.add_post("/api/messages", messages) +APP["agent_configuration"] = CONNECTION_MANAGER.get_default_connection_configuration() +APP["agent_app"] = AGENT_APP_INSTANCE +APP["adapter"] = ADAPTER + +if __name__ == "__main__": + try: + run_app(APP, host="localhost", port=3978) + except Exception as error: + raise error diff --git a/dev/integration/agents/basic_agent/python/src/config.py b/dev/integration/agents/basic_agent/python/src/config.py new file mode 100644 index 00000000..bd78d1cd --- /dev/null +++ b/dev/integration/agents/basic_agent/python/src/config.py @@ -0,0 +1,18 @@ +from os import environ +from microsoft_agents.hosting.core import AuthTypes, AgentAuthConfiguration + + +class DefaultConfig(AgentAuthConfiguration): + """Agent Configuration""" + + def __init__(self) -> None: + self.AUTH_TYPE = AuthTypes.client_secret + self.TENANT_ID = "" or environ.get("TENANT_ID") + self.CLIENT_ID = "" or environ.get("CLIENT_ID") + self.CLIENT_SECRET = "" or environ.get("CLIENT_SECRET") + self.AZURE_OPENAI_API_KEY = "" or environ.get("AZURE_OPENAI_API_KEY") + self.AZURE_OPENAI_ENDPOINT = "" or environ.get("AZURE_OPENAI_ENDPOINT") + self.AZURE_OPENAI_API_VERSION = "" or environ.get( + "AZURE_OPENAI_API_VERSION", "2024-06-01" + ) + self.PORT = 3978 diff --git a/dev/integration/agents/basic_agent/python/src/weather/__init__.py b/dev/integration/agents/basic_agent/python/src/weather/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/agents/basic_agent/python/src/weather/agents/__init__.py b/dev/integration/agents/basic_agent/python/src/weather/agents/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/agents/basic_agent/python/src/weather/agents/weather_forecast_agent.py b/dev/integration/agents/basic_agent/python/src/weather/agents/weather_forecast_agent.py new file mode 100644 index 00000000..b7a2d46c --- /dev/null +++ b/dev/integration/agents/basic_agent/python/src/weather/agents/weather_forecast_agent.py @@ -0,0 +1,110 @@ +import json +import os +from typing import Union, Literal, Any + +from pydantic import BaseModel + +from semantic_kernel import Kernel +from semantic_kernel.connectors.ai.open_ai import OpenAIPromptExecutionSettings +from semantic_kernel.connectors.ai.function_choice_behavior import ( + FunctionChoiceBehavior, +) +from semantic_kernel.functions import KernelArguments +from semantic_kernel.contents import ChatHistory +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from semantic_kernel.agents import ChatCompletionAgent, ChatHistoryAgentThread + +from ..plugins import DateTimePlugin, WeatherForecastPlugin, AdaptiveCardPlugin + + +class WeatherForecastAgentResponse(BaseModel): + contentType: str = Literal["Text", "AdaptiveCard"] + content: Union[dict, str] + + +class WeatherForecastAgent: + + agent_name = "WeatherForecastAgent" + + agent_instructions = """ + You are a friendly assistant that helps people find a weather forecast for a given time and place. + You may ask follow up questions until you have enough information to answer the customers question, + but once you have a forecast forecast, make sure to format it nicely using an adaptive card. + You should use adaptive JSON format to display the information in a visually appealing way + You should include a button for more details that points at https://www.msn.com/en-us/weather/forecast/in-{location} (replace {location} with the location the user asked about). + You should use adaptive cards version 1.5 or later. + + Respond only in JSON format with the following JSON schema: + + { + "contentType": "'Text' or 'AdaptiveCard' only", + "content": "{The content of the response, may be plain text, or JSON based adaptive card}" + } + """ + + def __init__(self, client: AzureChatCompletion | None = None): + + if not client: + client = AzureChatCompletion( + api_version=os.environ["AZURE_OPENAI_API_VERSION"], + endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + api_key=os.environ["AZURE_OPENAI_API_KEY"], + deployment_name=os.environ.get( + "AZURE_OPENAI_DEPLOYMENT_NAME", "gpt-4o" + ), + ) + + self.client = client + + execution_settings = OpenAIPromptExecutionSettings() + execution_settings.function_choice_behavior = FunctionChoiceBehavior.Auto() + execution_settings.temperature = 0 + execution_settings.top_p = 1 + self.execution_settings = execution_settings + + async def invoke_agent( + self, input: str, chat_history: ChatHistory + ) -> dict[str, Any]: + + thread = ChatHistoryAgentThread() + kernel = Kernel() + + chat_history.add_user_message(input) + + agent = ChatCompletionAgent( + service=self.client, + name=WeatherForecastAgent.agent_name, + instructions=WeatherForecastAgent.agent_instructions, + kernel=kernel, + arguments=KernelArguments( + settings=self.execution_settings, + ), + ) + + agent.kernel.add_plugin(plugin=DateTimePlugin(), plugin_name="datetime") + kernel.add_plugin(plugin=AdaptiveCardPlugin(), plugin_name="adaptiveCard") + kernel.add_plugin(plugin=WeatherForecastPlugin(), plugin_name="weatherForecast") + + resp: str = "" + + async for chat in agent.invoke(chat_history.to_prompt(), thread=thread): + chat_history.add_message(chat.content) + resp += chat.content.content + + # if resp has a json\n prefix, remove it + if "json\n" in resp: + resp = resp.replace("json\n", "") + resp = resp.replace("```", "") + + resp = resp.strip() + + try: + json_node: dict = json.loads(resp) + result = WeatherForecastAgentResponse.model_validate(json_node) + return result + except Exception as e: + return await self.invoke_agent( + "That response did not match the expected format. Please try again. Error: " + + str(e), + chat_history, + ) diff --git a/dev/integration/agents/basic_agent/python/src/weather/plugins/__init__.py b/dev/integration/agents/basic_agent/python/src/weather/plugins/__init__.py new file mode 100644 index 00000000..3638b566 --- /dev/null +++ b/dev/integration/agents/basic_agent/python/src/weather/plugins/__init__.py @@ -0,0 +1,9 @@ +from .date_time_plugin import DateTimePlugin +from .weather_forecast_plugin import WeatherForecastPlugin +from .adaptive_card_plugin import AdaptiveCardPlugin + +__all__ = [ + "DateTimePlugin", + "WeatherForecastPlugin", + "AdaptiveCardPlugin", +] diff --git a/dev/integration/agents/basic_agent/python/src/weather/plugins/adaptive_card_plugin.py b/dev/integration/agents/basic_agent/python/src/weather/plugins/adaptive_card_plugin.py new file mode 100644 index 00000000..33814600 --- /dev/null +++ b/dev/integration/agents/basic_agent/python/src/weather/plugins/adaptive_card_plugin.py @@ -0,0 +1,33 @@ +from semantic_kernel.functions import kernel_function +from semantic_kernel.connectors.ai.open_ai import OpenAIPromptExecutionSettings +from semantic_kernel.contents import ChatHistory +from os import environ, path +from semantic_kernel import Kernel + + +class AdaptiveCardPlugin: + + @kernel_function() + async def get_adaptive_card_for_data(self, data: str, kernel) -> str: + + instructions = """ + When given data about the weather forecast for a given time and place, generate an adaptive card + that displays the information in a visually appealing way. Only return the valid adaptive card + JSON string in the response. + """ + + # Set up chat + chat = ChatHistory(instructions=instructions) + chat.add_user_message(data) + + chat_completion = kernel.get_service("adaptive_card_service") + + # Get the response + result = await chat_completion.get_chat_message_contents( + chat, OpenAIPromptExecutionSettings() + ) + + # Extract the message text (if result is a list of ChatMessageContent) + message = result[0].content if result else "No response" + + return message diff --git a/dev/integration/agents/basic_agent/python/src/weather/plugins/date_time_plugin.py b/dev/integration/agents/basic_agent/python/src/weather/plugins/date_time_plugin.py new file mode 100644 index 00000000..bd115f79 --- /dev/null +++ b/dev/integration/agents/basic_agent/python/src/weather/plugins/date_time_plugin.py @@ -0,0 +1,31 @@ +from semantic_kernel.functions import kernel_function +from datetime import date +from datetime import datetime + + +class DateTimePlugin: + + @kernel_function( + name="today", + description="Get the current date", + ) + def today(self, formatProvider: str) -> str: + """ + Get the current date + """ + + _today = date.today() + formatted_date = _today.strftime(formatProvider) + return formatted_date + + @kernel_function( + name="now", + description="Get the current date and time in the local time zone", + ) + def now(self, formatProvider: str) -> str: + """ + Get the current date and time in the local time zone + """ + date_time = datetime.now() + formatted_date_time = date_time.strftime(formatProvider) + return formatted_date_time diff --git a/dev/integration/agents/basic_agent/python/src/weather/plugins/weather_forecast.py b/dev/integration/agents/basic_agent/python/src/weather/plugins/weather_forecast.py new file mode 100644 index 00000000..49f1c78a --- /dev/null +++ b/dev/integration/agents/basic_agent/python/src/weather/plugins/weather_forecast.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel + + +class WeatherForecast(BaseModel): + date: str + temperatureC: int + temperatureF: int diff --git a/dev/integration/agents/basic_agent/python/src/weather/plugins/weather_forecast_plugin.py b/dev/integration/agents/basic_agent/python/src/weather/plugins/weather_forecast_plugin.py new file mode 100644 index 00000000..757dd6f1 --- /dev/null +++ b/dev/integration/agents/basic_agent/python/src/weather/plugins/weather_forecast_plugin.py @@ -0,0 +1,26 @@ +from semantic_kernel.functions import kernel_function +from .weather_forecast import WeatherForecast +import random +from typing import Annotated + + +class WeatherForecastPlugin: + + @kernel_function( + name="get_forecast_for_date", + description="Get a weather forecast for a specific date and location", + ) + def get_forecast_for_date( + self, + date: Annotated[str, "The date for the forecast (e.g., '2025-08-01')"], + location: Annotated[str, "The location for the forecast (e.g., 'Seattle, WA'"], + ) -> Annotated[ + WeatherForecast, "Weather forecast object with temperature and date" + ]: + + temperatureC = int(random.uniform(15, 30)) + temperatureF = int((temperatureC * 9 / 5) + 32) + + return WeatherForecast( + date=date, temperatureC=temperatureC, temperatureF=temperatureF + ) diff --git a/dev/integration/pytest.ini b/dev/integration/pytest.ini new file mode 100644 index 00000000..9908f4bf --- /dev/null +++ b/dev/integration/pytest.ini @@ -0,0 +1,33 @@ +[pytest] +# Pytest configuration for Microsoft Agents for Python + +filterwarnings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning + ignore::aiohttp.web.NotAppKeyWarning + +# Test discovery configuration +testpaths = tests +python_files = test_*.py *_test.py +python_classes = Test* +python_functions = test_* +asyncio_mode=auto + +# Output configuration +addopts = + --strict-markers + --strict-config + --verbose + --tb=short + --durations=10 + +# Minimum version requirement +minversion = 6.0 + +# Markers for test categorization +markers = + unit: Unit tests + integration: Integration tests + slow: Slow tests that may take longer to run + requires_network: Tests that require network access + requires_auth: Tests that require authentication \ No newline at end of file diff --git a/dev/integration/samples/__init__.py b/dev/integration/samples/__init__.py new file mode 100644 index 00000000..4e712561 --- /dev/null +++ b/dev/integration/samples/__init__.py @@ -0,0 +1,7 @@ +from .basic_sample import BasicSample +from .quickstart_sample import QuickstartSample + +__all__ = [ + "BasicSample", + "QuickstartSample", +] diff --git a/dev/integration/samples/quickstart_sample.py b/dev/integration/samples/quickstart_sample.py new file mode 100644 index 00000000..7f32f283 --- /dev/null +++ b/dev/integration/samples/quickstart_sample.py @@ -0,0 +1,55 @@ +import re +import os +import sys +import traceback + +from dotenv import load_dotenv + +from microsoft_agents.activity import ConversationUpdateTypes +from microsoft_agents.hosting.core import ( + AgentApplication, + TurnContext, + TurnState, +) +from microsoft_agents.testing.integration.core import Sample + + +class QuickstartSample(Sample): + """A quickstart sample implementation.""" + + @classmethod + async def get_config(cls) -> dict: + """Retrieve the configuration for the sample.""" + load_dotenv("./src/tests/.env") + return dict(os.environ) + + async def init_app(self): + """Initialize the application for the quickstart sample.""" + + app: AgentApplication[TurnState] = self.env.agent_application + + @app.conversation_update(ConversationUpdateTypes.MEMBERS_ADDED) + async def on_members_added(context: TurnContext, state: TurnState) -> None: + await context.send_activity( + "Welcome to the empty agent! " + "This agent is designed to be a starting point for your own agent development." + ) + + @app.message(re.compile(r"^hello$")) + async def on_hello(context: TurnContext, state: TurnState) -> None: + await context.send_activity("Hello!") + + @app.activity("message") + async def on_message(context: TurnContext, state: TurnState) -> None: + await context.send_activity(f"you said: {context.activity.text}") + + @app.error + async def on_error(context: TurnContext, error: Exception): + # This check writes out errors to console log .vs. app insights. + # NOTE: In production environment, you should consider logging this to Azure + # application insights. + print(f"\n [on_turn_error] unhandled error: {error}", file=sys.stderr) + traceback.print_exc() + + # Send a message to the user + await context.send_activity("The bot encountered an error or bug.") diff --git a/dev/integration/tests/__init__.py b/dev/integration/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/tests/basic_agent/__init__.py b/dev/integration/tests/basic_agent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml new file mode 100644 index 00000000..09200090 --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml @@ -0,0 +1,36 @@ +test: +- type: input + activity: + type: conversationUpdate + id: activity-conv-update-001 + timestamp: '2025-07-30T23:01:11.000Z' + channelId: directline + from: + id: user1 + conversation: + id: conversation-001 + recipient: + id: basic-agent@sometext + name: basic-agent + membersAdded: + - id: basic-agent@sometext + name: basic-agent + - id: user1 + localTimestamp: '2025-07-30T15:59:55.000-07:00' + localTimezone: America/Los_Angeles + textFormat: plain + locale: en-US + attachments: [] + entities: + - type: ClientCapabilities + requiresBotState: true + supportsListening: true + supportsTts: true + channelData: + clientActivityID: client-activity-001 +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Hello and Welcome!"] diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_EndConversation_DeleteConversation.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_EndConversation_DeleteConversation.yaml new file mode 100644 index 00000000..fd8006cc --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendActivity_EndConversation_DeleteConversation.yaml @@ -0,0 +1,26 @@ +test: +- type: input + activity: + type: message + channelId: directline + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: end + locale: en-US +- type: assertion + selector: + index: -2 + activity: + type: message + text: ["CONTAINS", "Ending conversation..."] +- type: assertion + selector: + index: -1 + activity: + type: endOfConversation \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml new file mode 100644 index 00000000..e19537f5 --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml @@ -0,0 +1,31 @@ +test: +- type: input + activity: + reactionsRemoved: + - type: heart + type: messageReaction + timestamp: '2025-07-10T02:30:00.000Z' + id: '1752114287789' + channelId: directline + from: + id: from29ed + aadObjectId: aad-user1 + conversation: + conversationType: personal + tenantId: tenant6d4 + id: cpersonal-chat-id + recipient: + id: basic-agent@sometext + name: basic-agent + channelData: + tenant: + id: tenant6d4 + legacy: + replyToId: legacy_id + replyToId: '1752114287789' +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Message Reaction Removed: heart"] diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml new file mode 100644 index 00000000..1291a3ea --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml @@ -0,0 +1,31 @@ +test: +- type: input + activity: + reactionsAdded: + - type: heart + type: messageReaction + timestamp: '2025-07-10T02:25:04.000Z' + id: '1752114287789' + channelId: directline + from: + id: from29ed + aadObjectId: aad-user1 + conversation: + conversationType: personal + tenantId: tenant6d4 + id: cpersonal-chat-id + recipient: + id: basic-agent@sometext + name: basic-agent + channelData: + tenant: + id: tenant6d4 + legacy: + replyToId: legacy_id + replyToId: '1752114287789' +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Message Reaction Added: heart"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml new file mode 100644 index 00000000..d78e7bea --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml @@ -0,0 +1,34 @@ +test: +- type: input + activity: + type: message + id: activitiyA37 + timestamp: '2025-07-30T22:59:55.000Z' + localTimestamp: '2025-07-30T15:59:55.000-07:00' + localTimezone: America/Los_Angeles + channelId: directline + from: + id: fromid + name: '' + conversation: + id: coversation-id + recipient: + id: basic-agent@sometext + name: basic-agent + textFormat: plain + locale: en-US + text: hello world + attachments: [] + entities: + - type: ClientCapabilities + requiresBotState: true + supportsListening: true + supportsTts: true + channelData: + clientActivityID: client-act-id +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "You said: hello world"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendsHi5_Returns5HiActivities.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_SendsHi5_Returns5HiActivities.yaml new file mode 100644 index 00000000..0227d47a --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendActivity_SendsHi5_Returns5HiActivities.yaml @@ -0,0 +1,47 @@ +test: +- type: input + activity: + type: message + id: activity989 + channelId: directline + from: + id: user-id-0 + name: Alex Wilber + conversation: + id: personal-chat-id-hi5 + recipient: + id: bot-001 + name: Test Bot + text: hi 5 + locale: en-US +- type: assertion + quantifier: one + activity: + type: message + text: ["CONTAINS", "[0] You said: hi"] +- type: assertion + quantifier: one + activity: + type: message + text: ["CONTAINS", "[1] You said: hi"] +- type: assertion + quantifier: one + activity: + type: message + text: ["CONTAINS", "[2] You said: hi"] +- type: assertion + quantifier: one + activity: + type: message + text: ["CONTAINS", "[3] You said: hi"] +- type: assertion + quantifier: one + activity: + type: message + text: ["CONTAINS", "[4] You said: hi"] +- type: assertion # only 5 hi activities are returned + quantifier: none + selector: + index: 5 + activity: + type: message \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml new file mode 100644 index 00000000..633a4dd1 --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml @@ -0,0 +1,55 @@ +test: +- type: input + activity: + type: message + id: activityY1F + timestamp: '2025-07-30T23:06:37.000Z' + localTimestamp: '2025-07-30T16:06:37.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: https://webchat.botframework.com/ + channelId: directline + from: + id: fromid + name: '' + conversation: + id: conv-id + recipient: + id: basic-agent@sometext + name: basic-agent + locale: en-US + attachments: [] + channelData: + postBack: true + clientActivityID: client-act-id + value: + verb: doStuff + id: doStuff + type: Action.Submit + test: test + data: + name: test + usertext: hello +- type: assertion + selector: + index: -1 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "doStuff"] +- type: assertion + selector: + index: -1 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "Action.Submit"] +- type: assertion + selector: + index: -1 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "hello"] diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml new file mode 100644 index 00000000..e7d593c5 --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml @@ -0,0 +1,24 @@ +test: +- type: input + activity: + type: message + channelId: directline + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: 'w: What''s the weather in Seattle today?' + locale: en-US +- type: skip +- type: assertion + selector: + index: -1 + activity: + type: message + attachments: + - contentType: application/vnd.microsoft.card.adaptive + content: ["RE_MATCH", "(īŋŊ|\\u00B0|Missing temperature inside adaptive card:)"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SendsText_ReturnsPoem.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_SendsText_ReturnsPoem.yaml new file mode 100644 index 00000000..12999ce3 --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendActivity_SendsText_ReturnsPoem.yaml @@ -0,0 +1,28 @@ +test: +- type: input + activity: + type: message + channelId: directline + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: poem + locale: en-US +- type: skip +# - type: assertion +# selector: +# activity: +# type: typing +# activity: +# text: ["CONTAINS", "Hold on for an awesome poem about Apollo"] +# - type: assertion +# selector: +# index: -1 +# activity: +# text: ["CONTAINS", "Apollo"] +# - type: breakpoint \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml b/dev/integration/tests/basic_agent/directline/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml new file mode 100644 index 00000000..1b62d16d --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml @@ -0,0 +1,31 @@ +test: +- type: input + activity: + type: message + channelId: directline + from: + id: user1 + name: User + conversation: + id: conversation-simulate-002 + recipient: + id: bot1 + name: Bot + text: 'w: what''s the weather?' + locale: en-US +- type: input + activity: + type: message + channelId: directline + from: + id: user1 + name: User + conversation: + id: conversation-simulate-002 + recipient: + id: bot1 + name: Bot + text: 'w: Seattle for today' + locale: en-US +- type: skip +# - type: breakpoint \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml new file mode 100644 index 00000000..2477faea --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml @@ -0,0 +1,25 @@ +test: +- type: input + activity: + type: message + channelId: directline + deliveryMode: expectedReplies + from: + id: user1 + name: User + conversation: + id: conv1 + recipient: + id: bot1 + name: Bot + text: 'w: What''s the weather in Seattle today?''' + locale: en-US +- type: skip +- type: assertion + selector: + index: -1 + activity: + type: message + attachments: + - contentType: application/vnd.microsoft.card.adaptive + content: ["RE_MATCH", "(īŋŊ|\\u00B0|Missing temperature inside adaptive card:)"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml b/dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml new file mode 100644 index 00000000..8f34d64a --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml @@ -0,0 +1,29 @@ +test: +- type: input + activity: + type: message + channelId: directline + deliveryMode: expectedReplies + from: + id: user1 + name: User + conversation: + id: conv1 + recipient: + id: bot1 + name: Bot + text: poem + locale: en-US +- type: skip +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Apollo" ] +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "\n" ] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendInvoke_QueryLink_ReturnsText.yaml b/dev/integration/tests/basic_agent/directline/SendInvoke_QueryLink_ReturnsText.yaml new file mode 100644 index 00000000..6cf460b3 --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendInvoke_QueryLink_ReturnsText.yaml @@ -0,0 +1,21 @@ +test: +- type: input + activity: + type: invoke + channelId: directline + from: + id: user1 + name: User + conversation: + id: conversation123 + recipient: + id: bot1 + name: Bot + name: composeExtension/queryLink + value: + url: https://github.com/microsoft/Agents-for-net/blob/users/tracyboehrer/cards-sample/src/samples/Teams/TeamsAgent/TeamsAgent.cs + locale: en-US + assertion: + invokeResponse: + composeExtension: + text: ["CONTAINS", "On Query Link"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml b/dev/integration/tests/basic_agent/directline/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml new file mode 100644 index 00000000..4320d517 --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml @@ -0,0 +1,28 @@ +test: +- type: input + activity: + type: invoke + channelId: directline + from: + id: user1 + name: User + conversation: + id: conversation123 + recipient: + id: bot1 + name: Bot + name: composeExtension/query + value: + commandId: findNuGetPackage + parameters: + - name: NuGetPackageName + value: Newtonsoft.Json + queryOptions: + skip: 0 + count: 10 + locale: en-US + assertion: + invokeResponse: + composeExtension: + text: ["CONTAINS", "result"] + attachments: ["LEN_GREATER_THAN", 0] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendInvoke_SelectItem_ReceiveItem.yaml b/dev/integration/tests/basic_agent/directline/SendInvoke_SelectItem_ReceiveItem.yaml new file mode 100644 index 00000000..4a843c50 --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendInvoke_SelectItem_ReceiveItem.yaml @@ -0,0 +1,31 @@ +test: +- type: input + activity: + type: invoke + id: invoke123 + channelId: directline + from: + id: user-id-0 + name: Alex Wilber + conversation: + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + value: + '@id': https://www.nuget.org/packages/Newtonsoft.Json/13.0.1 + id: Newtonsoft.Json + version: 13.0.1 + description: Json.NET is a popular high-performance JSON framework for .NET + projectUrl: https://www.newtonsoft.com/json + iconUrl: https://www.newtonsoft.com/favicon.ico + name: composeExtension/selectItem + locale: en-US +- type: assertion + invokeResponse: + composeExtension: + type: result + text: ["CONTAINS", "Newtonsoft.Json"] + attachments: + contentType: application/vnd.microsoft.card.thumbnail +- type: skip \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml b/dev/integration/tests/basic_agent/directline/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml new file mode 100644 index 00000000..85f13369 --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml @@ -0,0 +1,38 @@ +test: +- type: input + activity: + type: invoke + id: invoke456 + channelId: directline + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-22T19:21:03.000Z' + localTimestamp: '2025-07-22T12:21:03.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:63676/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + value: + parameters: + - value: hi` +- type: assertion + invokeResponse: + message: ["EQUALS", "Invoke received."] + status: 200 + data: + parameters: + - value: ["CONTAINS", "hi"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml b/dev/integration/tests/basic_agent/directline/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml new file mode 100644 index 00000000..fd9b7dbb --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml @@ -0,0 +1,24 @@ +test: +- type: input + activity: + type: invoke + channelId: directline + from: + id: user1 + name: User + conversation: + id: conversation123 + recipient: + id: bot1 + name: Bot + name: adaptiveCard/action + value: + action: + type: Action.Execute + title: Execute doStuff + verb: doStuff + data: + usertext: hi + trigger: manual + locale: en-US +- type: skip \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/directline/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml b/dev/integration/tests/basic_agent/directline/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml new file mode 100644 index 00000000..79d8318f --- /dev/null +++ b/dev/integration/tests/basic_agent/directline/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml @@ -0,0 +1,29 @@ +test: +- type: input + activity: + type: message + id: activity-stream-001 + timestamp: '2025-06-18T18:47:46.000Z' + localTimestamp: '2025-06-18T11:47:46.000-07:00' + localTimezone: America/Los_Angeles + channelId: directline + from: + id: user1 + name: '' + conversation: + id: conversation-stream-001 + recipient: + id: basic-agent@sometext + name: basic-agent + textFormat: plain + locale: en-US + text: stream + attachments: [] + entities: + - type: ClientCapabilities + requiresBotState: true + supportsListening: true + supportsTts: true + channelData: + clientActivityID: client-activity-stream-001 +- type: skip \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml new file mode 100644 index 00000000..f46939fc --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml @@ -0,0 +1,39 @@ +test: +- type: input + activity: + type: conversationUpdate + id: activity123 + timestamp: '2025-06-23T19:48:15.625+00:00' + serviceUrl: http://localhost:62491/_connector + channelId: msteams + from: + id: user-id-0 + aadObjectId: aad-user-alex + role: user + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + membersAdded: + - id: user-id-0 + aadObjectId: aad-user-alex + - id: bot-001 + membersRemoved: [] + reactionsAdded: [] + reactionsRemoved: [] + attachments: [] + entities: [] + channelData: + tenant: + id: tenant-001 + listenFor: [] + textHighlights: [] +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Hello and Welcome!"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_EditMessage_ReceiveUpdate.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_EditMessage_ReceiveUpdate.yaml new file mode 100644 index 00000000..adc55806 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_EditMessage_ReceiveUpdate.yaml @@ -0,0 +1,78 @@ +test: +- type: input + activity: + type: message + id: activity989 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-07T21:24:15.930Z' + localTimestamp: '2025-07-07T14:24:15.930-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:60209/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + text: Hello + channelData: + tenant: + id: tenant-001 +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Hello"] +- type: input + activity: + type: messageUpdate + id: activity989 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-07T21:24:15.930Z' + localTimestamp: '2025-07-07T14:24:15.930-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:60209/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + text: This is the updated message content. + channelData: + eventType: editMessage + tenant: + id: tenant-001 +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Message Edited: activity989"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_EndConversation_DeleteConversation.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_EndConversation_DeleteConversation.yaml new file mode 100644 index 00000000..1fbb5d52 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_EndConversation_DeleteConversation.yaml @@ -0,0 +1,40 @@ +test: +- type: input + activity: + type: message + channelId: msteams + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: end + id: activity989 + timestamp: '2025-07-07T21:24:15.000Z' + localTimestamp: '2025-07-07T14:24:15.000-07:00' + localTimezone: America/Los_Angeles + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + channelData: + tenant: + id: tenant-001 +- type: assertion + selector: + index: -2 + activity: + type: message + text: "Ending conversation..." +- type: assertion + selector: + index: -1 + activity: + type: endOfConversation \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_EndTeamsMeeting_ExpectMessage.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_EndTeamsMeeting_ExpectMessage.yaml new file mode 100644 index 00000000..6b8250ef --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_EndTeamsMeeting_ExpectMessage.yaml @@ -0,0 +1,55 @@ +test: +- type: input + activity: + type: event + name: application/vnd.microsoft.meetingEnd + from: + id: user-001 + name: Jordan Lee + recipient: + id: bot-001 + name: TeamHelperBot + conversation: + id: conversation-abc123 + channelId: msteams + serviceUrl: https://smba.trafficmanager.net/amer/ + value: + trigger: onMeetingStart + id: meeting-12345 + title: Quarterly Planning Meeting + endTime: '2025-07-28T21:00:00Z' + joinUrl: https://teams.microsoft.com/l/meetup-join/... + meetingType: scheduled + meeting: + organizer: + id: user-002 + name: Morgan Rivera + participants: + - id: user-001 + name: Jordan Lee + - id: user-003 + name: Taylor Kim + - id: user-004 + name: Riley Chen + location: Microsoft Teams Meeting + id: activity989 + timestamp: '2025-07-07T21:24:15.000Z' + localTimestamp: '2025-07-07T14:24:15.000-07:00' + localTimezone: America/Los_Angeles + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + channelData: + tenant: + id: tenant-001 +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Meeting ended with ID: meeting-12345"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_ParticipantJoinsTeamMeeting_ExpectMessage.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_ParticipantJoinsTeamMeeting_ExpectMessage.yaml new file mode 100644 index 00000000..c58badbc --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_ParticipantJoinsTeamMeeting_ExpectMessage.yaml @@ -0,0 +1,55 @@ +test: +- type: input + activity: + type: event + name: application/vnd.microsoft.meetingParticipantJoin + from: + id: user-001 + name: Jordan Lee + recipient: + id: bot-001 + name: TeamHelperBot + conversation: + id: conversation-abc123 + channelId: msteams + serviceUrl: https://smba.trafficmanager.net/amer/ + value: + trigger: onMeetingStart + id: meeting-12345 + title: Quarterly Planning Meeting + endTime: '2025-07-28T21:00:00Z' + joinUrl: https://teams.microsoft.com/l/meetup-join/... + meetingType: scheduled + meeting: + organizer: + id: user-002 + name: Morgan Rivera + participants: + - id: user-001 + name: Jordan Lee + - id: user-003 + name: Taylor Kim + - id: user-004 + name: Riley Chen + location: Microsoft Teams Meeting + id: activity989 + timestamp: '2025-07-07T21:24:15.000Z' + localTimestamp: '2025-07-07T14:24:15.000-07:00' + localTimezone: America/Los_Angeles + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + channelData: + tenant: + id: tenant-001 +- type: assertion + selector: + index: -1 + activity: + type: message + text: "Welcome to the meeting!" \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml new file mode 100644 index 00000000..83a3b658 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml @@ -0,0 +1,30 @@ +test: +- type: input + activity: + reactionsRemoved: + - type: heart + type: messageReaction + timestamp: '2025-07-10T02:30:00.000Z' + id: activity175 + channelId: msteams + from: + id: from29ed + aadObjectId: d6dab + conversation: + conversationType: personal + tenantId: tenant6d4 + id: cpersonal-chat-id + recipient: + id: basic-agent@sometext + name: basic-agent + channelData: + tenant: + id: tenant6d4 + legacy: + replyToId: legacy_id + replyToId: activity175 +- type: assertion + selector: -1 + activity: + type: message + text: "Message Reaction Removed: heart" \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml new file mode 100644 index 00000000..86330b8a --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml @@ -0,0 +1,30 @@ +test: +- type: input + activity: + reactionsAdded: + - type: heart + type: messageReaction + timestamp: '2025-07-10T02:25:04.000Z' + id: activity175 + channelId: msteams + from: + id: from29ed + aadObjectId: aad-user1 + conversation: + conversationType: personal + tenantId: tenant6d4 + id: cpersonal-chat-id + recipient: + id: basic-agent@sometext + name: basic-agent + channelData: + tenant: + id: tenant6d4 + legacy: + replyToId: legacy_id + replyToId: activity175 +- type: assertion + selector: -1 + activity: + type: message + text: "Message Reaction Added: heart" \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml new file mode 100644 index 00000000..a915d0b4 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml @@ -0,0 +1,33 @@ +test: +- type: input + activity: + type: message + id: activity-hello-msteams-001 + timestamp: '2025-06-18T18:47:46.000Z' + localTimestamp: '2025-06-18T11:47:46.000-07:00' + localTimezone: America/Los_Angeles + channelId: msteams + from: + id: user1 + name: '' + conversation: + id: conversation-hello-msteams-001 + recipient: + id: basic-agent@sometext + name: basic-agent + textFormat: plain + locale: en-US + text: hello world + attachments: [] + entities: + - type: ClientCapabilities + requiresBotState: true + supportsListening: true + supportsTts: true + channelData: + clientActivityID: client-activity-hello-msteams-001 +- type: assertion + selector: -1 + activity: + type: message + text: ["CONTAINS", "You said: hello world"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsHi5_Returns5HiActivities.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_SendsHi5_Returns5HiActivities.yaml new file mode 100644 index 00000000..8b3dd428 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_SendsHi5_Returns5HiActivities.yaml @@ -0,0 +1,80 @@ +test: +- type: input + activity: + type: message + id: activity989 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-07T21:24:15.000Z' + localTimestamp: '2025-07-07T14:24:15.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:60209/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + text: hi 5 + channelData: + tenant: + id: tenant-001 +- type: skip +- type: assertion + selector: + index: 0 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "[0] You said: hi"] +- type: assertion + selector: + index: 1 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "[1] You said: hi"] +- type: assertion + selector: + index: 2 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "[2] You said: hi"] +- type: assertion + selector: + index: 3 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "[3] You said: hi"] +- type: assertion + selector: + index: 4 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "[4] You said: hi"] +- type: assertion # only 5 hi activities are returned + quantifier: none + selector: + index: 5 + activity: + type: message \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml new file mode 100644 index 00000000..dd9c74ae --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml @@ -0,0 +1,59 @@ +test: +- type: input + activity: + type: message + id: activity123 + channelId: msteams + from: + id: from29ed + name: Basic User + aadObjectId: aad-user1 + timestamp: '2025-06-27T17:24:16.000Z' + localTimestamp: '2025-06-27T17:24:16.000Z' + localTimezone: America/Los_Angeles + serviceUrl: https://smba.trafficmanager.net/amer/ + conversation: + conversationType: personal + tenantId: tenant6d4 + id: cpersonal-chat-id + recipient: + id: basic-agent@sometext + name: basic-agent + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + channelData: + tenant: + id: tenant6d4 + source: + name: message + legacy: + replyToId: legacy_id + replyToId: activity123 + value: + verb: doStuff + id: doStuff + type: Action.Submit + test: test + data: + name: test + usertext: hello +- type: assertion + selector: -1 + activity: + type: message + text: ["CONTAINS", "doStuff"] +- type: assertion + selector: -1 + activity: + type: message + text: ["CONTAINS", "Action.Submit"] +- type: assertion + selector: -1 + activity: + type: message + text: ["CONTAINS", "hello"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml new file mode 100644 index 00000000..4051612a --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml @@ -0,0 +1,41 @@ +test: +- type: input + activity: + type: message + id: activity989 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-07T21:24:15.000Z' + localTimestamp: '2025-07-07T14:24:15.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:60209/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + text: 'w: What''s the weather in Seattle today?' + channelData: + tenant: + id: tenant-001 +- type: skip +# - type: assertion +# selector: -1 +# activity: +# type: message +# attachments: +# - contentType: application/vnd.microsoft.card.adaptive +# content: ["RE_MATCH", "(īŋŊ|\\u00B0|Missing temperature inside adaptive card:)"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SendsText_ReturnsPoem.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_SendsText_ReturnsPoem.yaml new file mode 100644 index 00000000..5313bde1 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_SendsText_ReturnsPoem.yaml @@ -0,0 +1,42 @@ +test: +- type: input + activity: + type: message + channelId: msteams + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: poem + id: activity989 + timestamp: '2025-07-07T21:24:15.000Z' + localTimestamp: '2025-07-07T14:24:15.000-07:00' + localTimezone: America/Los_Angeles + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + channelData: + tenant: + id: tenant-001 +- type: skip +- type: assertion + selector: + activity: + type: typing + activity: + text: ["CONTAINS", "Hold on for an awesome poem about Apollo"] +- type: assertion + selector: + index: -1 + activity: + text: ["CONTAINS", "Apollo"] +- type: breakpoint \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml new file mode 100644 index 00000000..8c12b584 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml @@ -0,0 +1,66 @@ +test: +- type: input + activity: + type: message + id: activity989 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-07T21:24:15.930Z' + localTimestamp: '2025-07-07T14:24:15.930-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:60209/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + text: 'w: What''s the weather?' + channelData: + tenant: + id: tenant-001 +- type: input + activity: + type: message + id: activity990 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-07T21:24:15.000Z' + localTimestamp: '2025-07-07T14:24:15.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:60209/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + text: 'w: Seattle for Today' + channelData: + tenant: + id: tenant-001 +- type: skip \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendActivity_StartTeamsMeeting_ExpectMessage.yaml b/dev/integration/tests/basic_agent/msteams/SendActivity_StartTeamsMeeting_ExpectMessage.yaml new file mode 100644 index 00000000..abd33918 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendActivity_StartTeamsMeeting_ExpectMessage.yaml @@ -0,0 +1,54 @@ +test: +- type: input + activity: + type: event + name: application/vnd.microsoft.meetingStart + from: + id: user-001 + name: Jordan Lee + recipient: + id: bot-001 + name: TeamHelperBot + conversation: + id: conversation-abc123 + channelId: msteams + serviceUrl: https://smba.trafficmanager.net/amer/ + value: + trigger: onMeetingStart + id: meeting-12345 + title: Quarterly Planning Meeting + startTime: '2025-07-28T21:00:00Z' + joinUrl: https://teams.microsoft.com/l/meetup-join/... + meetingType: scheduled + meeting: + organizer: + id: user-002 + name: Morgan Rivera + participants: + - id: user-001 + name: Jordan Lee + - id: user-003 + name: Taylor Kim + - id: user-004 + name: Riley Chen + location: Microsoft Teams Meeting + id: activity989 + timestamp: '2025-07-07T21:24:15.000Z' + localTimestamp: '2025-07-07T14:24:15.000-07:00' + localTimezone: America/Los_Angeles + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + channelData: + tenant: + id: tenant-001 +- type: assertion + selector: -1 + activity: + type: message + text: "Meeting started with ID: meeting-12345" \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/integration/tests/basic_agent/msteams/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml new file mode 100644 index 00000000..d4beacc4 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml @@ -0,0 +1,42 @@ +test: +- type: input + activity: + type: message + id: activity989 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-07T21:24:15.000Z' + localTimestamp: '2025-07-07T14:24:15.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:60209/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + text: 'w: What''s the weather in Seattle today?' + channelData: + tenant: + id: tenant-001 +- type: skip +- type: assertion + selector: + index: -1 + activity: + type: message + attachments: + - contentType: application/vnd.microsoft.card.adaptive + content: ["RE_MATCH", "(īŋŊ|\\u00B0|Missing temperature inside adaptive card:)"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml b/dev/integration/tests/basic_agent/msteams/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml new file mode 100644 index 00000000..c995665e --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml @@ -0,0 +1,46 @@ +test: +- type: input + activity: + type: message + id: activity989 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-07T21:24:15.000Z' + localTimestamp: '2025-07-07T14:24:15.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:60209/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + textFormat: plain + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + text: poem + channelData: + tenant: + id: tenant-001 +- type: skip +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Apollo" ] +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "\n" ] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendInvoke_QueryLink_ReturnsText.yaml b/dev/integration/tests/basic_agent/msteams/SendInvoke_QueryLink_ReturnsText.yaml new file mode 100644 index 00000000..34f0e04c --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendInvoke_QueryLink_ReturnsText.yaml @@ -0,0 +1,41 @@ +test: +- type: input + activity: + type: invoke + id: invoke123 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-08T22:53:24.000Z' + localTimestamp: '2025-07-08T15:53:24.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:52065/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + channelData: + source: + name: compose + tenant: + id: tenant-001 + value: + url: https://github.com/microsoft/Agents-for-net/blob/users/tracyboehrer/cards-sample/src/samples/Teams/TeamsAgent/TeamsAgent.cs + name: composeExtension/queryLink +- type: skip +- type: assertion + invokeResponse: + composeExtension: + text: ["CONTAINS", "On Query Link"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml b/dev/integration/tests/basic_agent/msteams/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml new file mode 100644 index 00000000..719d8e35 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml @@ -0,0 +1,48 @@ +test: +- type: input + activity: + type: invoke + id: invoke123 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-08T22:53:24.000Z' + localTimestamp: '2025-07-08T15:53:24.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:52065/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + channelData: + source: + name: compose + tenant: + id: tenant-001 + value: + commandId: findNuGetPackage + parameters: + - name: NuGetPackageName + value: Newtonsoft.Json + queryOptions: + skip: 0 + count: 10 + name: composeExtension/query +- type: skip +- type: assertion + invokeResponse: + composeExtension: + text: ["CONTAINS", "result"] + attachments: ["LEN_GREATER_THAN", 0] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendInvoke_SelectItem_ReceiveItem.yaml b/dev/integration/tests/basic_agent/msteams/SendInvoke_SelectItem_ReceiveItem.yaml new file mode 100644 index 00000000..c5c9871b --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendInvoke_SelectItem_ReceiveItem.yaml @@ -0,0 +1,49 @@ +test: +- type: input + activity: + type: invoke + id: invoke123 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-08T22:53:24.000Z' + localTimestamp: '2025-07-08T15:53:24.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:52065/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + channelData: + source: + name: compose + tenant: + id: tenant-001 + value: + '@id': https://www.nuget.org/packages/Newtonsoft.Json/13.0.1 + id: Newtonsoft.Json + version: 13.0.1 + description: Json.NET is a popular high-performance JSON framework for .NET + projectUrl: https://www.newtonsoft.com/json + iconUrl: https://www.newtonsoft.com/favicon.ico + name: composeExtension/selectItem +- type: skip +- type: assertion + invokeResponse: + composeExtension: + type: result + text: ["CONTAINS", "Newtonsoft.Json"] + attachments: + contentType: application/vnd.microsoft.card.thumbnail \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml b/dev/integration/tests/basic_agent/msteams/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml new file mode 100644 index 00000000..146e361f --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml @@ -0,0 +1,39 @@ +test: +- type: input + activity: + type: invoke + id: invoke456 + channelId: msteams + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-22T19:21:03.000Z' + localTimestamp: '2025-07-22T12:21:03.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:63676/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + value: + parameters: + - value: hi +- type: skip +- type: assertion + invokeResponse: + message: ["EQUALS", "Invoke received."] + status: 200 + data: + parameters: + - value: ["CONTAINS", "hi"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml b/dev/integration/tests/basic_agent/msteams/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml new file mode 100644 index 00000000..dce4b188 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml @@ -0,0 +1,23 @@ +test: +- type: input + activity: + type: invoke + channelId: msteams + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + name: adaptiveCard/action + value: + action: + type: Action.Execute + title: Execute doStuff + verb: doStuff + data: + usertext: hi + trigger: manual +- type: skip \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/msteams/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml b/dev/integration/tests/basic_agent/msteams/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml new file mode 100644 index 00000000..22daad44 --- /dev/null +++ b/dev/integration/tests/basic_agent/msteams/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml @@ -0,0 +1,29 @@ +test: +- type: input + activity: + type: message + id: activityEvS8 + timestamp: '2025-06-18T18:47:46.000Z' + localTimestamp: '2025-06-18T11:47:46.000-07:00' + localTimezone: America/Los_Angeles + channelId: msteams + from: + id: user1 + name: '' + conversation: + id: conv1 + recipient: + id: basic-agent@sometext + name: basic-agent + textFormat: plain + locale: en-US + text: stream + attachments: [] + entities: + - type: ClientCapabilities + requiresBotState: true + supportsListening: true + supportsTts: true + channelData: + clientActivityID: activityAZ8 +- type: skip \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/test_basic_agent.py b/dev/integration/tests/basic_agent/test_basic_agent.py new file mode 100644 index 00000000..840ab4cb --- /dev/null +++ b/dev/integration/tests/basic_agent/test_basic_agent.py @@ -0,0 +1,15 @@ +import pytest + +from microsoft_agents.testing import ( + ddt, + Integration, +) + + +@ddt("tests/basic_agent/directline", prefix="directline") +@ddt("tests/basic_agent/webchat", prefix="webchat") +@ddt("tests/basic_agent/msteams", prefix="msteams") +class TestBasicAgent(Integration): + _agent_url = "http://localhost:3978/" + _service_url = "http://localhost:8001/" + _config_path = "agents/basic_agent/python/.env" diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml b/dev/integration/tests/basic_agent/webchat/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml new file mode 100644 index 00000000..738bb9e8 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendActivity_ConversationUpdate_ReturnsWelcomeMessage.yaml @@ -0,0 +1,25 @@ +test: +- type: input + activity: + type: conversationUpdate + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation123 + recipient: + id: bot1 + name: Bot + membersAdded: + - id: user1 + name: User + locale: en-US +- type: assertion + selector: + index: -1 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "Hello and Welcome!"] diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_EndConversation_DeleteConversation.yaml b/dev/integration/tests/basic_agent/webchat/SendActivity_EndConversation_DeleteConversation.yaml new file mode 100644 index 00000000..e530f06f --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendActivity_EndConversation_DeleteConversation.yaml @@ -0,0 +1,26 @@ +test: +- type: input + activity: + type: message + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: end + locale: en-US +- type: assertion + selector: + index: -2 + activity: + type: message + text: ["CONTAINS", "Ending conversation..."] +- type: assertion + selector: + index: -1 + activity: + type: endOfConversation \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/integration/tests/basic_agent/webchat/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml new file mode 100644 index 00000000..572def5e --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendActivity_RemoveHeartMessageReaction_ReturnsMessageReactionHeart.yaml @@ -0,0 +1,32 @@ +test: +- type: input + activity: + reactionsRemoved: + - type: heart + type: messageReaction + timestamp: '2025-07-10T02:30:00.000Z' + id: '1752114287789' + channelId: webchat + from: + id: from29ed + aadObjectId: aad-user1 + conversation: + conversationType: personal + tenantId: tenant6d4 + id: cpersonal-chat-id + recipient: + id: basic-agent@sometext + name: basic-agent + channelData: + tenant: + id: tenant6d4 + legacy: + replyToId: legacy_id + replyToId: '1752114287789' + locale: en-US +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Message Reaction Removed: heart"] diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml b/dev/integration/tests/basic_agent/webchat/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml new file mode 100644 index 00000000..ea00712f --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendActivity_SendHeartMessageReaction_ReturnsMessageReactionHeart.yaml @@ -0,0 +1,32 @@ +test: +- type: input + activity: + reactionsAdded: + - type: heart + type: messageReaction + timestamp: '2025-07-10T02:25:04.000Z' + id: '1752114287789' + channelId: webchat + from: + id: from29ed + aadObjectId: aad-user1 + conversation: + conversationType: personal + tenantId: tenant6d4 + id: cpersonal-chat-id + recipient: + id: basic-agent@sometext + name: basic-agent + channelData: + tenant: + id: tenant6d4 + legacy: + replyToId: legacy_id + replyToId: '1752114287789' + locale: en-US +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Message Reaction Added: heart"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml b/dev/integration/tests/basic_agent/webchat/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml new file mode 100644 index 00000000..74f6d7fa --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendActivity_SendsHelloWorld_ReturnsHelloWorld.yaml @@ -0,0 +1,36 @@ +test: +- type: input + activity: + type: message + id: activity-hello-webchat-001 + timestamp: '2025-07-30T22:59:55.000Z' + localTimestamp: '2025-07-30T15:59:55.000-07:00' + localTimezone: America/Los_Angeles + channelId: webchat + from: + id: user1 + name: '' + conversation: + id: conversation-hello-webchat-001 + recipient: + id: basic-agent@sometext + name: basic-agent + textFormat: plain + locale: en-US + text: hello world + attachments: [] + entities: + - type: ClientCapabilities + requiresBotState: true + supportsListening: true + supportsTts: true + channelData: + clientActivityID: client-activity-hello-webchat-001 +- type: assertion + selector: + index: -1 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "You said: hello world"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendsHi5_Returns5HiActivities.yaml b/dev/integration/tests/basic_agent/webchat/SendActivity_SendsHi5_Returns5HiActivities.yaml new file mode 100644 index 00000000..8e4b46cb --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendActivity_SendsHi5_Returns5HiActivities.yaml @@ -0,0 +1,47 @@ +test: +- type: input + activity: + type: message + id: activity989 + channelId: webchat + from: + id: user-id-0 + name: Alex Wilber + conversation: + id: personal-chat-id-hi5 + recipient: + id: bot-001 + name: Test Bot + text: hi 5 + locale: en-US +- type: assertion + quantifier: one + activity: + type: message + text: ["CONTAINS", "[0] You said: hi"] +- type: assertion + quantifier: one + activity: + type: message + text: ["CONTAINS", "[1] You said: hi"] +- type: assertion + quantifier: one + activity: + type: message + text: ["CONTAINS", "[2] You said: hi"] +- type: assertion + quantifier: one + activity: + type: message + text: ["CONTAINS", "[3] You said: hi"] +- type: assertion + quantifier: one + activity: + type: message + text: ["CONTAINS", "[4] You said: hi"] +- type: assertion # only 5 hi activities are returned + quantifier: none + selector: + index: 5 + activity: + type: message \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml b/dev/integration/tests/basic_agent/webchat/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml new file mode 100644 index 00000000..484b7ab6 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendActivity_SendsMessageActivityToAcSubmit_ReturnValidResponse.yaml @@ -0,0 +1,55 @@ +test: +- type: input + activity: + type: message + id: activity-submit-001 + timestamp: '2025-07-30T23:06:37.000Z' + localTimestamp: '2025-07-30T16:06:37.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: https://webchat.botframework.com/ + channelId: webchat + from: + id: user1 + name: '' + conversation: + id: conversation-submit-001 + recipient: + id: basic-agent@sometext + name: basic-agent + locale: en-US + attachments: [] + channelData: + postBack: true + clientActivityID: client-activity-submit-001 + value: + verb: doStuff + id: doStuff + type: Action.Submit + test: test + data: + name: test + usertext: hello +- type: assertion + selector: + index: -1 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "doStuff"] +- type: assertion + selector: + index: -1 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "Action.Submit"] +- type: assertion + selector: + index: -1 + activity: + type: message + activity: + type: message + text: ["CONTAINS", "hello"] diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/integration/tests/basic_agent/webchat/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml new file mode 100644 index 00000000..5b0b7881 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml @@ -0,0 +1,24 @@ +test: +- type: input + activity: + type: message + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: 'w: Get the weather in Seattle for Today' + locale: en-US +- type: skip +- type: assertion + selector: + index: -1 + activity: + type: message + attachments: + - contentType: application/vnd.microsoft.card.adaptive + content: ["RE_MATCH", "(īŋŊ|\\u00B0|Missing temperature inside adaptive card:)"] diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SendsText_ReturnsPoem.yaml b/dev/integration/tests/basic_agent/webchat/SendActivity_SendsText_ReturnsPoem.yaml new file mode 100644 index 00000000..5dc1f2f1 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendActivity_SendsText_ReturnsPoem.yaml @@ -0,0 +1,27 @@ +test: +- type: input + activity: + type: message + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: poem + locale: en-US +- type: skip +- type: assertion + selector: + activity: + type: typing + activity: + text: ["CONTAINS", "Hold on for an awesome poem about Apollo"] +- type: assertion + selector: + index: -1 + activity: + text: ["CONTAINS", "Apollo"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml b/dev/integration/tests/basic_agent/webchat/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml new file mode 100644 index 00000000..f5da99f7 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendActivity_SimulateMessageLoop_ExpectQuestionAboutTimeAndReturnsWeather.yaml @@ -0,0 +1,30 @@ +test: +- type: input + activity: + type: message + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: 'w: what''s the weather?''' + locale: en-US +- type: input + activity: + type: message + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: 'w: Seattle for today' + locale: en-US +- type: skip \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml b/dev/integration/tests/basic_agent/webchat/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml new file mode 100644 index 00000000..24c23b5c --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendExpectedRepliesActivity_SendsSeattleTodayWeather_ReturnsWeather.yaml @@ -0,0 +1,24 @@ +test: +- type: input + activity: + type: message + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: 'w: Get the weather in Seattle for Today' + locale: en-US +- type: skip +- type: assertion + selector: + index: -1 + activity: + type: message + attachments: + - contentType: application/vnd.microsoft.card.adaptive + content: ["RE_MATCH", "(īŋŊ|\\u00B0|Missing temperature inside adaptive card:)"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml b/dev/integration/tests/basic_agent/webchat/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml new file mode 100644 index 00000000..e48fc29d --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendExpectedRepliesActivity_SendsText_ReturnsPoem.yaml @@ -0,0 +1,28 @@ +test: +- type: input + activity: + type: message + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + text: poem + locale: en-US +- type: skip +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "Apollo" ] +- type: assertion + selector: + index: -1 + activity: + type: message + text: ["CONTAINS", "\n" ] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendInvoke_QueryLink_ReturnsText.yaml b/dev/integration/tests/basic_agent/webchat/SendInvoke_QueryLink_ReturnsText.yaml new file mode 100644 index 00000000..6f56b393 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendInvoke_QueryLink_ReturnsText.yaml @@ -0,0 +1,22 @@ +test: +- type: input + activity: + type: invoke + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + name: composeExtension/queryLink + value: + url: https://github.com/microsoft/Agents-for-net/blob/users/tracyboehrer/cards-sample/src/samples/Teams/TeamsAgent/TeamsAgent.cs +- type: skip +- type: assertion + quantifier: any + invokeResponse: + composeExtension: + text: ["CONTAINS", "On Query Link"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml b/dev/integration/tests/basic_agent/webchat/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml new file mode 100644 index 00000000..a0939858 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendInvoke_QueryPackage_ReceiveInvokeResponse.yaml @@ -0,0 +1,30 @@ +test: +- type: input + activity: + type: invoke + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + name: composeExtension/query + value: + commandId: findNuGetPackage + parameters: + - name: NuGetPackageName + value: Newtonsoft.Json + queryOptions: + skip: 0 + count: 10 + locale: en-US +- type: skip +- type: assertion + quantifier: any + invokeResponse: + composeExtension: + text: ["CONTAINS", "result"] + attachments: ["LEN_GREATER_THAN", 0] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendInvoke_SelectItem_ReceiveItem.yaml b/dev/integration/tests/basic_agent/webchat/SendInvoke_SelectItem_ReceiveItem.yaml new file mode 100644 index 00000000..11b159e8 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendInvoke_SelectItem_ReceiveItem.yaml @@ -0,0 +1,32 @@ +test: +- type: input + activity: + type: invoke + id: invoke123 + channelId: webchat + from: + id: user-id-0 + name: Alex Wilber + conversation: + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + value: + '@id': https://www.nuget.org/packages/Newtonsoft.Json/13.0.1 + id: Newtonsoft.Json + version: 13.0.1 + description: Json.NET is a popular high-performance JSON framework for .NET + projectUrl: https://www.newtonsoft.com/json + iconUrl: https://www.newtonsoft.com/favicon.ico + name: composeExtension/selectItem + locale: en-US +- type: skip +- type: assertion + quantifier: any + invokeResponse: + composeExtension: + type: result + text: ["CONTAINS", "Newtonsoft.Json"] + attachments: + contentType: application/vnd.microsoft.card.thumbnail \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml b/dev/integration/tests/basic_agent/webchat/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml new file mode 100644 index 00000000..b963d360 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendInvoke_SendBasicInvokeActivity_ReceiveInvokeResponse.yaml @@ -0,0 +1,40 @@ +test: +- type: input + activity: + type: invoke + id: invoke456 + channelId: webchat + from: + id: user-id-0 + name: Alex Wilber + aadObjectId: aad-user-alex + timestamp: '2025-07-22T19:21:03.000Z' + localTimestamp: '2025-07-22T12:21:03.000-07:00' + localTimezone: America/Los_Angeles + serviceUrl: http://localhost:63676/_connector + conversation: + conversationType: personal + tenantId: tenant-001 + id: personal-chat-id + recipient: + id: bot-001 + name: Test Bot + locale: en-US + entities: + - type: clientInfo + locale: en-US + country: US + platform: Web + timezone: America/Los_Angeles + value: + parameters: + - value: hi +- type: skip +- type: assertion + quantifier: any + invokeResponse: + message: ["EQUALS", "Invoke received."] + status: 200 + data: + parameters: + - value: ["CONTAINS", "hi"] \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml b/dev/integration/tests/basic_agent/webchat/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml new file mode 100644 index 00000000..af1d32e5 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendInvoke_SendsInvokeActivityToAcExecute_ReturnsValidAdaptiveCardInvokeResponse.yaml @@ -0,0 +1,24 @@ +test: +- type: input + activity: + type: invoke + channelId: webchat + from: + id: user1 + name: User + conversation: + id: conversation-abc123 + recipient: + id: bot1 + name: Bot + name: adaptiveCard/action + value: + action: + type: Action.Execute + title: Execute doStuff + verb: doStuff + data: + usertext: hi + trigger: manual + locale: en-US +- type: skip \ No newline at end of file diff --git a/dev/integration/tests/basic_agent/webchat/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml b/dev/integration/tests/basic_agent/webchat/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml new file mode 100644 index 00000000..90a5bc45 --- /dev/null +++ b/dev/integration/tests/basic_agent/webchat/SendStreamActivity_SendStreamMessage_ExpectStreamResponses.yaml @@ -0,0 +1,29 @@ +test: +- type: input + activity: + type: message + id: activity-stream-webchat-001 + timestamp: '2025-06-18T18:47:46.000Z' + localTimestamp: '2025-06-18T11:47:46.000-07:00' + localTimezone: America/Los_Angeles + channelId: webchat + from: + id: user1 + name: '' + conversation: + id: conversation-stream-webchat-001 + recipient: + id: basic-agent@sometext + name: basic-agent + textFormat: plain + locale: en-US + text: stream + attachments: [] + entities: + - type: ClientCapabilities + requiresBotState: true + supportsListening: true + supportsTts: true + channelData: + clientActivityID: client-activity-stream-webchat-001 +- type: skip \ No newline at end of file diff --git a/dev/integration/tests/quickstart/__init__.py b/dev/integration/tests/quickstart/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/integration/tests/quickstart/directline/_parent.yaml b/dev/integration/tests/quickstart/directline/_parent.yaml new file mode 100644 index 00000000..fcac07b1 --- /dev/null +++ b/dev/integration/tests/quickstart/directline/_parent.yaml @@ -0,0 +1,16 @@ +name: quickstart +defaults: + input: + activity: + channelId: webchat + locale: en-US + # serviceUrl: http://localhost:56150 + # deliveryMode: expectReplies + conversation: + id: conv1 + from: + id: user1 + name: User + recipient: + id: bot + name: Bot \ No newline at end of file diff --git a/dev/integration/tests/quickstart/directline/conversation_update.yaml b/dev/integration/tests/quickstart/directline/conversation_update.yaml new file mode 100644 index 00000000..3ff217c9 --- /dev/null +++ b/dev/integration/tests/quickstart/directline/conversation_update.yaml @@ -0,0 +1,36 @@ +parent: _parent.yaml +test: + - type: input + activity: + type: conversationUpdate + id: "123" + timestamp: 2025-07-30T23:01:11.0447215Z + localTimestamp: 2025-07-30T15:59:55.595-07:00 + localTimezone: America/Los_Angeles + from: + id: user + recipient: + id: bot-id + name: bot + membersAdded: + - id: bot-id + name: bot + - id: user + textFormat: plain + attachments: [] + entities: + - type: ClientCapabilities + requiresBotState: true + supportsListening: true + supportsTts: true + channelData: + clientActivityId: 123 + - type: sleep + duration: .5 + - type: assertion + selector: + activity: + type: message + activity: + type: message + text: ["CONTAINS", "Welcome to the empty agent!"] diff --git a/dev/integration/tests/quickstart/directline/send_hello.yaml b/dev/integration/tests/quickstart/directline/send_hello.yaml new file mode 100644 index 00000000..3e3c6bef --- /dev/null +++ b/dev/integration/tests/quickstart/directline/send_hello.yaml @@ -0,0 +1,16 @@ +parent: _parent.yaml +test: + - type: input + activity: + type: message + text: hello + - type: sleep + duration: .5 + - type: assertion # assert that a typing activity was sent + selector: + index: -1 + activity: + type: message + activity: + type: message + text: "Hello!" \ No newline at end of file diff --git a/dev/integration/tests/quickstart/directline/send_hi.yaml b/dev/integration/tests/quickstart/directline/send_hi.yaml new file mode 100644 index 00000000..ab6eabbc --- /dev/null +++ b/dev/integration/tests/quickstart/directline/send_hi.yaml @@ -0,0 +1,25 @@ +parent: _parent.yaml +test: + - type: input + activity: + type: message + text: hi + - type: input + activity: + type: message + text: hi + - type: sleep + duration: .5 + - type: assertion + selector: + activity: + type: message + activity: + type: message + text: "you said: hi" + - type: assertion # assert that a typing activity was sent + selector: + index: -1 + activity: + type: typing + quantifier: one \ No newline at end of file diff --git a/dev/integration/tests/quickstart/test_quickstart_sample.py b/dev/integration/tests/quickstart/test_quickstart_sample.py new file mode 100644 index 00000000..afd45e6c --- /dev/null +++ b/dev/integration/tests/quickstart/test_quickstart_sample.py @@ -0,0 +1,20 @@ +import pytest + +from microsoft_agents.testing import ( + ddt, + Integration, + AiohttpEnvironment, +) + +from ...samples import QuickstartSample + + +@ddt("tests/quickstart/directline") +class TestQuickstartDirectline(Integration): + _sample_cls = QuickstartSample + _environment_cls = AiohttpEnvironment + + +@ddt("tests/quickstart/directline") +@pytest.mark.skipif(True, reason="Skipping external agent tests for now.") +class TestQuickstartExternalDirectline(Integration): ... diff --git a/dev/integration/tests/test_expect_replies.py b/dev/integration/tests/test_expect_replies.py new file mode 100644 index 00000000..86d23cd7 --- /dev/null +++ b/dev/integration/tests/test_expect_replies.py @@ -0,0 +1,47 @@ +import pytest +import logging + +from microsoft_agents.activity import Activity + +from microsoft_agents.testing import ( + ddt, + Integration, + AiohttpEnvironment, +) + +from ..samples import BasicSample + + +class BasicSampleWithLogging(BasicSample): + + async def init_app(self): + + logging.getLogger("microsoft_agents").setLevel(logging.DEBUG) + + await super().init_app() + + +class TestBasicDirectline(Integration): + _sample_cls = BasicSampleWithLogging + _environment_cls = AiohttpEnvironment + + @pytest.mark.asyncio + async def test_expect_replies_without_service_url( + self, agent_client, response_client + ): + + activity = Activity( + type="message", + text="hi", + conversation={"id": "conv-id"}, + channel_id="test", + from_property={"id": "from-id"}, + to={"id": "to-id"}, + delivery_mode="expectReplies", + locale="en-US", + ) + + res = await agent_client.send_expect_replies(activity) + + breakpoint() + res = Activity.model_validate(res) diff --git a/dev/microsoft-agents-testing/README.md b/dev/microsoft-agents-testing/README.md index ef50bf39..2c52b935 100644 --- a/dev/microsoft-agents-testing/README.md +++ b/dev/microsoft-agents-testing/README.md @@ -1,17 +1,48 @@ # Microsoft 365 Agents SDK for Python - Testing Framework -A comprehensive testing framework designed specifically for Microsoft 365 Agents SDK, providing essential utilities and abstractions to streamline integration testing, authentication, and end-to-end agent validation. +A comprehensive testing framework designed specifically for Microsoft 365 Agents SDK, providing essential utilities and abstractions to streamline integration testing, authentication, data-driven testing, and end-to-end agent validation. + +## Table of Contents + +- [Why This Package Exists](#why-this-package-exists) +- [Key Features](#key-features) + - [Authentication Utilities](#authentication-utilities) + - [Integration Test Framework](#integration-test-framework) + - [Agent Communication Clients](#agent-communication-clients) + - [Data-Driven Testing](#data-driven-testing) + - [Advanced Assertions Framework](#advanced-assertions-framework) + - [Testing Utilities](#testing-utilities) +- [Installation](#installation) +- [Quick Start](#quick-start) +- [Usage Guide](#usage-guide) +- [Advanced Examples](#advanced-examples) +- [API Reference](#api-reference) +- [CI/CD Integration](#cicd-integration) +- [Contributing](#contributing) ## Why This Package Exists -Building and testing conversational agents presents unique challenges that standard testing frameworks don't address. This package eliminates these pain points by providing useful abstractions specifically designed for agent testing scenarios. +Building and testing conversational agents presents unique challenges that standard testing frameworks don't address. This package eliminates these pain points by providing powerful abstractions specifically designed for agent testing scenarios, including support for data-driven testing with YAML/JSON configurations. + +**Key Benefits:** +- Write tests once in YAML/JSON, run them everywhere +- Reduce boilerplate code with pre-built fixtures and clients +- Validate complex conversation flows with declarative assertions +- Maintain test suites that are easy to read and maintain +- Integrate seamlessly with pytest and CI/CD pipelines ## Key Features ### 🔐 Authentication Utilities -- **OAuth2 Token Generation**: Generate access tokens using client credentials flow -- **Configuration-Based Auth**: Load credentials from environment variables or config objects -- **MSAL Integration**: Built-in support for Microsoft Authentication Library + +Generate OAuth2 access tokens for testing secured agents with Microsoft Authentication Library (MSAL) integration. + +**Features:** +- Client credentials flow support +- Environment variable configuration +- SDK config integration + +**Example:** ```python from microsoft_agents.testing import generate_token, generate_token_from_config @@ -28,13 +59,28 @@ token = generate_token_from_config(sdk_config) ``` ### đŸ§Ē Integration Test Framework -- **Pytest Fixtures**: Pre-built fixtures for common test scenarios -- **Environment Abstraction**: Reusable environment setup for different hosting configurations -- **Sample Management**: Base classes for organizing test samples and configurations -- **Application Runners**: Abstract server lifecycle management for integration tests + +Pre-built pytest fixtures and abstractions for agent integration testing. + +**Features:** +- Pytest fixture integration +- Environment abstraction for different hosting configurations +- Sample management for test organization +- Application lifecycle management +- Automatic setup and teardown + +**Example:** ```python -from microsoft_agents.testing import Integration, Environment, Sample +from microsoft_agents.testing import Integration, AiohttpEnvironment, Sample + +class MyAgentSample(Sample): + async def init_app(self): + self.app = create_my_agent_app(self.env) + + @classmethod + async def get_config(cls): + return {"service_url": "http://localhost:3978"} class MyAgentTests(Integration): _sample_cls = MyAgentSample @@ -48,13 +94,21 @@ class MyAgentTests(Integration): ``` ### 🤖 Agent Communication Clients -- **AgentClient**: High-level client for sending Activities to agents -- **ResponseClient**: Handle responses from agent services -- **Automatic Token Management**: Clients handle authentication automatically -- **Delivery Mode Support**: Test both standard and `ExpectReplies` delivery patterns + +High-level clients for sending and receiving activities from agents under test. + +**Features:** +- Simple text message sending +- Full Activity object support +- Automatic token management +- Support for `expectReplies` delivery mode +- Response collection and management + +**AgentClient Example:** ```python from microsoft_agents.testing import AgentClient +from microsoft_agents.activity import Activity, ActivityTypes client = AgentClient( agent_url="http://localhost:3978", @@ -67,54 +121,468 @@ client = AgentClient( # Send simple text message response = await client.send_activity("What's the weather?") -# Send Activity with ExpectReplies -replies = await client.send_expect_replies( - Activity(type=ActivityTypes.message, text="Hello") +# Send full Activity object +activity = Activity(type=ActivityTypes.message, text="Hello") +response = await client.send_activity(activity) + +# Send with expectReplies delivery mode +replies = await client.send_expect_replies("What can you do?") +for reply in replies: + print(reply.text) +``` + +**ResponseClient Example:** + +```python +from microsoft_agents.testing import ResponseClient + +# Create response client to collect agent responses +async with ResponseClient(host="localhost", port=9873) as response_client: + # ... send activities with agent_client ... + + # Collect all responses + responses = await response_client.pop() + assert len(responses) > 0 +``` + +### 📋 Data-Driven Testing + +Write test scenarios in YAML or JSON files and execute them automatically. Perfect for creating reusable test suites, regression tests, and living documentation. + +**Features:** +- Declarative test definition in YAML/JSON +- Parent/child file inheritance for shared defaults +- Multiple step types (input, assertion, sleep, breakpoint) +- Flexible assertions with selectors and quantifiers +- Automatic test discovery and generation +- Field-level assertion operators + +#### Using the @ddt Decorator + +The @ddt (data-driven tests) decorator automatically loads test files and generates pytest test methods: + +```python +from microsoft_agents.testing import Integration, AiohttpEnvironment, ddt + +@ddt("tests/my_agent/test_cases", recursive=True) +class TestMyAgent(Integration): + _sample_cls = MyAgentSample + _environment_cls = AiohttpEnvironment + _agent_url = "http://localhost:3978" + _cid = "test-conversation" +``` + +This will: +1. Load all `.yaml` and `.json` files from `tests/my_agent/test_cases` (and subdirectories if `recursive=True`) +2. Create a pytest test method for each file (e.g., `test_data_driven__greeting_test`) +3. Execute the test flow defined in each file + +#### Test File Format + +**Shared Defaults (parent.yaml):** + +```yaml +name: directline +defaults: + input: + activity: + channelId: directline + locale: en-US + serviceUrl: http://localhost:56150 + deliveryMode: expectReplies + conversation: + id: conv1 + from: + id: user1 + name: User + recipient: + id: bot + name: Bot +``` + +**Test File (greeting_test.yaml):** + +```yaml +parent: parent.yaml +name: greeting_test +description: Test basic greeting conversation +test: + - type: input + activity: + type: message + text: hello world + + - type: assertion + selector: + activity: + type: message + activity: + type: message + text: "[0] You said: hello world" + + - type: input + activity: + type: message + text: hello again + + - type: assertion + selector: + index: -1 # Select the last matching activity + activity: + type: message + activity: + type: message + text: "[1] You said: hello again" +``` + +#### Test Step Types + +##### Input Steps + +Send activities to the agent under test: + +```yaml +- type: input + activity: + type: message + text: "What's the weather?" +``` + +With overrides: + +```yaml +- type: input + activity: + type: message + text: "Hello" + locale: "fr-FR" # Override default locale + channelData: + custom: "value" +``` + +##### Assertion Steps + +Verify agent responses with flexible matching: + +```yaml +- type: assertion + quantifier: all # Options: all, any, one, none + selector: + index: 0 # Optional: select by index (0, -1, etc.) + activity: + type: message # Filter by activity fields + activity: + type: message + text: ["CONTAINS", "sunny"] # Use operators for flexible matching +``` + +**Quantifiers:** +- `all` (default): Every selected activity must match +- `any`: At least one activity must match +- `one`: Exactly one activity must match +- `none`: No activities should match + +**Selectors:** +- `activity`: Filter activities by field values +- `index`: Select specific activity by index (supports negative indices) + +**Field Assertion Operators:** +- `["CONTAINS", "substring"]`: Check if string contains substring +- `["NOT_CONTAINS", "substring"]`: Check if string doesn't contain substring +- `["RE_MATCH", "pattern"]`: Check if string matches regex pattern +- `["IN", [list]]`: Check if value is in list +- `["NOT_IN", [list]]`: Check if value is not in list +- `["EQUALS", value]`: Explicit equality check +- `["NOT_EQUALS", value]`: Explicit inequality check +- `["GREATER_THAN", number]`: Numeric comparison +- `["LESS_THAN", number]`: Numeric comparison +- Direct value: Implicit equality check + +##### Sleep Steps + +Add delays between operations: + +```yaml +- type: sleep + duration: 0.5 # seconds +``` + +With default duration: + +```yaml +defaults: + sleep: + duration: 0.2 + +test: + - type: sleep # Uses default duration +``` + +##### Breakpoint Steps + +Pause execution for debugging: + +```yaml +- type: breakpoint +``` + +When the test reaches this step, it will trigger a Python breakpoint, allowing you to inspect state in a debugger. + +#### Loading Tests Programmatically + +Load and run tests manually without the decorator: + +```python +from microsoft_agents.testing import load_ddts, DataDrivenTest + +# Load all test files from a directory +tests = load_ddts("tests/my_agent", recursive=True) + +# Run specific tests +for test in tests: + print(f"Running: {test.name}") + await test.run(agent_client, response_client) +``` + +Load from specific file: + +```python +tests = load_ddts("tests/greeting_test.yaml", recursive=False) +test = tests[0] +await test.run(agent_client, response_client) +``` + +### ✅ Advanced Assertions Framework + +Powerful assertion system for validating agent responses with flexible matching criteria. + +#### ModelAssertion + +Create assertions for validating lists of activities: + +```python +from microsoft_agents.testing import ModelAssertion, Selector, AssertionQuantifier + +# Create an assertion +assertion = ModelAssertion( + assertion={"type": "message", "text": "Hello"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ALL ) + +# Test activities +activities = [...] # List of Activity objects +passes, error = assertion.check(activities) + +# Or use as callable (raises AssertionError on failure) +assertion(activities) +``` + +From configuration dictionary: + +```python +config = { + "activity": {"type": "message", "text": "Hello"}, + "selector": {"activity": {"type": "message"}}, + "quantifier": "all" +} +assertion = ModelAssertion.from_config(config) +``` + +#### Selectors + +Filter activities before validation: + +```python +from microsoft_agents.testing import Selector + +# Select all message activities +selector = Selector(selector={"type": "message"}) +messages = selector(activities) + +# Select the first message activity +selector = Selector(selector={"type": "message"}, index=0) +first_message = selector.select_first(activities) + +# Select the last message activity +selector = Selector(selector={"type": "message"}, index=-1) +last_message = selector(activities)[0] + +# Select by multiple fields +selector = Selector(selector={ + "type": "message", + "locale": "en-US", + "channelId": "directline" +}) +``` + +From configuration: + +```python +config = { + "activity": {"type": "message"}, + "index": -1 +} +selector = Selector.from_config(config) +``` + +#### Quantifiers + +Control how many activities must match the assertion: + +```python +from microsoft_agents.testing import AssertionQuantifier + +# ALL: Every selected activity must match (default) +quantifier = AssertionQuantifier.ALL + +# ANY: At least one activity must match +quantifier = AssertionQuantifier.ANY + +# ONE: Exactly one activity must match +quantifier = AssertionQuantifier.ONE + +# NONE: No activities should match +quantifier = AssertionQuantifier.NONE + +# From string +quantifier = AssertionQuantifier.from_config("all") +``` + +#### Field Assertions + +Test individual fields with operators: + +```python +from microsoft_agents.testing import check_field, FieldAssertionType + +# String contains +result = check_field("Hello world", ["CONTAINS", "world"]) # True + +# Regex match +result = check_field("ID-12345", ["RE_MATCH", r"ID-\d+"]) # True + +# Value in list +result = check_field(5, ["IN", [1, 3, 5, 7]]) # True + +# Value not in list +result = check_field(2, ["NOT_IN", [1, 3, 5, 7]]) # True + +# Numeric comparisons +result = check_field(10, ["GREATER_THAN", 5]) # True +result = check_field(3, ["LESS_THAN", 10]) # True + +# String doesn't contain +result = check_field("Hello", ["NOT_CONTAINS", "world"]) # True + +# Exact equality +result = check_field("test", "test") # True +result = check_field(42, ["EQUALS", 42]) # True + +# Inequality +result = check_field("foo", ["NOT_EQUALS", "bar"]) # True +``` + +Verbose checking with error details: + +```python +from microsoft_agents.testing import check_field_verbose + +passes, error_data = check_field_verbose("Hello", ["CONTAINS", "world"]) +if not passes: + print(f"Field: {error_data.field_path}") + print(f"Actual: {error_data.actual_value}") + print(f"Expected: {error_data.assertion}") + print(f"Type: {error_data.assertion_type}") +``` + +#### Activity Assertions + +Check entire activities: + +```python +from microsoft_agents.testing import check_model, assert_model + +activity = Activity(type="message", text="Hello", locale="en-US") + +# Check without raising exception +assertion = {"type": "message", "text": ["CONTAINS", "Hello"]} +result = check_activity(activity, assertion) # True + +# Check with detailed error information +passes, error_data = check_activity_verbose(activity, assertion) + +# Assert with exception on failure +assert_model(activity, assertion) # Raises AssertionError if fails +``` + +Nested field checking: + +```python +assertion = { + "type": "message", + "channelData": { + "user": { + "id": ["RE_MATCH", r"user-\d+"] + } + } +} +assert_model(activity, assertion) ``` ### 🛠ī¸ Testing Utilities -- **Activity Population**: Automatically fill default Activity properties for testing -- **URL Parsing**: Extract host and port from service URLs -- **Configuration Management**: Centralized SDK configuration for tests + +Helper functions for common testing operations. + +#### populate_activity + +Fill activity objects with default values: ```python -from microsoft_agents.testing import populate_activity, get_host_and_port +from microsoft_agents.testing import populate_activity +from microsoft_agents.activity import Activity -# Populate test activity with defaults -activity = populate_activity( - Activity(text="Hello"), - defaults={"service_url": "http://localhost", "channel_id": "test"} -) +defaults = { + "service_url": "http://localhost", + "channel_id": "test", + "locale": "en-US" +} -# Parse service URLs -host, port = get_host_and_port("http://localhost:3978/api/messages") +activity = Activity(type="message", text="Hello") +activity = populate_activity(activity, defaults) + +# activity now has service_url, channel_id, and locale set ``` -## Who Should Use This Package +#### get_host_and_port -- **Agent Developers**: Testing agents built with `microsoft-agents-hosting-core` and related packages -- **QA Engineers**: Writing integration and E2E tests for conversational AI systems -- **DevOps Teams**: Automating agent validation in CI/CD pipelines -- **Sample Authors**: Creating reproducible examples and documentation +Parse URLs to extract host and port: -## Integration with CI/CD +```python +from microsoft_agents.testing import get_host_and_port -This package is designed for seamless integration into continuous integration pipelines: +host, port = get_host_and_port("http://localhost:3978/api/messages") +# Returns: ("localhost", 3978) -```yaml -# Example: GitHub Actions -- name: Run Agent Integration Tests - run: | - pip install microsoft-agents-testing pytest pytest-asyncio - pytest tests/integration/ -v - env: - CLIENT_ID: ${{ secrets.AGENT_CLIENT_ID }} - CLIENT_SECRET: ${{ secrets.AGENT_CLIENT_SECRET }} - TENANT_ID: ${{ secrets.TENANT_ID }} +host, port = get_host_and_port("https://myagent.azurewebsites.net") +# Returns: ("myagent.azurewebsites.net", 443) +``` + +## Installation + +```bash +pip install microsoft-agents-testing +``` + +For development: + +```bash +pip install microsoft-agents-testing[dev] ``` -## Quick Start Example +## Quick Start + +### Traditional Integration Testing ```python import pytest @@ -124,11 +592,15 @@ from microsoft_agents.activity import Activity class MyAgentSample(Sample): async def init_app(self): # Initialize your agent application - self.app = create_my_agent_app(self.env) + from my_agent import create_app + self.app = create_app(self.env) @classmethod async def get_config(cls): - return {"service_url": "http://localhost:3978"} + return { + "service_url": "http://localhost:3978", + "app_id": "test-app-id", + } class TestMyAgent(Integration): _sample_cls = MyAgentSample @@ -149,23 +621,691 @@ class TestMyAgent(Integration): assert replies[0].type == "message" ``` +### Data-Driven Testing + +**Step 1:** Create test YAML files in `tests` directory + +```yaml +# tests/greeting.yaml +name: greeting_test +description: Test basic greeting functionality +defaults: + input: + activity: + type: message + locale: en-US + channelId: directline +test: + - type: input + activity: + text: Hello + + - type: assertion + activity: + type: message + text: ["CONTAINS", "Hi"] +``` + +**Step 2:** Add the @ddt decorator to your test class + +```python +from microsoft_agents.testing import Integration, AiohttpEnvironment, ddt + +@ddt("tests", recursive=True) +class TestMyAgent(Integration): + _sample_cls = MyAgentSample + _environment_cls = AiohttpEnvironment + _agent_url = "http://localhost:3978" +``` + +**Step 3:** Run tests with pytest + +```bash +pytest tests/ -v +``` + +Output: +``` +tests/test_my_agent.py::TestMyAgent::test_data_driven__greeting_test PASSED +``` + +## Usage Guide + +### Setting Up Authentication + +#### From Environment Variables + +```python +import os +from microsoft_agents.testing import generate_token + +token = generate_token( + app_id=os.getenv("CLIENT_ID"), + app_secret=os.getenv("CLIENT_SECRET"), + tenant_id=os.getenv("TENANT_ID") +) +``` + +#### From SDK Config + +```python +from microsoft_agents.testing import SDKConfig, generate_token_from_config + +config = SDKConfig() +# config loads from environment or config file +token = generate_token_from_config(config) +``` + +### Creating Custom Environments + +```python +from microsoft_agents.testing import Environment +from aiohttp import web + +class MyCustomEnvironment(Environment): + async def init_env(self, config: dict): + # Custom initialization + self.config = config + # Set up any required services, databases, etc. + + def create_runner(self, host: str, port: int): + # Return application runner + from my_agent import create_app + app = create_app(self) + return MyAppRunner(app, host, port) +``` + +### Writing Complex Assertions + +```yaml +test: + - type: input + activity: + type: message + text: "Get user profile for user123" + + - type: assertion + quantifier: one + selector: + activity: + type: message + activity: + type: message + text: ["RE_MATCH", ".*user123.*"] + attachments: + - contentType: "application/vnd.microsoft.card.adaptive" + channelData: + userId: "user123" +``` + +## Advanced Examples + +### Complex Weather Conversation + +```yaml +name: weather_conversation +description: Test multi-turn weather conversation flow +defaults: + input: + activity: + type: message + channelId: directline + locale: en-US + conversation: + id: weather-conv-1 + assertion: + quantifier: all +test: + # Initial weather query + - type: input + activity: + text: "What's the weather in Seattle?" + + - type: assertion + selector: + activity: + type: message + activity: + type: message + text: ["CONTAINS", "Seattle"] + + # Wait for async processing + - type: sleep + duration: 0.2 + + # Follow-up question + - type: input + activity: + text: "What about tomorrow?" + + - type: assertion + selector: + activity: + type: message + activity: + type: message + text: ["RE_MATCH", "tomorrow.*forecast"] + + # Verify we got exactly one final response + - type: assertion + quantifier: one + selector: + index: -1 + activity: + type: message + activity: + type: message +``` + +### Testing Invoke Activities + +```yaml +parent: parent.yaml +name: test_invoke_profile +test: + - type: input + activity: + type: invoke + name: getUserProfile + value: + userId: "12345" + + # Ensure we don't get error responses + - type: assertion + quantifier: none + activity: + type: invokeResponse + value: + status: ["IN", [400, 404, 500]] + + # Verify successful response + - type: assertion + selector: + activity: + type: invokeResponse + activity: + type: invokeResponse + value: + status: 200 + body: + userId: "12345" + name: ["CONTAINS", "John"] + email: ["RE_MATCH", ".*@example\\.com"] +``` + +### Testing Conversation Update + +```yaml +parent: parent.yaml +name: conversation_update_test +test: + - type: input + activity: + type: conversationUpdate + membersAdded: + - id: bot-id + name: bot + - id: user + from: + id: user + recipient: + id: bot-id + name: bot + channelData: + clientActivityId: "123" + + - type: assertion + selector: + activity: + type: message + activity: + type: message + text: ["CONTAINS", "Hello and Welcome!"] +``` + +### Conditional Responses + +```yaml +test: + - type: input + activity: + text: "Show me options" + + # Verify at least one message was sent + - type: assertion + quantifier: any + selector: + activity: + type: message + activity: + type: message + + # Verify adaptive card was included + - type: assertion + quantifier: one + selector: + activity: + attachments: + - contentType: "application/vnd.microsoft.card.adaptive" + activity: + type: message +``` + +### Testing with Message Reactions + +```yaml +parent: parent.yaml +test: + # Send initial message + - type: input + activity: + type: message + text: "Great job!" + id: "msg-123" + + # Add a reaction + - type: input + activity: + type: messageReaction + reactionsAdded: + - type: like + replyToId: "msg-123" + + - type: assertion + selector: + activity: + type: message + activity: + type: message + text: ["CONTAINS", "Thanks for the reaction"] +``` + +## API Reference + +### Classes + +#### Integration +Base class for integration tests with pytest fixtures. + +```python +class Integration: + _sample_cls: type[Sample] + _environment_cls: type[Environment] + _agent_url: str + _service_url: str + _cid: str + _client_id: str + _tenant_id: str + _client_secret: str + + @pytest.fixture + async def environment(self) -> Environment: ... + + @pytest.fixture + async def sample(self, environment) -> Sample: ... + + @pytest.fixture + async def agent_client(self, sample, environment) -> AgentClient: ... + + @pytest.fixture + async def response_client(self) -> ResponseClient: ... +``` + +#### AgentClient +Client for sending activities to agents. + +```python +class AgentClient: + def __init__( + self, + agent_url: str, + cid: str, + client_id: str, + tenant_id: str, + client_secret: str, + service_url: Optional[str] = None, + default_timeout: float = 5.0, + default_activity_data: Optional[Activity | dict] = None + ): ... + + async def send_activity( + self, + activity_or_text: Activity | str, + sleep: float = 0, + timeout: Optional[float] = None + ) -> str: ... + + async def send_expect_replies( + self, + activity_or_text: Activity | str, + sleep: float = 0, + timeout: Optional[float] = None + ) -> list[Activity]: ... + + async def close(self) -> None: ... +``` + +#### ResponseClient +Client for receiving activities from agents. + +```python +class ResponseClient: + def __init__(self, host: str = "localhost", port: int = 9873): ... + + async def pop(self) -> list[Activity]: ... + + async def __aenter__(self) -> ResponseClient: ... + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: ... +``` + +#### DataDrivenTest +Runner for YAML/JSON test definitions. + +```python +class DataDrivenTest: + def __init__(self, test_flow: dict) -> None: ... + + @property + def name(self) -> str: ... + + async def run( + self, + agent_client: AgentClient, + response_client: ResponseClient + ) -> None: ... +``` + +#### ModelAssertion +Assertion engine for validating activities. + +```python +class ModelAssertion: + def __init__( + self, + assertion: dict | Activity | None = None, + selector: Selector | None = None, + quantifier: AssertionQuantifier = AssertionQuantifier.ALL + ): ... + + def check(self, activities: list[Activity]) -> tuple[bool, Optional[str]]: ... + + def __call__(self, activities: list[Activity]) -> None: ... + + @staticmethod + def from_config(config: dict) -> ModelAssertion: ... +``` + +#### Selector +Filter activities based on criteria. + +```python +class Selector: + def __init__( + self, + selector: dict | Activity | None = None, + index: int | None = None + ): ... + + def select(self, activities: list[Activity]) -> list[Activity]: ... + + def select_first(self, activities: list[Activity]) -> Activity | None: ... + + def __call__(self, activities: list[Activity]) -> list[Activity]: ... + + @staticmethod + def from_config(config: dict) -> Selector: ... +``` + +#### AssertionQuantifier +Quantifiers for assertions. + +```python +class AssertionQuantifier(str, Enum): + ALL = "ALL" + ANY = "ANY" + ONE = "ONE" + NONE = "NONE" + + @staticmethod + def from_config(value: str) -> AssertionQuantifier: ... +``` + +#### FieldAssertionType +Types of field assertions. + +```python +class FieldAssertionType(str, Enum): + EQUALS = "EQUALS" + NOT_EQUALS = "NOT_EQUALS" + GREATER_THAN = "GREATER_THAN" + LESS_THAN = "LESS_THAN" + CONTAINS = "CONTAINS" + NOT_CONTAINS = "NOT_CONTAINS" + IN = "IN" + NOT_IN = "NOT_IN" + RE_MATCH = "RE_MATCH" +``` + +### Decorators + +#### @ddt +Load and execute data-driven tests. + +```python +def ddt(test_path: str, recursive: bool = True) -> Callable: + """ + Decorator to add data-driven tests to an integration test class. + + :param test_path: Path to test files directory + :param recursive: Load tests from subdirectories + """ +``` + +### Functions + +#### generate_token +Generate OAuth2 access token. + +```python +def generate_token(app_id: str, app_secret: str, tenant_id: str) -> str: ... +``` + +#### generate_token_from_config +Generate token from SDK config. + +```python +def generate_token_from_config(sdk_config: SDKConfig) -> str: ... +``` + +#### load_ddts +Load data-driven test files. + +```python +def load_ddts( + path: str | Path | None = None, + recursive: bool = False +) -> list[DataDrivenTest]: ... +``` + +#### populate_activity +Fill activity with default values. + +```python +def populate_activity( + activity: Activity, + defaults: dict | Activity +) -> Activity: ... +``` + +#### get_host_and_port +Parse host and port from URL. + +```python +def get_host_and_port(url: str) -> tuple[str, int]: ... +``` + +#### check_activity +Check if activity matches assertion. + +```python +def check_activity(activity: Activity, assertion: dict | Activity) -> bool: ... +``` + +#### check_activity_verbose +Check activity with detailed error information. + +```python +def check_activity_verbose( + activity: Activity, + assertion: dict | Activity +) -> tuple[bool, Optional[AssertionErrorData]]: ... +``` + +#### check_field +Check if field value matches assertion. + +```python +def check_field(value: Any, assertion: Any) -> bool: ... +``` + +#### check_field_verbose +Check field with detailed error information. + +```python +def check_field_verbose( + value: Any, + assertion: Any, + field_path: str = "" +) -> tuple[bool, Optional[AssertionErrorData]]: ... +``` + +#### assert_model +Assert activity matches, raise on failure. + +```python +def assert_model(activity: Activity, assertion: dict | Activity) -> None: ... +``` + +#### assert_field +Assert field matches, raise on failure. + +```python +def assert_field(value: Any, assertion: Any, field_path: str = "") -> None: ... +``` + +## CI/CD Integration + +### GitHub Actions + +```yaml +name: Agent Tests + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install microsoft-agents-testing pytest pytest-asyncio + + - name: Run integration tests + run: pytest tests/integration/ -v + env: + CLIENT_ID: ${{ secrets.AGENT_CLIENT_ID }} + CLIENT_SECRET: ${{ secrets.AGENT_CLIENT_SECRET }} + TENANT_ID: ${{ secrets.TENANT_ID }} + + - name: Run data-driven tests + run: pytest tests/data_driven/ -v +``` + +### Azure DevOps + +```yaml +trigger: +- main + +pool: + vmImage: 'ubuntu-latest' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '3.11' + +- script: | + pip install -r requirements.txt + pip install microsoft-agents-testing pytest pytest-asyncio + displayName: 'Install dependencies' + +- script: | + pytest tests/ -v --junitxml=test-results.xml + displayName: 'Run tests' + env: + CLIENT_ID: $(CLIENT_ID) + CLIENT_SECRET: $(CLIENT_SECRET) + TENANT_ID: $(TENANT_ID) + +- task: PublishTestResults@2 + inputs: + testResultsFiles: 'test-results.xml' + testRunTitle: 'Agent Integration Tests' +``` + +## Who Should Use This Package + +- **Agent Developers**: Testing agents built with `microsoft-agents-hosting-core` and related packages +- **QA Engineers**: Writing integration, E2E, and regression tests for conversational AI systems +- **DevOps Teams**: Automating agent validation in CI/CD pipelines +- **Sample Authors**: Creating reproducible examples and living documentation +- **Test Engineers**: Building comprehensive test suites with data-driven testing +- **Product Managers**: Writing human-readable test specifications in YAML + ## Related Packages This package complements the Microsoft 365 Agents SDK ecosystem: -- `microsoft-agents-activity`: Activity types and protocols -- `microsoft-agents-hosting-core`: Core hosting framework -- `microsoft-agents-hosting-aiohttp`: aiohttp hosting integration -- `microsoft-agents-authentication-msal`: MSAL authentication +- **`microsoft-agents-activity`**: Activity types and protocols +- **`microsoft-agents-hosting-core`**: Core hosting framework +- **`microsoft-agents-hosting-aiohttp`**: aiohttp hosting integration +- **`microsoft-agents-hosting-fastapi`**: FastAPI hosting integration +- **`microsoft-agents-hosting-teams`**: Teams-specific hosting features +- **`microsoft-agents-authentication-msal`**: MSAL authentication +- **`microsoft-agents-storage-blob`**: Azure Blob storage for agent state +- **`microsoft-agents-storage-cosmos`**: Azure Cosmos DB storage for agent state ## Contributing -This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA). For details, visit [https://cla.opensource.microsoft.com](https://cla.opensource.microsoft.com). +This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit [https://cla.opensource.microsoft.com](https://cla.opensource.microsoft.com). + +When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. ## License -MIT +MIT License + +Copyright (c) Microsoft Corporation. ## Support -For issues, questions, or contributions, please visit the [GitHub repository](https://github.com/microsoft/Agents-for-python). +For issues, questions, or contributions: +- **GitHub Issues**: [https://github.com/microsoft/Agents-for-python/issues](https://github.com/microsoft/Agents-for-python/issues) +- **Documentation**: [https://github.com/microsoft/Agents-for-python](https://github.com/microsoft/Agents-for-python) +- **Stack Overflow**: Tag your questions with `microsoft-agents-sdk` + +## Changelog + +See CHANGELOG.md for version history and release notes. diff --git a/dev/microsoft-agents-testing/_manual_test/__init__.py b/dev/microsoft-agents-testing/_manual_test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/manual_test/env.TEMPLATE b/dev/microsoft-agents-testing/_manual_test/env.TEMPLATE similarity index 100% rename from dev/microsoft-agents-testing/tests/manual_test/env.TEMPLATE rename to dev/microsoft-agents-testing/_manual_test/env.TEMPLATE diff --git a/dev/microsoft-agents-testing/tests/manual_test/main.py b/dev/microsoft-agents-testing/_manual_test/main.py similarity index 100% rename from dev/microsoft-agents-testing/tests/manual_test/main.py rename to dev/microsoft-agents-testing/_manual_test/main.py diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py index c8364fff..d2b52a63 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py @@ -1,5 +1,20 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .sdk_config import SDKConfig +from .assertions import ( + ModelAssertion, + Selector, + AssertionQuantifier, + assert_model, + assert_field, + check_model, + check_model_verbose, + check_field, + check_field_verbose, + FieldAssertionType, +) from .auth import generate_token, generate_token_from_config from .utils import populate_activity, get_host_and_port @@ -12,6 +27,8 @@ ResponseClient, AiohttpEnvironment, Integration, + ddt, + DataDrivenTest, ) __all__ = [ @@ -27,4 +44,16 @@ "Integration", "populate_activity", "get_host_and_port", + "ModelAssertion", + "Selector", + "AssertionQuantifier", + "assert_model", + "assert_field", + "check_model", + "check_model_verbose", + "check_field", + "check_field_verbose", + "FieldAssertionType", + "ddt", + "DataDrivenTest", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py new file mode 100644 index 00000000..c51c1f98 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .model_assertion import ModelAssertion +from .assertions import ( + assert_model, + assert_field, +) +from .check_model import check_model, check_model_verbose +from .check_field import check_field, check_field_verbose +from .type_defs import FieldAssertionType, AssertionQuantifier, UNSET_FIELD +from .model_selector import ModelSelector + +__all__ = [ + "ModelAssertion", + "assert_model", + "assert_field", + "check_model", + "check_model_verbose", + "check_field", + "check_field_verbose", + "FieldAssertionType", + "ModelSelector", + "AssertionQuantifier", + "UNSET_FIELD", +] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py new file mode 100644 index 00000000..04955fcd --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/assertions.py @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Any + +from microsoft_agents.activity import AgentsModel + +from .type_defs import FieldAssertionType +from .check_model import check_model_verbose +from .check_field import check_field_verbose + + +def assert_field( + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType +) -> None: + """Asserts that a specific field in the target matches the baseline. + + :param key_in_baseline: The key of the field to be tested. + :param target: The target dictionary containing the actual values. + :param assertion: The baseline dictionary containing the expected values. + """ + res, assertion_error_message = check_field_verbose( + actual_value, assertion, assertion_type + ) + assert res, assertion_error_message + + +def assert_model(model: AgentsModel | dict, assertion: AgentsModel | dict) -> None: + """Asserts that the given model matches the baseline model. + + :param model: The model to be tested. + :param assertion: The baseline model or a dictionary representing the expected model data. + """ + res, assertion_error_data = check_model_verbose(model, assertion) + assert res, str(assertion_error_data) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py new file mode 100644 index 00000000..6693f706 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_field.py @@ -0,0 +1,101 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import re +from typing import Any, Optional + +from .type_defs import FieldAssertionType, UNSET_FIELD + + +_OPERATIONS = { + FieldAssertionType.EQUALS: lambda a, b: a == b or (a is UNSET_FIELD and b is None), + FieldAssertionType.NOT_EQUALS: lambda a, b: a != b + or (a is UNSET_FIELD and b is not None), + FieldAssertionType.GREATER_THAN: lambda a, b: a > b, + FieldAssertionType.LESS_THAN: lambda a, b: a < b, + FieldAssertionType.CONTAINS: lambda a, b: b in a, + FieldAssertionType.NOT_CONTAINS: lambda a, b: b not in a, + FieldAssertionType.RE_MATCH: lambda a, b: re.match(b, a) is not None, +} + + +def _parse_assertion(field: Any) -> tuple[Any, Optional[FieldAssertionType]]: + """Parses the assertion information and returns the assertion type and baseline value. + + :param assertion_info: The assertion information to be parsed. + :return: A tuple containing the assertion type and baseline value. + """ + + assertion_type = FieldAssertionType.EQUALS + assertion = None + + if ( + isinstance(field, dict) + and "assertion_type" in field + and "assertion" in field + and field["assertion_type"] in FieldAssertionType.__members__ + ): + # format: + # {"assertion_type": "__EQ__", "assertion": "value"} + assertion_type = FieldAssertionType[field["assertion_type"]] + assertion = field.get("assertion") + + elif ( + isinstance(field, list) + and len(field) >= 2 + and isinstance(field[0], str) + and field[0] in FieldAssertionType.__members__ + ): + # format: + # ["__EQ__", "assertion"] + assertion_type = FieldAssertionType[field[0]] + assertion = field[1] + elif isinstance(field, list) or isinstance(field, dict): + assertion_type = None + else: + # default format: direct value + assertion = field + + return assertion, assertion_type + + +def check_field( + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType +) -> bool: + """Checks if the actual value satisfies the given assertion based on the assertion type. + + :param actual_value: The value to be checked. + :param assertion: The expected value or pattern to check against. + :param assertion_type: The type of assertion to perform. + :return: True if the assertion is satisfied, False otherwise. + """ + + operation = _OPERATIONS.get(assertion_type) + if not operation: + raise ValueError(f"Unsupported assertion type: {assertion_type}") + return operation(actual_value, assertion) + + +def check_field_verbose( + actual_value: Any, assertion: Any, assertion_type: FieldAssertionType +) -> tuple[bool, Optional[str]]: + """Checks if the actual value satisfies the given assertion based on the assertion type. + + :param actual_value: The value to be checked. + :param assertion: The expected value or pattern to check against. + :param assertion_type: The type of assertion to perform. + :return: A tuple containing a boolean indicating if the assertion is satisfied and an optional error message. + """ + + operation = _OPERATIONS.get(assertion_type) + if not operation: + raise ValueError(f"Unsupported assertion type: {assertion_type}") + + result = operation(actual_value, assertion) + if result: + return True, None + else: + return ( + False, + f"Assertion failed: actual value '{actual_value}' does not satisfy '{assertion_type.name}' with assertion '{assertion}'", + ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_model.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_model.py new file mode 100644 index 00000000..e88564be --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/check_model.py @@ -0,0 +1,90 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Any, Optional + +from microsoft_agents.activity import AgentsModel +from microsoft_agents.testing.utils import normalize_model_data + +from .check_field import check_field, _parse_assertion +from .type_defs import UNSET_FIELD, FieldAssertionType, AssertionErrorData + + +def _check( + actual: Any, baseline: Any, field_path: str = "" +) -> tuple[bool, Optional[AssertionErrorData]]: + """Recursively checks the actual data against the baseline data. + + :param actual: The actual data to be tested. + :param baseline: The baseline data to compare against. + :param field_path: The current field path being checked (for error reporting). + :return: A tuple containing a boolean indicating success and optional assertion error data. + """ + + assertion, assertion_type = _parse_assertion(baseline) + + if assertion_type is None: + if isinstance(baseline, dict): + for key in baseline: + new_field_path = f"{field_path}.{key}" if field_path else key + new_actual = actual.get(key, UNSET_FIELD) + new_baseline = baseline[key] + + res, assertion_error_data = _check( + new_actual, new_baseline, new_field_path + ) + if not res: + return False, assertion_error_data + return True, None + + elif isinstance(baseline, list): + for index, item in enumerate(baseline): + new_field_path = ( + f"{field_path}[{index}]" if field_path else f"[{index}]" + ) + new_actual = actual[index] if index < len(actual) else UNSET_FIELD + new_baseline = item + + res, assertion_error_data = _check( + new_actual, new_baseline, new_field_path + ) + if not res: + return False, assertion_error_data + return True, None + else: + raise ValueError("Unsupported baseline type for complex assertion.") + else: + assert isinstance(assertion_type, FieldAssertionType) + res = check_field(actual, assertion, assertion_type) + if res: + return True, None + else: + assertion_error_data = AssertionErrorData( + field_path=field_path, + actual_value=actual, + assertion=assertion, + assertion_type=assertion_type, + ) + return False, assertion_error_data + + +def check_model(actual: dict | AgentsModel, baseline: dict | AgentsModel) -> bool: + """Asserts that the given activity matches the baseline activity. + + :param activity: The activity to be tested. + :param baseline: The baseline activity or a dictionary representing the expected activity data. + """ + return check_model_verbose(actual, baseline)[0] + + +def check_model_verbose( + actual: dict | AgentsModel, baseline: dict | AgentsModel +) -> tuple[bool, Optional[AssertionErrorData]]: + """Asserts that the given activity matches the baseline activity. + + :param actual: The actual data to be tested. + :param baseline: The baseline data or a dictionary representing the expected data. + """ + actual = normalize_model_data(actual) + baseline = normalize_model_data(baseline) + return _check(actual, baseline, "model") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_assertion.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_assertion.py new file mode 100644 index 00000000..f01abdae --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_assertion.py @@ -0,0 +1,104 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from __future__ import annotations + +from typing import Optional + +from microsoft_agents.activity import AgentsModel + +from .check_model import check_model_verbose +from .model_selector import ModelSelector +from .type_defs import AssertionQuantifier, AssertionErrorData + + +class ModelAssertion: + """Class for asserting activities based on a selector and assertion criteria.""" + + _selector: ModelSelector + _quantifier: AssertionQuantifier + _assertion: dict | AgentsModel + + def __init__( + self, + assertion: dict | None = None, + selector: ModelSelector | None = None, + quantifier: AssertionQuantifier = AssertionQuantifier.ALL, + ) -> None: + """Initializes the ModelAssertion with the given configuration. + + :param config: The configuration dictionary containing quantifier, selector, and assertion. + """ + + self._assertion = assertion or {} + self._selector = selector or ModelSelector() + self._quantifier = quantifier + + @staticmethod + def _combine_assertion_errors(errors: list[AssertionErrorData]) -> str: + """Combines multiple assertion errors into a single string representation. + + :param errors: The list of assertion errors to be combined. + :return: A string representation of the combined assertion errors. + """ + return "\n".join(str(error) for error in errors) + + def check(self, items: list[dict]) -> tuple[bool, Optional[str]]: + """Asserts that the given items match the assertion criteria. + + :param items: The list of items to be tested. + :return: A tuple containing a boolean indicating if the assertion passed and an optional error message. + """ + + items = self._selector(items) + + count = 0 + for item in items: + res, assertion_error_data = check_model_verbose(item, self._assertion) + if self._quantifier == AssertionQuantifier.ALL and not res: + return ( + False, + f"Item did not match the assertion: {item}\nError: {assertion_error_data}", + ) + if self._quantifier == AssertionQuantifier.NONE and res: + return ( + False, + f"Item matched the assertion when none were expected: {item}", + ) + if res: + count += 1 + + passes = True + if self._quantifier == AssertionQuantifier.ONE and count != 1: + return ( + False, + f"Expected exactly one item to match the assertion, but found {count}.", + ) + + return passes, None + + def __call__(self, items: list[dict]) -> None: + """Allows the ModelAssertion instance to be called directly. + + :param items: The list of items to be tested. + :return: A tuple containing a boolean indicating if the assertion passed and an optional error message. + """ + passes, error = self.check(items) + assert passes, error + + @staticmethod + def from_config(config: dict) -> ModelAssertion: + """Creates a ModelAssertion instance from a configuration dictionary. + + :param config: The configuration dictionary containing quantifier, selector, and assertion. + :return: A ModelAssertion instance. + """ + assertion = config.get("assertion", {}) + selector = ModelSelector.from_config(config.get("selector", {})) + quantifier = AssertionQuantifier.from_config(config.get("quantifier", "all")) + + return ModelAssertion( + assertion=assertion, + selector=selector, + quantifier=quantifier, + ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_selector.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_selector.py new file mode 100644 index 00000000..5a2c3dca --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/model_selector.py @@ -0,0 +1,91 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from __future__ import annotations + +from .check_model import check_model + + +class ModelSelector: + """Class for selecting activities based on a model and an index.""" + + _model: dict + _index: int | None + + def __init__( + self, + model: dict | None = None, + index: int | None = None, + ) -> None: + """Initializes the ModelSelector with the given configuration. + + :param model: The model to use for selecting activities. + The model is an object holding the fields to match and assertions to pass. + :param index: The index of the item to select when quantifier is ONE. + """ + + if model is None: + model = {} + + self._model = model + self._index = index + + def select_first(self, items: list[dict]) -> dict | None: + """Selects the first item from the list of items. + + :param items: The list of items to select from. + :return: The first item, or None if no items exist. + """ + res = self.select(items) + if res: + return res[0] + return None + + def select(self, items: list[dict]) -> list[dict]: + """Selects items based on the selector configuration. + + :param items: The list of items to select from. + :return: A list of selected items. + """ + if self._index is None: + return list( + filter( + lambda item: check_model(item, self._model), + items, + ) + ) + else: + filtered_list = [] + for item in items: + if check_model(item, self._model): + filtered_list.append(item) + + if self._index < 0 and abs(self._index) <= len(filtered_list): + return [filtered_list[self._index]] + elif self._index >= 0 and self._index < len(filtered_list): + return [filtered_list[self._index]] + else: + return [] + + def __call__(self, items: list[dict]) -> list[dict]: + """Allows the Selector instance to be called as a function. + + :param items: The list of items to select from. + :return: A list of selected items. + """ + return self.select(items) + + @staticmethod + def from_config(config: dict) -> ModelSelector: + """Creates a ModelSelector instance from a configuration dictionary. + + :param config: The configuration dictionary containing selector, and index. + :return: A Selector instance. + """ + model = config.get("model", {}) + index = config.get("index", None) + + return ModelSelector( + model=model, + index=index, + ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py new file mode 100644 index 00000000..97c4be49 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/assertions/type_defs.py @@ -0,0 +1,70 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from __future__ import annotations + +from enum import Enum +from dataclasses import dataclass +from typing import Any + + +class UNSET_FIELD: + """Singleton to represent an unset field in activity comparisons.""" + + @staticmethod + def get(*args, **kwargs): + """Returns the singleton instance.""" + return UNSET_FIELD + + +class FieldAssertionType(str, Enum): + """Defines the types of assertions that can be made on fields.""" + + EQUALS = "EQUALS" + NOT_EQUALS = "NOT_EQUALS" + GREATER_THAN = "GREATER_THAN" + LESS_THAN = "LESS_THAN" + CONTAINS = "CONTAINS" + NOT_CONTAINS = "NOT_CONTAINS" + IN = "IN" + NOT_IN = "NOT_IN" + RE_MATCH = "RE_MATCH" + + +class AssertionQuantifier(str, Enum): + """Defines quantifiers for assertions on activities.""" + + ANY = "ANY" + ALL = "ALL" + ONE = "ONE" + NONE = "NONE" + + @staticmethod + def from_config(value: str) -> AssertionQuantifier: + """Creates an AssertionQuantifier from a configuration string. + + :param value: The configuration string. + :return: The corresponding AssertionQuantifier. + """ + value = value.upper() + if value not in AssertionQuantifier: + raise ValueError(f"Invalid AssertionQuantifier value: {value}") + return AssertionQuantifier(value) + + +@dataclass +class AssertionErrorData: + """Data class to hold information about assertion errors.""" + + field_path: str + actual_value: Any + assertion: Any + assertion_type: FieldAssertionType + + def __str__(self) -> str: + return ( + f"Assertion failed at '{self.field_path}': " + f"actual value '{self.actual_value}' " + f"does not satisfy assertion '{self.assertion}' " + f"of type '{self.assertion_type}'." + ) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py index 3fe2a78f..80bb0402 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .generate_token import generate_token, generate_token_from_config __all__ = ["generate_token", "generate_token_from_config"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py index 83106639..57556a73 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import requests from microsoft_agents.hosting.core import AgentAuthConfiguration @@ -42,10 +45,10 @@ def generate_token_from_config(sdk_config: SDKConfig) -> str: settings: AgentAuthConfiguration = sdk_config.get_connection() - app_id = settings.CLIENT_ID - app_secret = settings.CLIENT_SECRET + client_id = settings.CLIENT_ID + client_secret = settings.CLIENT_SECRET tenant_id = settings.TENANT_ID - if not app_id or not app_secret or not tenant_id: + if not client_id or not client_secret or not tenant_id: raise ValueError("Incorrect configuration provided for token generation.") - return generate_token(app_id, app_secret, tenant_id) + return generate_token(client_id, client_secret, tenant_id) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py index 3ad1e376..77a605ae 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .core import ( AgentClient, ApplicationRunner, @@ -7,6 +10,11 @@ Integration, Sample, ) +from .data_driven import ( + DataDrivenTest, + ddt, + load_ddts, +) __all__ = [ "AgentClient", @@ -16,4 +24,7 @@ "Environment", "Integration", "Sample", + "DataDrivenTest", + "ddt", + "load_ddts", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py index 9c69a2ae..a1161336 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .application_runner import ApplicationRunner from .aiohttp import AiohttpEnvironment from .client import ( diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py index 82d2d1d0..4625620e 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .aiohttp_environment import AiohttpEnvironment from .aiohttp_runner import AiohttpRunner diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py index 7ff83dc0..cd630697 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from aiohttp.web import Request, Response, Application from microsoft_agents.hosting.aiohttp import ( diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py index c8fe23c2..2fec48ea 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_runner.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from typing import Optional from threading import Thread, Event import asyncio @@ -66,9 +69,6 @@ async def __aenter__(self): return self async def _stop_server(self): - if not self._server_thread: - raise RuntimeError("AiohttpRunner is not running.") - try: async with ClientSession() as session: async with session.get( @@ -81,10 +81,6 @@ async def _stop_server(self): # Set shutdown event as fallback self._shutdown_event.set() - # Wait for the server thread to finish - self._server_thread.join(timeout=5.0) - self._server_thread = None - async def _shutdown_route(self, request: Request) -> Response: """Handle shutdown request by setting the shutdown event""" self._shutdown_event.set() @@ -93,17 +89,8 @@ async def _shutdown_route(self, request: Request) -> Response: async def __aexit__(self, exc_type, exc, tb): if not self._server_thread: raise RuntimeError("AiohttpRunner is not running.") - try: - async with ClientSession() as session: - async with session.get( - f"http://{self._host}:{self._port}/shutdown" - ) as response: - pass # Just trigger the shutdown - except Exception: - pass # Ignore errors during shutdown request - # Set shutdown event as fallback - self._shutdown_event.set() + await self._stop_server() # Wait for the server thread to finish self._server_thread.join(timeout=5.0) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py index ebbc56f9..9c77d745 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/application_runner.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import asyncio from abc import ABC, abstractmethod from typing import Any, Optional diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py index 1d59411e..7b778407 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/__init__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from .agent_client import AgentClient from .response_client import ResponseClient diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py index 73067207..7fdf5e79 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/agent_client.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import json import asyncio from typing import Optional, cast @@ -93,17 +96,10 @@ async def send_request(self, activity: Activity, sleep: float = 0) -> str: await self._init_client() assert self._client - if activity.conversation: - activity.conversation.id = self._cid - else: - activity.conversation = ConversationAccount( - id=self._cid or "" - ) - if self.service_url: activity.service_url = self.service_url - activity = populate_activity(activity, self._default_activity_data) + # activity = populate_activity(activity, self._default_activity_data) async with self._client.post( "api/messages", diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py index b283efdf..280195d1 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/client/response_client.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from __future__ import annotations import sys @@ -75,7 +78,11 @@ async def _handle_conversation(self, request: Request) -> Response: else: if activity.type != ActivityTypes.typing: await asyncio.sleep(0.1) # Simulate processing delay - return Response(status=200, text="Activity received") + return Response( + status=200, + content_type="application/json", + text='{"message": "Activity received"}', + ) except Exception as e: return Response(status=500, text=str(e)) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py index 0aa99f24..a351e735 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/environment.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from abc import ABC, abstractmethod from typing import Awaitable, Callable @@ -34,7 +37,7 @@ async def init_env(self, environ_config: dict) -> None: @abstractmethod def create_runner(self, *args, **kwargs) -> ApplicationRunner: """Create an application runner for the environment. - + Subclasses may accept additional arguments as needed. """ raise NotImplementedError() diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py index 3b459617..ce56da9c 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/integration.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import pytest import os @@ -28,8 +31,9 @@ class Integration: _config: dict[str, Any] = {} - _service_url: Optional[str] = None - _agent_url: Optional[str] = None + _service_url: Optional[str] = "http://localhost:9378" + _agent_url: Optional[str] = "http://localhost:3978" + _config_path: Optional[str] = "./src/tests/.env" _cid: Optional[str] = None _client_id: Optional[str] = None _tenant_id: Optional[str] = None @@ -48,6 +52,25 @@ def service_url(self) -> str: def agent_url(self) -> str: return self._agent_url or self._config.get("agent_url", "") + def setup_method(self): + if not self._config: + self._config = {} + + load_dotenv(self._config_path) + self._config.update( + { + "client_id": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", "" + ), + "tenant_id": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID", "" + ), + "client_secret": os.getenv( + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET", "" + ), + } + ) + @pytest.fixture async def environment(self): """Provides the test environment instance.""" @@ -66,31 +89,14 @@ async def sample(self, environment): assert self._sample_cls sample = self._sample_cls(environment) await sample.init_app() - # host, port = get_host_and_port(self.agent_url) - app_runner = environment.create_runner(sample.app) + host, port = get_host_and_port(self.agent_url) + app_runner = environment.create_runner(host, port) async with app_runner: yield sample else: yield None def create_agent_client(self) -> AgentClient: - if not self._config: - self._config = {} - - load_dotenv("./src/tests/.env") - self._config.update( - { - "client_id": os.getenv( - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID", "" - ), - "tenant_id": os.getenv( - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID", "" - ), - "client_secret": os.getenv( - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET", "" - ), - } - ) agent_client = AgentClient( agent_url=self.agent_url, @@ -98,6 +104,7 @@ def create_agent_client(self) -> AgentClient: client_id=self._client_id or self._config.get("client_id", ""), tenant_id=self._tenant_id or self._config.get("tenant_id", ""), client_secret=self._client_secret or self._config.get("client_secret", ""), + service_url=self.service_url, ) return agent_client diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py index 6dde3668..d97298cc 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/sample.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from abc import ABC, abstractmethod from .environment import Environment diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py new file mode 100644 index 00000000..a0ddd2e7 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .data_driven_test import DataDrivenTest +from .ddt import ddt +from .load_ddts import load_ddts + +__all__ = ["DataDrivenTest", "ddt", "load_ddts"] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py new file mode 100644 index 00000000..051042cc --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/data_driven_test.py @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License.s + +import pytest +import asyncio + +import yaml + +from copy import deepcopy + +from microsoft_agents.activity import Activity + +from microsoft_agents.testing.assertions import ModelAssertion +from microsoft_agents.testing.utils import ( + update_with_defaults, +) + +from ..core import AgentClient, ResponseClient + + +class DataDrivenTest: + """Data driven test runner.""" + + def __init__(self, test_flow: dict) -> None: + self._name: str = test_flow.get("name", "") + if not self._name: + raise ValueError("Test flow must have a 'name' field.") + self._description = test_flow.get("description", "") + + defaults = test_flow.get("defaults", {}) + self._input_defaults = defaults.get("input", {}) + self._assertion_defaults = defaults.get("assertion", {}) + self._sleep_defaults = defaults.get("sleep", {}) + + parent = test_flow.get("parent") + if parent: + parent_input_defaults = parent.get("defaults", {}).get("input", {}) + parent_sleep_defaults = parent.get("defaults", {}).get("sleep", {}) + parent_assertion_defaults = parent.get("defaults", {}).get("assertion", {}) + + update_with_defaults(self._input_defaults, parent_input_defaults) + update_with_defaults(self._sleep_defaults, parent_sleep_defaults) + update_with_defaults(self._assertion_defaults, parent_assertion_defaults) + + self._test = test_flow.get("test", []) + + @property + def name(self) -> str: + """Get the name of the data driven test.""" + return self._name + + def _load_input(self, input_data: dict) -> Activity: + defaults = deepcopy(self._input_defaults) + update_with_defaults(input_data, defaults) + return Activity.model_validate(input_data.get("activity", {})) + + def _load_assertion(self, assertion_data: dict) -> ModelAssertion: + defaults = deepcopy(self._assertion_defaults) + update_with_defaults(assertion_data, defaults) + return ModelAssertion.from_config(assertion_data) + + async def _sleep(self, sleep_data: dict) -> None: + duration = sleep_data.get("duration") + if duration is None: + duration = self._sleep_defaults.get("duration", 0) + await asyncio.sleep(duration) + + def _pre_process(self) -> None: + """Compile the data driven test to ensure all steps are valid.""" + for step in self._test: + if step.get("type") == "assertion": + if "assertion" not in step: + if "activity" in step: + step["assertion"] = step["activity"] + selector = step.get("selector") + if selector is not None: + if isinstance(selector, int): + step["selector"] = {"index": selector} + elif isinstance(selector, dict): + if "selector" not in selector: + if "activity" in selector: + selector["selector"] = selector["activity"] + + async def run( + self, agent_client: AgentClient, response_client: ResponseClient + ) -> None: + """Run the data driven test. + + :param agent_client: The agent client to send activities to. + """ + + self._pre_process() + + responses = [] + for step in self._test: + step_type = step.get("type") + if not step_type: + raise ValueError("Each step must have a 'type' field.") + + if step_type == "input": + input_activity = self._load_input(step) + if input_activity.delivery_mode == "expectReplies": + replies = await agent_client.send_expect_replies(input_activity) + responses.extend(replies) + else: + await agent_client.send_activity(input_activity) + + elif step_type == "assertion": + activity_assertion = self._load_assertion(step) + responses.extend(await response_client.pop()) + + res, err = activity_assertion.check(responses) + + if not res: + err = "Assertion failed: {}\n\n{}".format(step, err) + assert res, err + + elif step_type == "sleep": + await self._sleep(step) + + elif step_type == "breakpoint": + breakpoint() + + elif step_type == "skip": + pytest.skip("Skipping step as per test definition.") diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py new file mode 100644 index 00000000..57ae7129 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/ddt.py @@ -0,0 +1,56 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import Callable, TypeVar + +import pytest + +from microsoft_agents.testing.integration.core import Integration + +from .data_driven_test import DataDrivenTest +from .load_ddts import load_ddts + +IntegrationT = TypeVar("IntegrationT", bound=type[Integration]) + + +def _add_test_method( + test_cls: type[Integration], data_driven_test: DataDrivenTest +) -> None: + """Add a test method to the test class for the given data driven test. + + :param test_cls: The test class to add the test method to. + :param data_driven_test: The data driven test to add as a method. + """ + + test_case_name = ( + f"test_data_driven__{data_driven_test.name.replace('/', '_').replace('.', '_')}" + ) + + @pytest.mark.asyncio + async def _func(self, agent_client, response_client) -> None: + await data_driven_test.run(agent_client, response_client) + + setattr(test_cls, test_case_name, _func) + + +def ddt( + test_path: str, recursive: bool = True, prefix: str = "" +) -> Callable[[IntegrationT], IntegrationT]: + """Decorator to add data driven tests to an integration test class. + + :param test_path: The path to the data driven test files. + :param recursive: Whether to load data driven tests recursively from subdirectories. + :return: The decorated test class. + """ + + ddts = load_ddts(test_path, recursive=recursive, prefix=prefix) + if not ddts: + raise RuntimeError(f"No data driven tests found in path: {test_path}") + + def decorator(test_cls: IntegrationT) -> IntegrationT: + for data_driven_test in ddts: + # scope data_driven_test to avoid late binding in loop + _add_test_method(test_cls, data_driven_test) + return test_cls + + return decorator diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/load_ddts.py b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/load_ddts.py new file mode 100644 index 00000000..c0341a59 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/integration/data_driven/load_ddts.py @@ -0,0 +1,96 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json, yaml +from glob import glob +from pathlib import Path +from .data_driven_test import DataDrivenTest + + +def _resolve_parent(path: str, test_modules: dict) -> None: + """Resolve the parent test flow for a given test flow data. + + :param data: The test flow data. + :param tests: A dictionary of all test flows keyed by their file paths. + """ + + module = test_modules[str(path)] + parent_field = module.get("parent") + if parent_field and isinstance(parent_field, str): + # resolve a parent path reference to the data itself + parent_path = Path(path).parent / parent_field + parent_path_str = str(parent_path) + if parent_path_str not in test_modules: + raise RuntimeError("Parent module not found in tests collection.") + module["parent"] = test_modules[parent_path_str] + + +_resolve_name_seen_set = set() + + +def _resolve_name(module: dict) -> str: + """Resolve the name for a given test flow data. + + :param data: The test flow data. + :param tests: A dictionary of all test flows keyed by their file paths. + :return: The resolved name. + """ + + if id(module) in _resolve_name_seen_set: + return module.get("name", module["path"]) + _resolve_name_seen_set.add(id(module)) + + parent = module.get("parent") + if parent: + return f"{_resolve_name(parent)}.{module.get('name', module['path'])}" + else: + return module.get("name", module["path"]) + + +def load_ddts( + path: str | Path | None = None, recursive: bool = True, prefix: str = "" +) -> list[DataDrivenTest]: + """Load data driven tests from JSON and YAML files in a given path. + + :param path: The path to load test files from. If None, the current working directory is used. + :param recursive: Whether to search for test files recursively in subdirectories. + :return: A list of DataDrivenTest instances. + """ + + if not path: + path = Path.cwd() + + # collect test file paths + if recursive: + json_file_paths = glob(f"{path}/**/*.json", recursive=True) + yaml_file_paths = glob(f"{path}/**/*.yaml", recursive=True) + else: + json_file_paths = glob(f"{path}/*.json") + yaml_file_paths = glob(f"{path}/*.yaml") + + # load files + tests_json = dict() + for json_file_path in json_file_paths: + with open(json_file_path, "r", encoding="utf-8") as f: + tests_json[str(Path(json_file_path).absolute())] = json.load(f) + + tests_yaml = dict() + for yaml_file_path in yaml_file_paths: + with open(yaml_file_path, "r", encoding="utf-8") as f: + tests_yaml[str(Path(yaml_file_path).absolute())] = yaml.safe_load(f) + + test_modules = {**tests_json, **tests_yaml} + + for file_path, module in test_modules.items(): + _resolve_parent(file_path, test_modules) + module["path"] = Path(file_path).stem # store path for name resolution + for file_path, module in test_modules.items(): + module["name"] = _resolve_name(module) + if prefix: + module["name"] = f"{prefix}.{module['name']}" + + return [ + DataDrivenTest(test_flow=data) + for data in test_modules.values() + if "test" in data + ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py index c1824ae5..61e1def8 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/sdk_config.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import os from copy import deepcopy from dotenv import load_dotenv, dotenv_values diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py index 0c902992..eddb25de 100644 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/__init__.py @@ -1,7 +1,12 @@ -from .populate_activity import populate_activity -from .urls import get_host_and_port +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .populate import update_with_defaults, populate_activity +from .misc import get_host_and_port, normalize_model_data __all__ = [ + "update_with_defaults", "populate_activity", "get_host_and_port", + "normalize_model_data", ] diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py new file mode 100644 index 00000000..66771de5 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/misc.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from urllib.parse import urlparse + +from microsoft_agents.activity import AgentsModel + + +def get_host_and_port(url: str) -> tuple[str, int]: + """Extract host and port from a URL.""" + + parsed_url = urlparse(url) + host = parsed_url.hostname + port = parsed_url.port + if not host or not port: + raise ValueError(f"Invalid URL: {url}") + return host, port + + +def normalize_model_data(source: AgentsModel | dict) -> dict: + """Normalize AgentsModel data to a dictionary format.""" + + if isinstance(source, AgentsModel): + return source.model_dump(exclude_unset=True, mode="json") + return source diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py new file mode 100644 index 00000000..acec37a9 --- /dev/null +++ b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from microsoft_agents.activity import Activity + + +def update_with_defaults(original: dict, defaults: dict) -> None: + """Populate a dictionary with default values. + + :param original: The original dictionary to populate. + :param defaults: The dictionary containing default values. + """ + + for key in defaults.keys(): + if key not in original: + original[key] = defaults[key] + elif isinstance(original[key], dict) and isinstance(defaults[key], dict): + update_with_defaults(original[key], defaults[key]) + + +def populate_activity(original: Activity, defaults: Activity | dict) -> Activity: + """Populate an Activity object with default values. + + :param original: The original Activity object to populate. + :param defaults: The Activity object or dictionary containing default values. + """ + + if isinstance(defaults, Activity): + defaults = defaults.model_dump(exclude_unset=True) + + new_activity_dict = original.model_dump(exclude_unset=True) + + for key in defaults.keys(): + if key not in new_activity_dict: + new_activity_dict[key] = defaults[key] + + return Activity.model_validate(new_activity_dict) diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py deleted file mode 100644 index a6b1c19f..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/populate_activity.py +++ /dev/null @@ -1,16 +0,0 @@ -from microsoft_agents.activity import Activity - - -def populate_activity(original: Activity, defaults: Activity | dict) -> Activity: - """Populate an Activity object with default values.""" - - if isinstance(defaults, Activity): - defaults = defaults.model_dump(exclude_unset=True) - - new_activity = original.model_copy() - - for key in defaults.keys(): - if getattr(new_activity, key) is None: - setattr(new_activity, key, defaults[key]) - - return new_activity \ No newline at end of file diff --git a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py b/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py deleted file mode 100644 index d964ebd2..00000000 --- a/dev/microsoft-agents-testing/microsoft_agents/testing/utils/urls.py +++ /dev/null @@ -1,10 +0,0 @@ -from urllib.parse import urlparse - - -def get_host_and_port(url: str) -> tuple[str, int]: - """Extract host and port from a URL.""" - - parsed_url = urlparse(url) - host = parsed_url.hostname or "localhost" - port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) - return host, port diff --git a/dev/microsoft-agents-testing/pyproject.toml b/dev/microsoft-agents-testing/pyproject.toml index cf659e6f..5557ac38 100644 --- a/dev/microsoft-agents-testing/pyproject.toml +++ b/dev/microsoft-agents-testing/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "microsoft-agents-hosting-core" +name = "microsoft-agents-testing" dynamic = ["version", "dependencies"] description = "Core library for Microsoft Agents" readme = {file = "README.md", content-type = "text/markdown"} diff --git a/dev/microsoft-agents-testing/pytest.ini b/dev/microsoft-agents-testing/pytest.ini new file mode 100644 index 00000000..fee2ab83 --- /dev/null +++ b/dev/microsoft-agents-testing/pytest.ini @@ -0,0 +1,40 @@ +[pytest] +# Pytest configuration for Microsoft Agents for Python + +# Treat all warnings as errors by default +# This ensures that any code generating warnings will fail tests, +# promoting cleaner code and early detection of issues +filterwarnings = + error + # Ignore specific warnings that are not actionable or are from dependencies + ignore::DeprecationWarning:pkg_resources.* + ignore::DeprecationWarning:setuptools.* + ignore::PendingDeprecationWarning + # pytest-asyncio warnings that are safe to ignore + ignore:.*deprecated.*asyncio.*:DeprecationWarning:pytest_asyncio.* + +# Test discovery configuration +testpaths = tests +python_files = test_*.py *_test.py +python_classes = Test* +python_functions = test_* +asyncio_mode=auto + +# Output configuration +addopts = + --strict-markers + --strict-config + --verbose + --tb=short + --durations=10 + +# Minimum version requirement +minversion = 6.0 + +# Markers for test categorization +markers = + unit: Unit tests + integration: Integration tests + slow: Slow tests that may take longer to run + requires_network: Tests that require network access + requires_auth: Tests that require authentication \ No newline at end of file diff --git a/dev/microsoft-agents-testing/tests/assertions/__init__.py b/dev/microsoft-agents-testing/tests/assertions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/assertions/_common.py b/dev/microsoft-agents-testing/tests/assertions/_common.py new file mode 100644 index 00000000..83e666e4 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/_common.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest + +from microsoft_agents.activity import Activity + + +@pytest.fixture +def activity(): + return Activity(type="message", text="Hello, World!") + + +@pytest.fixture( + params=[ + Activity(type="message", text="Hello, World!"), + {"type": "message", "text": "Hello, World!"}, + ] +) +def baseline(request): + return request.param diff --git a/dev/microsoft-agents-testing/tests/assertions/test_assert_model.py b/dev/microsoft-agents-testing/tests/assertions/test_assert_model.py new file mode 100644 index 00000000..870500a0 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_assert_model.py @@ -0,0 +1,261 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from microsoft_agents.activity import Activity, Attachment +from microsoft_agents.testing.assertions import assert_model, check_model + + +class TestAssertModel: + """Tests for assert_model function.""" + + def test_assert_model_with_matching_simple_fields(self): + """Test that activity matches baseline with simple equal fields.""" + activity = Activity(type="message", text="Hello, World!") + baseline = {"type": "message", "text": "Hello, World!"} + assert_model(activity, baseline) + + def test_assert_model_with_non_matching_fields(self): + """Test that activity doesn't match baseline with different field values.""" + activity = Activity(type="message", text="Hello") + baseline = {"type": "message", "text": "Goodbye"} + assert not check_model(activity, baseline) + + def test_assert_model_with_activity_baseline(self): + """Test that baseline can be an Activity object.""" + activity = Activity(type="message", text="Hello") + baseline = Activity(type="message", text="Hello") + assert_model(activity, baseline) + + def test_assert_model_with_partial_baseline(self): + """Test that only fields in baseline are checked.""" + activity = Activity( + type="message", + text="Hello", + channel_id="test-channel", + conversation={"id": "conv123"}, + ) + baseline = {"type": "message", "text": "Hello"} + assert_model(activity, baseline) + + def test_assert_model_with_missing_field(self): + """Test that activity with missing field doesn't match baseline.""" + activity = Activity(type="message") + baseline = {"type": "message", "text": "Hello"} + assert not check_model(activity, baseline) + + def test_assert_model_with_none_values(self): + """Test that None values are handled correctly.""" + activity = Activity(type="message") + baseline = {"type": "message", "text": None} + assert_model(activity, baseline) + + def test_assert_model_with_empty_baseline(self): + """Test that empty baseline always matches.""" + activity = Activity(type="message", text="Hello") + baseline = {} + assert_model(activity, baseline) + + def test_assert_model_with_dict_assertion_format(self): + """Test using dict format for assertions.""" + activity = Activity(type="message", text="Hello, World!") + baseline = { + "type": "message", + "text": {"assertion_type": "CONTAINS", "assertion": "Hello"}, + } + assert_model(activity, baseline) + + def test_assert_model_with_list_assertion_format(self): + """Test using list format for assertions.""" + activity = Activity(type="message", text="Hello, World!") + baseline = {"type": "message", "text": ["CONTAINS", "World"]} + assert_model(activity, baseline) + + def test_assert_model_with_not_equals_assertion(self): + """Test NOT_EQUALS assertion type.""" + activity = Activity(type="message", text="Hello") + baseline = { + "type": "message", + "text": {"assertion_type": "NOT_EQUALS", "assertion": "Goodbye"}, + } + assert_model(activity, baseline) + + def test_assert_model_with_contains_assertion(self): + """Test CONTAINS assertion type.""" + activity = Activity(type="message", text="Hello, World!") + baseline = {"text": {"assertion_type": "CONTAINS", "assertion": "World"}} + assert_model(activity, baseline) + + def test_assert_model_with_not_contains_assertion(self): + """Test NOT_CONTAINS assertion type.""" + activity = Activity(type="message", text="Hello") + baseline = {"text": {"assertion_type": "NOT_CONTAINS", "assertion": "Goodbye"}} + assert_model(activity, baseline) + + def test_assert_model_with_regex_assertion(self): + """Test RE_MATCH assertion type.""" + activity = Activity(type="message", text="msg_20250112_001") + baseline = { + "text": {"assertion_type": "RE_MATCH", "assertion": r"^msg_\d{8}_\d{3}$"} + } + assert_model(activity, baseline) + + def test_assert_model_with_multiple_fields_and_mixed_assertions(self): + """Test multiple fields with different assertion types.""" + activity = Activity( + type="message", text="Hello, World!", channel_id="test-channel" + ) + baseline = { + "type": "message", + "text": ["CONTAINS", "Hello"], + "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "prod-channel"}, + } + assert_model(activity, baseline) + + def test_assert_model_fails_on_any_field_mismatch(self): + """Test that activity check fails if any field doesn't match.""" + activity = Activity(type="message", text="Hello", channel_id="test-channel") + baseline = {"type": "message", "text": "Hello", "channel_id": "prod-channel"} + assert not check_model(activity, baseline) + + def test_assert_model_with_numeric_fields(self): + """Test with numeric field values.""" + activity = Activity(type="message", locale="en-US") + activity.channel_data = {"timestamp": 1234567890} + baseline = {"type": "message", "channel_data": {"timestamp": 1234567890}} + assert_model(activity, baseline) + + def test_assert_model_with_greater_than_assertion(self): + """Test GREATER_THAN assertion on numeric fields.""" + activity = Activity(type="message") + activity.channel_data = {"count": 100} + baseline = { + "channel_data": { + "count": {"assertion_type": "GREATER_THAN", "assertion": 50} + } + } + + # This test depends on how nested dicts are handled + # If channel_data is compared as a whole dict, this might not work as expected + # Keeping this test to illustrate the concept + assert_model(activity, baseline) + + def test_assert_model_with_complex_nested_structures(self): + """Test with complex nested structures in baseline.""" + activity = Activity( + type="message", conversation={"id": "conv123", "name": "Test Conversation"} + ) + baseline = { + "type": "message", + "conversation": {"id": "conv123", "name": "Test Conversation"}, + } + assert_model(activity, baseline) + + def test_assert_model_with_boolean_fields(self): + """Test with boolean field values.""" + activity = Activity(type="message") + activity.channel_data = {"is_active": True} + baseline = {"channel_data": {"is_active": True}} + assert_model(activity, baseline) + + def test_assert_model_type_mismatch(self): + """Test that different activity types don't match.""" + activity = Activity(type="message", text="Hello") + baseline = {"type": "event", "text": "Hello"} + assert not check_model(activity, baseline) + + def test_assert_model_with_list_fields(self): + """Test with list field values.""" + activity = Activity(type="message") + activity.attachments = [Attachment(content_type="text/plain", content="test")] + baseline = { + "type": "message", + "attachments": [{"content_type": "text/plain", "content": "test"}], + } + assert_model(activity, baseline) + + +class TestAssertModelRealWorldScenarios: + """Tests simulating real-world usage scenarios.""" + + def test_validate_bot_response_message(self): + """Test validating a typical bot response.""" + activity = Activity( + type="message", + text="I found 3 results for your query.", + from_property={"id": "bot123", "name": "HelpBot"}, + ) + baseline = { + "type": "message", + "text": ["RE_MATCH", r"I found \d+ results"], + "from_property": {"id": "bot123"}, + } + assert_model(activity, baseline) + + def test_validate_user_message(self): + """Test validating a user message with flexible text matching.""" + activity = Activity( + type="message", + text="help me with something", + from_property={"id": "user456"}, + ) + baseline = { + "type": "message", + "text": {"assertion_type": "CONTAINS", "assertion": "help"}, + } + assert_model(activity, baseline) + + def test_validate_event_activity(self): + """Test validating an event activity.""" + activity = Activity( + type="event", name="conversationUpdate", value={"action": "add"} + ) + baseline = {"type": "event", "name": "conversationUpdate"} + + assert_model(activity, baseline) + + def test_partial_match_allows_extra_fields(self): + """Test that extra fields in activity don't cause failure.""" + activity = Activity( + type="message", + text="Hello", + channel_id="teams", + conversation={"id": "conv123"}, + from_property={"id": "user123"}, + timestamp="2025-01-12T10:00:00Z", + ) + baseline = {"type": "message", "text": "Hello"} + assert_model(activity, baseline) + + def test_strict_match_with_multiple_fields(self): + """Test strict matching with multiple fields specified.""" + activity = Activity(type="message", text="Hello", channel_id="teams") + baseline = {"type": "message", "text": "Hello", "channel_id": "teams"} + assert_model(activity, baseline) + + def test_flexible_text_matching_with_regex(self): + """Test flexible text matching using regex patterns.""" + activity = Activity(type="message", text="Order #12345 has been confirmed") + baseline = {"type": "message", "text": ["RE_MATCH", r"Order #\d+ has been"]} + assert_model(activity, baseline) + + def test_negative_assertions(self): + """Test using negative assertions to ensure fields don't match.""" + activity = Activity(type="message", text="Success", channel_id="teams") + baseline = { + "type": "message", + "text": {"assertion_type": "NOT_CONTAINS", "assertion": "Error"}, + "channel_id": {"assertion_type": "NOT_EQUALS", "assertion": "slack"}, + } + assert_model(activity, baseline) + + def test_combined_positive_and_negative_assertions(self): + """Test combining positive and negative assertions.""" + activity = Activity( + type="message", text="Operation completed successfully", channel_id="teams" + ) + baseline = { + "type": "message", + "text": ["CONTAINS", "completed"], + "channel_id": ["NOT_EQUALS", "slack"], + } + assert_model(activity, baseline) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_check_field.py b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py new file mode 100644 index 00000000..cafc556d --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_check_field.py @@ -0,0 +1,296 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest + +from microsoft_agents.testing.assertions.check_field import ( + check_field, + _parse_assertion, +) +from microsoft_agents.testing.assertions.type_defs import FieldAssertionType + + +class TestParseAssertion: + + @pytest.fixture( + params=[ + FieldAssertionType.EQUALS, + FieldAssertionType.NOT_EQUALS, + FieldAssertionType.GREATER_THAN, + ] + ) + def assertion_type_str(self, request): + return request.param + + @pytest.fixture(params=["simple_value", {"key": "value"}, 42]) + def assertion_value(self, request): + return request.param + + def test_parse_assertion_dict(self, assertion_value, assertion_type_str): + + assertion, assertion_type = _parse_assertion( + {"assertion_type": assertion_type_str, "assertion": assertion_value} + ) + assert assertion == assertion_value + assert assertion_type == FieldAssertionType(assertion_type_str) + + def test_parse_assertion_list(self, assertion_value, assertion_type_str): + assertion, assertion_type = _parse_assertion( + [assertion_type_str, assertion_value] + ) + assert assertion == assertion_value + assert assertion_type.value == assertion_type_str + + @pytest.mark.parametrize( + "field", + ["value", 123, 12.34], + ) + def test_parse_assertion_default(self, field): + assertion, assertion_type = _parse_assertion(field) + assert assertion == field + assert assertion_type == FieldAssertionType.EQUALS + + @pytest.mark.parametrize( + "field", + [ + {"assertion_type": FieldAssertionType.IN}, + {"assertion_type": FieldAssertionType.IN, "key": "value"}, + [FieldAssertionType.RE_MATCH], + [], + {"assertion_type": "invalid", "assertion": "test"}, + ], + ) + def test_parse_assertion_none(self, field): + assertion, assertion_type = _parse_assertion(field) + assert assertion is None + assert assertion_type is None + + +class TestCheckFieldEquals: + """Tests for EQUALS assertion type.""" + + def test_equals_with_matching_strings(self): + assert check_field("hello", "hello", FieldAssertionType.EQUALS) is True + + def test_equals_with_non_matching_strings(self): + assert check_field("hello", "world", FieldAssertionType.EQUALS) is False + + def test_equals_with_matching_integers(self): + assert check_field(42, 42, FieldAssertionType.EQUALS) is True + + def test_equals_with_non_matching_integers(self): + assert check_field(42, 43, FieldAssertionType.EQUALS) is False + + def test_equals_with_none_values(self): + assert check_field(None, None, FieldAssertionType.EQUALS) is True + + def test_equals_with_boolean_values(self): + assert check_field(True, True, FieldAssertionType.EQUALS) is True + assert check_field(False, False, FieldAssertionType.EQUALS) is True + assert check_field(True, False, FieldAssertionType.EQUALS) is False + + +class TestCheckFieldNotEquals: + """Tests for NOT_EQUALS assertion type.""" + + def test_not_equals_with_different_strings(self): + assert check_field("hello", "world", FieldAssertionType.NOT_EQUALS) is True + + def test_not_equals_with_matching_strings(self): + assert check_field("hello", "hello", FieldAssertionType.NOT_EQUALS) is False + + def test_not_equals_with_different_integers(self): + assert check_field(42, 43, FieldAssertionType.NOT_EQUALS) is True + + def test_not_equals_with_matching_integers(self): + assert check_field(42, 42, FieldAssertionType.NOT_EQUALS) is False + + +class TestCheckFieldGreaterThan: + """Tests for GREATER_THAN assertion type.""" + + def test_greater_than_with_larger_value(self): + assert check_field(10, 5, FieldAssertionType.GREATER_THAN) is True + + def test_greater_than_with_smaller_value(self): + assert check_field(5, 10, FieldAssertionType.GREATER_THAN) is False + + def test_greater_than_with_equal_value(self): + assert check_field(10, 10, FieldAssertionType.GREATER_THAN) is False + + def test_greater_than_with_floats(self): + assert check_field(10.5, 10.2, FieldAssertionType.GREATER_THAN) is True + assert check_field(10.2, 10.5, FieldAssertionType.GREATER_THAN) is False + + def test_greater_than_with_negative_numbers(self): + assert check_field(-5, -10, FieldAssertionType.GREATER_THAN) is True + assert check_field(-10, -5, FieldAssertionType.GREATER_THAN) is False + + +class TestCheckFieldLessThan: + """Tests for LESS_THAN assertion type.""" + + def test_less_than_with_smaller_value(self): + assert check_field(5, 10, FieldAssertionType.LESS_THAN) is True + + def test_less_than_with_larger_value(self): + assert check_field(10, 5, FieldAssertionType.LESS_THAN) is False + + def test_less_than_with_equal_value(self): + assert check_field(10, 10, FieldAssertionType.LESS_THAN) is False + + def test_less_than_with_floats(self): + assert check_field(10.2, 10.5, FieldAssertionType.LESS_THAN) is True + assert check_field(10.5, 10.2, FieldAssertionType.LESS_THAN) is False + + +class TestCheckFieldContains: + """Tests for CONTAINS assertion type.""" + + def test_contains_substring_in_string(self): + assert check_field("hello world", "world", FieldAssertionType.CONTAINS) is True + + def test_contains_substring_not_in_string(self): + assert check_field("hello world", "foo", FieldAssertionType.CONTAINS) is False + + def test_contains_element_in_list(self): + assert check_field([1, 2, 3, 4], 3, FieldAssertionType.CONTAINS) is True + + def test_contains_element_not_in_list(self): + assert check_field([1, 2, 3, 4], 5, FieldAssertionType.CONTAINS) is False + + def test_contains_key_in_dict(self): + assert check_field({"a": 1, "b": 2}, "a", FieldAssertionType.CONTAINS) is True + + def test_contains_key_not_in_dict(self): + assert check_field({"a": 1, "b": 2}, "c", FieldAssertionType.CONTAINS) is False + + def test_contains_empty_string(self): + assert check_field("hello", "", FieldAssertionType.CONTAINS) is True + + +class TestCheckFieldNotContains: + """Tests for NOT_CONTAINS assertion type.""" + + def test_not_contains_substring_not_in_string(self): + assert ( + check_field("hello world", "foo", FieldAssertionType.NOT_CONTAINS) is True + ) + + def test_not_contains_substring_in_string(self): + assert ( + check_field("hello world", "world", FieldAssertionType.NOT_CONTAINS) + is False + ) + + def test_not_contains_element_not_in_list(self): + assert check_field([1, 2, 3, 4], 5, FieldAssertionType.NOT_CONTAINS) is True + + def test_not_contains_element_in_list(self): + assert check_field([1, 2, 3, 4], 3, FieldAssertionType.NOT_CONTAINS) is False + + +class TestCheckFieldReMatch: + """Tests for RE_MATCH assertion type.""" + + def test_re_match_simple_pattern(self): + assert check_field("hello123", r"hello\d+", FieldAssertionType.RE_MATCH) is True + + def test_re_match_no_match(self): + assert check_field("hello", r"\d+", FieldAssertionType.RE_MATCH) is False + + def test_re_match_email_pattern(self): + pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + assert ( + check_field("test@example.com", pattern, FieldAssertionType.RE_MATCH) + is True + ) + assert ( + check_field("invalid-email", pattern, FieldAssertionType.RE_MATCH) is False + ) + + def test_re_match_anchored_pattern(self): + assert ( + check_field("hello world", r"^hello", FieldAssertionType.RE_MATCH) is True + ) + assert ( + check_field("hello world", r"^world", FieldAssertionType.RE_MATCH) is False + ) + + def test_re_match_full_string(self): + assert check_field("abc", r"^abc$", FieldAssertionType.RE_MATCH) is True + assert check_field("abcd", r"^abc$", FieldAssertionType.RE_MATCH) is False + + def test_re_match_case_sensitive(self): + assert check_field("Hello", r"hello", FieldAssertionType.RE_MATCH) is False + assert check_field("Hello", r"Hello", FieldAssertionType.RE_MATCH) is True + + +class TestCheckFieldEdgeCases: + """Tests for edge cases and error handling.""" + + def test_invalid_assertion_type(self): + # Passing an unsupported assertion type should return False + with pytest.raises(ValueError): + assert check_field("test", "test", "INVALID_TYPE") + + def test_none_actual_value_with_equals(self): + assert check_field(None, "test", FieldAssertionType.EQUALS) is False + assert check_field(None, None, FieldAssertionType.EQUALS) is True + + def test_empty_string_comparisons(self): + assert check_field("", "", FieldAssertionType.EQUALS) is True + assert check_field("", "test", FieldAssertionType.EQUALS) is False + + def test_empty_list_contains(self): + assert check_field([], "item", FieldAssertionType.CONTAINS) is False + + def test_zero_comparisons(self): + assert check_field(0, 0, FieldAssertionType.EQUALS) is True + assert check_field(0, 1, FieldAssertionType.LESS_THAN) is True + assert check_field(0, -1, FieldAssertionType.GREATER_THAN) is True + + def test_type_mismatch_comparisons(self): + # Different types should work with equality checks + assert check_field("42", 42, FieldAssertionType.EQUALS) is False + assert check_field("42", 42, FieldAssertionType.NOT_EQUALS) is True + + def test_complex_data_structures(self): + actual = {"nested": {"value": 123}} + expected = {"nested": {"value": 123}} + assert check_field(actual, expected, FieldAssertionType.EQUALS) is True + + def test_list_equality(self): + assert check_field([1, 2, 3], [1, 2, 3], FieldAssertionType.EQUALS) is True + assert check_field([1, 2, 3], [3, 2, 1], FieldAssertionType.EQUALS) is False + + +class TestCheckFieldWithRealWorldScenarios: + """Tests simulating real-world usage scenarios.""" + + def test_validate_response_status_code(self): + assert check_field(200, 200, FieldAssertionType.EQUALS) is True + assert check_field(404, 200, FieldAssertionType.NOT_EQUALS) is True + + def test_validate_response_contains_keyword(self): + response = "Success: Operation completed successfully" + assert check_field(response, "Success", FieldAssertionType.CONTAINS) is True + assert check_field(response, "Error", FieldAssertionType.NOT_CONTAINS) is True + + def test_validate_numeric_threshold(self): + temperature = 72.5 + assert check_field(temperature, 100, FieldAssertionType.LESS_THAN) is True + assert check_field(temperature, 0, FieldAssertionType.GREATER_THAN) is True + + def test_validate_message_format(self): + message_id = "msg_20250112_001" + pattern = r"^msg_\d{8}_\d{3}$" + assert check_field(message_id, pattern, FieldAssertionType.RE_MATCH) is True + + def test_validate_list_membership(self): + allowed_roles = ["admin", "user", "guest"] + assert check_field(allowed_roles, "admin", FieldAssertionType.CONTAINS) is True + assert ( + check_field(allowed_roles, "superuser", FieldAssertionType.NOT_CONTAINS) + is True + ) diff --git a/dev/microsoft-agents-testing/tests/assertions/test_integration_assertion.py b/dev/microsoft-agents-testing/tests/assertions/test_integration_assertion.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/assertions/test_model_assertion.py b/dev/microsoft-agents-testing/tests/assertions/test_model_assertion.py new file mode 100644 index 00000000..61b6b29e --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_model_assertion.py @@ -0,0 +1,626 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest + +from microsoft_agents.activity import Activity +from microsoft_agents.testing import ( + ModelAssertion, + Selector, + AssertionQuantifier, + FieldAssertionType, +) + + +class TestModelAssertionCheckWithQuantifierAll: + """Tests for check() method with ALL quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="message", text="World"), + Activity(type="event", name="test_event"), + Activity(type="message", text="Goodbye"), + ] + + def test_check_all_matching_activities(self, activities): + """Test that all matching activities pass the assertion.""" + assertion = ModelAssertion( + assertion={"type": "message"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert passes is True + assert error is None + + def test_check_all_with_one_failing_activity(self, activities): + """Test that one failing activity causes ALL assertion to fail.""" + assertion = ModelAssertion( + assertion={"text": "Hello"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert passes is False + assert error is not None + assert "Item did not match the assertion" in error + + def test_check_all_with_empty_selector(self, activities): + """Test ALL quantifier with empty selector (matches all activities).""" + assertion = ModelAssertion( + assertion={"type": "message"}, + selector=Selector(selector={}), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + # Should fail because not all activities are messages + assert passes is False + + def test_check_all_with_empty_activities(self): + """Test ALL quantifier with empty activities list.""" + assertion = ModelAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check([]) + assert passes is True + assert error is None + + def test_check_all_with_complex_assertion(self, activities): + """Test ALL quantifier with complex nested assertion.""" + complex_activities = [ + Activity(type="message", text="Hello", channelData={"id": 1}), + Activity(type="message", text="World", channelData={"id": 2}), + ] + assertion = ModelAssertion( + assertion={"type": "message"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(complex_activities) + assert passes is True + + +class TestModelAssertionCheckWithQuantifierNone: + """Tests for check() method with NONE quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="message", text="World"), + Activity(type="event", name="test_event"), + ] + + def test_check_none_with_no_matches(self, activities): + """Test NONE quantifier when no activities match.""" + assertion = ModelAssertion( + assertion={"text": "Nonexistent"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.NONE, + ) + passes, error = assertion.check(activities) + assert passes is True + assert error is None + + def test_check_none_with_one_match(self, activities): + """Test NONE quantifier fails when one activity matches.""" + assertion = ModelAssertion( + assertion={"text": "Hello"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.NONE, + ) + passes, error = assertion.check(activities) + assert passes is False + assert error is not None + assert "Item matched the assertion when none were expected" in error + + def test_check_none_with_all_matching(self, activities): + """Test NONE quantifier fails when all activities match.""" + assertion = ModelAssertion( + assertion={"type": "message"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.NONE, + ) + passes, error = assertion.check(activities) + assert passes is False + + def test_check_none_with_empty_activities(self): + """Test NONE quantifier with empty activities list.""" + assertion = ModelAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.NONE + ) + passes, error = assertion.check([]) + assert passes is True + assert error is None + + +class TestModelAssertionCheckWithQuantifierOne: + """Tests for check() method with ONE quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="First"), + Activity(type="message", text="Second"), + Activity(type="event", name="test_event"), + Activity(type="message", text="Third"), + ] + + def test_check_one_with_exactly_one_match(self, activities): + """Test ONE quantifier passes when exactly one activity matches.""" + assertion = ModelAssertion( + assertion={"text": "First"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = assertion.check(activities) + assert passes is True + assert error is None + + def test_check_one_with_no_matches(self, activities): + """Test ONE quantifier fails when no activities match.""" + assertion = ModelAssertion( + assertion={"text": "Nonexistent"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = assertion.check(activities) + assert passes is False + assert error is not None + assert "Expected exactly one item" in error + assert "found 0" in error + + def test_check_one_with_multiple_matches(self, activities): + """Test ONE quantifier fails when multiple activities match.""" + assertion = ModelAssertion( + assertion={"type": "message"}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = assertion.check(activities) + assert passes is False + assert error is not None + assert "Expected exactly one item" in error + assert "found 3" in error + + def test_check_one_with_empty_activities(self): + """Test ONE quantifier with empty activities list.""" + assertion = ModelAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ONE + ) + passes, error = assertion.check([]) + assert passes is False + assert "found 0" in error + + +class TestModelAssertionCheckWithQuantifierAny: + """Tests for check() method with ANY quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="message", text="World"), + Activity(type="event", name="test_event"), + ] + + def test_check_any_basic_functionality(self, activities): + """Test that ANY quantifier exists and can be used.""" + # ANY quantifier doesn't have special logic in the current implementation + # but should not cause errors + assertion = ModelAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ANY + ) + passes, error = assertion.check(activities) + # Based on the implementation, ANY behaves like checking if count > 0 + assert passes is True + assert error is None + + +class TestModelAssertionFromConfig: + """Tests for from_config static method.""" + + def test_from_config_minimal(self): + """Test creating assertion from minimal config.""" + config = {} + assertion = ModelAssertion.from_config(config) + assert assertion._assertion == {} + assert assertion._quantifier == AssertionQuantifier.ALL + + def test_from_config_with_assertion(self): + """Test creating assertion from config with assertion field.""" + config = {"assertion": {"type": "message", "text": "Hello"}} + assertion = ModelAssertion.from_config(config) + assert assertion._assertion == config["assertion"] + + def test_from_config_with_selector(self): + """Test creating assertion from config with selector field.""" + config = {"selector": {"selector": {"type": "message"}, "quantifier": "ALL"}} + assertion = ModelAssertion.from_config(config) + assert assertion._selector is not None + + def test_from_config_with_quantifier(self): + """Test creating assertion from config with quantifier field.""" + config = {"quantifier": "one"} + assertion = ModelAssertion.from_config(config) + assert assertion._quantifier == AssertionQuantifier.ONE + + def test_from_config_with_all_fields(self): + """Test creating assertion from config with all fields.""" + config = { + "assertion": {"type": "message"}, + "selector": { + "selector": {"text": "Hello"}, + "quantifier": "ONE", + "index": 0, + }, + "quantifier": "all", + } + assertion = ModelAssertion.from_config(config) + assert assertion._assertion == {"type": "message"} + assert assertion._quantifier == AssertionQuantifier.ALL + + def test_from_config_with_case_insensitive_quantifier(self): + """Test from_config handles case-insensitive quantifier strings.""" + for quantifier_str in ["all", "ALL", "All", "ONE", "one", "NONE", "none"]: + config = {"quantifier": quantifier_str} + assertion = ModelAssertion.from_config(config) + assert isinstance(assertion._quantifier, AssertionQuantifier) + + def test_from_config_with_complex_assertion(self): + """Test creating assertion from config with complex nested assertion.""" + config = { + "assertion": {"type": "message", "channelData": {"nested": {"value": 123}}}, + "quantifier": "all", + } + assertion = ModelAssertion.from_config(config) + assert assertion._assertion["type"] == "message" + assert assertion._assertion["channelData"]["nested"]["value"] == 123 + + +class TestModelAssertionCombineErrors: + """Tests for _combine_assertion_errors static method.""" + + def test_combine_empty_errors(self): + """Test combining empty error list.""" + result = ModelAssertion._combine_assertion_errors([]) + assert result == "" + + def test_combine_single_error(self): + """Test combining single error.""" + from microsoft_agents.testing.assertions.type_defs import ( + AssertionErrorData, + FieldAssertionType, + ) + + error = AssertionErrorData( + field_path="activity.text", + actual_value="Hello", + assertion="World", + assertion_type=FieldAssertionType.EQUALS, + ) + result = ModelAssertion._combine_assertion_errors([error]) + assert "activity.text" in result + assert "Hello" in result + + def test_combine_multiple_errors(self): + """Test combining multiple errors.""" + from microsoft_agents.testing.assertions.type_defs import ( + AssertionErrorData, + FieldAssertionType, + ) + + errors = [ + AssertionErrorData( + field_path="activity.text", + actual_value="Hello", + assertion="World", + assertion_type=FieldAssertionType.EQUALS, + ), + AssertionErrorData( + field_path="activity.type", + actual_value="message", + assertion="event", + assertion_type=FieldAssertionType.EQUALS, + ), + ] + result = ModelAssertion._combine_assertion_errors(errors) + assert "activity.text" in result + assert "activity.type" in result + assert "\n" in result + + +class TestModelAssertionIntegration: + """Integration tests with realistic scenarios.""" + + @pytest.fixture + def conversation_activities(self): + """Create a realistic conversation flow.""" + return [ + Activity(type="conversationUpdate", name="add_member"), + Activity(type="message", text="Hello bot", from_property={"id": "user1"}), + Activity(type="message", text="Hi there!", from_property={"id": "bot"}), + Activity( + type="message", text="How are you?", from_property={"id": "user1"} + ), + Activity( + type="message", text="I'm doing well!", from_property={"id": "bot"} + ), + Activity(type="typing"), + Activity(type="message", text="Goodbye", from_property={"id": "user1"}), + ] + + def test_assert_all_user_messages_have_from_property(self, conversation_activities): + """Test that all user messages have a from_property.""" + assertion = ModelAssertion( + assertion={"from_property": {"id": "user1"}}, + selector=Selector( + selector={"type": "message", "from_property": {"id": "user1"}}, + ), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(conversation_activities) + assert passes is True + + def test_assert_no_error_messages(self, conversation_activities): + """Test that there are no error messages in the conversation.""" + assertion = ModelAssertion( + assertion={"type": "error"}, + selector=Selector(selector={}), + quantifier=AssertionQuantifier.NONE, + ) + passes, error = assertion.check(conversation_activities) + assert passes is True + + def test_assert_exactly_one_conversation_update(self, conversation_activities): + """Test that there's exactly one conversation update.""" + assertion = ModelAssertion( + assertion={"type": "conversationUpdate"}, + selector=Selector(selector={"type": "conversationUpdate"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = assertion.check(conversation_activities) + assert passes is True + + def test_assert_first_message_is_greeting(self, conversation_activities): + """Test that the first message contains a greeting.""" + assertion = ModelAssertion( + assertion={"text": {"assertion_type": "CONTAINS", "assertion": "Hello"}}, + selector=Selector(selector={"type": "message"}, index=0), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(conversation_activities) + assert passes is True + + def test_complex_multi_field_assertion(self, conversation_activities): + """Test complex assertion with multiple fields.""" + assertion = ModelAssertion( + assertion={"type": "message", "from_property": {"id": "bot"}}, + selector=Selector( + selector={"type": "message", "from_property": {"id": "bot"}}, + ), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(conversation_activities) + assert passes is True + + +class TestModelAssertionEdgeCases: + """Tests for edge cases and boundary conditions.""" + + def test_empty_assertion_matches_all(self): + """Test that empty assertion matches all activities.""" + activities = [ + Activity(type="message", text="Hello"), + Activity(type="event", name="test"), + ] + assertion = ModelAssertion(assertion={}, quantifier=AssertionQuantifier.ALL) + passes, error = assertion.check(activities) + assert passes is True + + def test_assertion_with_none_values(self): + """Test assertion with None values.""" + activities = [Activity(type="message")] + assertion = ModelAssertion( + assertion={"text": None}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check(activities) + # This behavior depends on check_activity implementation + assert isinstance(passes, bool) + + def test_selector_filters_before_assertion(self): + """Test that selector filters activities before assertion check.""" + activities = [ + Activity(type="message", text="Hello"), + Activity(type="event", name="test"), + Activity(type="message", text="World"), + ] + # Selector gets only messages, assertion checks for specific text + assertion = ModelAssertion( + assertion={"text": "Hello"}, + selector=Selector(selector={"type": "message"}, index=0), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert passes is True + + def test_assertion_error_message_format(self): + """Test that error messages are properly formatted.""" + activities = [Activity(type="message", text="Wrong")] + assertion = ModelAssertion( + assertion={"text": "Expected"}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check(activities) + assert passes is False + assert error is not None + assert "Item did not match the assertion" in error + assert "Error:" in error + + def test_multiple_activities_same_content(self): + """Test handling multiple activities with identical content.""" + activities = [ + Activity(type="message", text="Hello"), + Activity(type="message", text="Hello"), + Activity(type="message", text="Hello"), + ] + assertion = ModelAssertion( + assertion={"text": "Hello"}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check(activities) + assert passes is True + + def test_assertion_with_unset_fields(self): + """Test assertion against activities with unset fields.""" + activities = [ + Activity(type="message"), # No text field set + ] + assertion = ModelAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check(activities) + assert passes is True + + +class TestModelAssertionErrorMessages: + """Tests specifically for error message content and formatting.""" + + def test_all_quantifier_error_includes_activity(self): + """Test that ALL quantifier error includes the failing activity.""" + activities = [Activity(type="message", text="Wrong")] + assertion = ModelAssertion( + assertion={"text": "Expected"}, quantifier=AssertionQuantifier.ALL + ) + passes, error = assertion.check(activities) + assert passes is False + assert "Item did not match the assertion" in error + + def test_none_quantifier_error_includes_activity(self): + """Test that NONE quantifier error includes the matching activity.""" + activities = [Activity(type="message", text="Unexpected")] + assertion = ModelAssertion( + assertion={"text": "Unexpected"}, quantifier=AssertionQuantifier.NONE + ) + passes, error = assertion.check(activities) + assert passes is False + assert "Item matched the assertion when none were expected" in error + + def test_one_quantifier_error_includes_count(self): + """Test that ONE quantifier error includes the actual count.""" + activities = [ + Activity(type="message"), + Activity(type="message"), + ] + assertion = ModelAssertion( + assertion={"type": "message"}, quantifier=AssertionQuantifier.ONE + ) + passes, error = assertion.check(activities) + assert passes is False + assert "Expected exactly one item" in error + assert "2" in error + + +class TestModelAssertionRealWorldScenarios: + """Tests simulating real-world bot testing scenarios.""" + + def test_validate_welcome_message_sent(self): + """Test that a welcome message is sent when user joins.""" + activities = [ + Activity(type="conversationUpdate", name="add_member"), + Activity(type="message", text="Welcome to our bot!"), + ] + assertion = ModelAssertion( + assertion={ + "type": "message", + "text": {"assertion_type": "CONTAINS", "assertion": "Welcome"}, + }, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert passes is True + + def test_validate_no_duplicate_responses(self): + """Test that bot doesn't send duplicate responses.""" + activities = [ + Activity(type="message", text="Response 1"), + Activity(type="message", text="Response 2"), + Activity(type="message", text="Response 3"), + ] + # Check that exactly one of each unique response exists + for response_text in ["Response 1", "Response 2", "Response 3"]: + assertion = ModelAssertion( + assertion={"text": response_text}, + selector=Selector(selector={"type": "message"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = assertion.check(activities) + assert passes is True + + def test_validate_error_handling_response(self): + """Test that bot responds appropriately to errors.""" + activities = [ + Activity(type="message", text="invalid command"), + Activity(type="message", text="I'm sorry, I didn't understand that."), + ] + assertion = ModelAssertion( + assertion={ + "text": { + "assertion_type": "RE_MATCH", + "assertion": "sorry|understand|help", + } + }, + selector=Selector(selector={"type": "message"}, index=-1), # Last message + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert not passes + assert "sorry" in error and "understand" in error and "help" in error + assert FieldAssertionType.RE_MATCH.name in error + + def test_validate_typing_indicator_before_response(self): + """Test that typing indicator is sent before response.""" + activities = [ + Activity(type="message", text="User question"), + Activity(type="typing"), + Activity(type="message", text="Bot response"), + ] + # Verify typing indicator exists + typing_assertion = ModelAssertion( + assertion={"type": "typing"}, + selector=Selector(selector={"type": "typing"}), + quantifier=AssertionQuantifier.ONE, + ) + passes, error = typing_assertion.check(activities) + assert passes is True + + def test_validate_conversation_flow_order(self): + """Test that conversation follows expected flow.""" + activities = [ + Activity(type="conversationUpdate"), + Activity(type="message", text="User: Hello"), + Activity(type="typing"), + Activity(type="message", text="Bot: Hi!"), + ] + + # Test each step individually + steps = [ + ({"type": "conversationUpdate"}, 0), + ({"type": "message"}, 1), + ({"type": "typing"}, 2), + ({"type": "message"}, 3), + ] + + for assertion_dict, expected_index in steps: + assertion = ModelAssertion( + assertion=assertion_dict, + selector=Selector(selector={}, index=expected_index), + quantifier=AssertionQuantifier.ALL, + ) + passes, error = assertion.check(activities) + assert passes is True, f"Failed at index {expected_index}: {error}" diff --git a/dev/microsoft-agents-testing/tests/assertions/test_selector.py b/dev/microsoft-agents-testing/tests/assertions/test_selector.py new file mode 100644 index 00000000..fc676639 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/assertions/test_selector.py @@ -0,0 +1,309 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest + +from microsoft_agents.activity import Activity +from microsoft_agents.testing.assertions.model_selector import Selector + + +class TestSelectorSelectWithQuantifierAll: + """Tests for select() method with ALL quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="message", text="World"), + Activity(type="event", name="test_event"), + Activity(type="message", text="Goodbye"), + ] + + def test_select_all_matching_type(self, activities): + """Test selecting all activities with matching type.""" + selector = Selector(selector={"type": "message"}) + result = selector.select(activities) + assert len(result) == 3 + assert all(a.type == "message" for a in result) + + def test_select_all_matching_multiple_fields(self, activities): + """Test selecting all activities matching multiple fields.""" + selector = Selector( + selector={"type": "message", "text": "Hello"}, + ) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Hello" + + def test_select_all_no_matches(self, activities): + """Test selecting all with no matches returns empty list.""" + selector = Selector( + selector={"type": "nonexistent"}, + ) + result = selector.select(activities) + assert len(result) == 0 + + def test_select_all_empty_selector(self, activities): + """Test selecting all with empty selector returns all activities.""" + selector = Selector(selector={}) + result = selector.select(activities) + assert len(result) == len(activities) + + def test_select_all_from_empty_list(self): + """Test selecting from empty activity list.""" + selector = Selector(selector={"type": "message"}) + result = selector.select([]) + assert len(result) == 0 + + +class TestSelectorSelectWithQuantifierOne: + """Tests for select() method with ONE quantifier.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="First"), + Activity(type="message", text="Second"), + Activity(type="event", name="test_event"), + Activity(type="message", text="Third"), + ] + + def test_select_one_default_index(self, activities): + """Test selecting one activity with default index (0).""" + selector = Selector(selector={"type": "message"}, index=0) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "First" + + def test_select_one_explicit_index(self, activities): + """Test selecting one activity with explicit index.""" + selector = Selector(selector={"type": "message"}, index=1) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Second" + + def test_select_one_last_index(self, activities): + """Test selecting one activity with last valid index.""" + selector = Selector(selector={"type": "message"}, index=2) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Third" + + def test_select_one_negative_index(self, activities): + """Test selecting one activity with negative index.""" + selector = Selector(selector={"type": "message"}, index=-1) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Third" + + def test_select_one_negative_index_from_start(self, activities): + """Test selecting one activity with negative index from start.""" + selector = Selector(selector={"type": "message"}, index=-2) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Second" + + def test_select_one_index_out_of_range(self, activities): + """Test selecting with index out of range returns empty list.""" + selector = Selector(selector={"type": "message"}, index=10) + result = selector.select(activities) + assert len(result) == 0 + + def test_select_one_negative_index_out_of_range(self, activities): + """Test selecting with negative index out of range returns empty list.""" + selector = Selector(selector={"type": "message"}, index=-10) + result = selector.select(activities) + assert len(result) == 0 + + def test_select_one_no_matches(self, activities): + """Test selecting one with no matches returns empty list.""" + selector = Selector(selector={"type": "nonexistent"}, index=0) + result = selector.select(activities) + assert len(result) == 0 + + def test_select_one_from_empty_list(self): + """Test selecting one from empty list returns empty list.""" + selector = Selector(selector={"type": "message"}, index=0) + result = selector.select([]) + assert len(result) == 0 + + +class TestSelectorSelectFirst: + """Tests for select_first() method.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="First"), + Activity(type="message", text="Second"), + Activity(type="event", name="test_event"), + ] + + def test_select_first_with_matches(self, activities): + """Test select_first returns first matching activity.""" + selector = Selector(selector={"type": "message"}) + result = selector.select_first(activities) + assert result is not None + assert result.text == "First" + + def test_select_first_no_matches(self, activities): + """Test select_first with no matches returns None.""" + selector = Selector( + selector={"type": "nonexistent"}, + ) + result = selector.select_first(activities) + assert result is None + + def test_select_first_empty_list(self): + """Test select_first on empty list returns None.""" + selector = Selector(selector={"type": "message"}) + result = selector.select_first([]) + assert result is None + + def test_select_first_with_one_quantifier(self, activities): + """Test select_first with ONE quantifier and specific index.""" + selector = Selector(selector={"type": "message"}, index=1) + result = selector.select_first(activities) + assert result is not None + assert result.text == "Second" + + +class TestSelectorCallable: + """Tests for __call__ method.""" + + @pytest.fixture + def activities(self): + """Create a list of test activities.""" + return [ + Activity(type="message", text="Hello"), + Activity(type="event", name="test_event"), + ] + + def test_call_invokes_select(self, activities): + """Test that calling selector instance invokes select().""" + selector = Selector(selector={"type": "message"}) + result = selector(activities) + assert len(result) == 1 + assert result[0].text == "Hello" + + def test_call_returns_same_as_select(self, activities): + """Test that __call__ returns same result as select().""" + selector = Selector(selector={"type": "event"}, index=0) + call_result = selector(activities) + select_result = selector.select(activities) + assert call_result == select_result + + +class TestSelectorIntegration: + """Integration tests with realistic scenarios.""" + + @pytest.fixture + def conversation_activities(self): + """Create a realistic conversation flow.""" + return [ + Activity(type="conversationUpdate", name="add_member"), + Activity(type="message", text="Hello bot", from_property={"id": "user1"}), + Activity(type="message", text="Hi there!", from_property={"id": "bot"}), + Activity( + type="message", text="How are you?", from_property={"id": "user1"} + ), + Activity( + type="message", text="I'm doing well!", from_property={"id": "bot"} + ), + Activity(type="typing"), + Activity(type="message", text="Goodbye", from_property={"id": "user1"}), + ] + + def test_select_all_user_messages(self, conversation_activities): + """Test selecting all messages from a specific user.""" + selector = Selector( + selector={"type": "message", "from_property": {"id": "user1"}}, + ) + result = selector.select(conversation_activities) + assert len(result) == 3 + + def test_select_first_bot_response(self, conversation_activities): + """Test selecting first bot response.""" + selector = Selector( + selector={"type": "message", "from_property": {"id": "bot"}}, index=0 + ) + result = selector.select(conversation_activities) + assert len(result) == 1 + assert result[0].text == "Hi there!" + + def test_select_last_message_negative_index(self, conversation_activities): + """Test selecting last message using negative index.""" + selector = Selector(selector={"type": "message"}, index=-1) + result = selector.select(conversation_activities) + assert len(result) == 1 + assert result[0].text == "Goodbye" + + def test_select_typing_indicator(self, conversation_activities): + """Test selecting typing indicator.""" + selector = Selector( + selector={"type": "typing"}, + ) + result = selector.select(conversation_activities) + assert len(result) == 1 + + def test_select_conversation_update(self, conversation_activities): + """Test selecting conversation update events.""" + selector = Selector( + selector={"type": "conversationUpdate"}, + ) + result = selector.select(conversation_activities) + assert len(result) == 1 + assert result[0].name == "add_member" + + +class TestSelectorEdgeCases: + """Tests for edge cases and boundary conditions.""" + + def test_select_with_partial_match(self): + """Test that partial matches work correctly.""" + activities = [ + Activity(type="message", text="Hello", channelData={"id": 1}), + Activity(type="message", text="World"), + ] + # Only matching on type, not text + selector = Selector(selector={"type": "message"}) + result = selector.select(activities) + assert len(result) == 2 + + def test_select_with_none_values(self): + """Test selecting activities with None values.""" + activities = [ + Activity(type="message"), + Activity(type="message", text="Hello"), + ] + selector = Selector( + selector={"type": "message", "text": None}, + ) + result = selector.select(activities) + # This depends on how check_activity handles None + assert isinstance(result, list) + + def test_select_single_activity_list(self): + """Test selecting from list with single activity.""" + activities = [Activity(type="message", text="Only one")] + selector = Selector(selector={"type": "message"}, index=0) + result = selector.select(activities) + assert len(result) == 1 + assert result[0].text == "Only one" + + def test_select_with_boundary_index_zero(self): + """Test selecting with index 0 on single item.""" + activities = [Activity(type="message", text="Single")] + selector = Selector(selector={"type": "message"}, index=0) + result = selector.select(activities) + assert len(result) == 1 + + def test_select_with_boundary_negative_one(self): + """Test selecting with index -1 on single item.""" + activities = [Activity(type="message", text="Single")] + selector = Selector(selector={"type": "message"}, index=-1) + result = selector.select(activities) + assert len(result) == 1 diff --git a/dev/microsoft-agents-testing/tests/integration/core/client/test_response_client.py b/dev/microsoft-agents-testing/tests/integration/core/client/test_response_client.py index f5d1ed6d..888adb52 100644 --- a/dev/microsoft-agents-testing/tests/integration/core/client/test_response_client.py +++ b/dev/microsoft-agents-testing/tests/integration/core/client/test_response_client.py @@ -33,7 +33,7 @@ async def test_endpoint(self, response_client): ) as resp: assert resp.status == 200 text = await resp.text() - assert text == "Activity received" + assert text == '{"message": "Activity received"}' await asyncio.sleep(0.1) # Give some time for the server to process diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/__init__.py b/dev/microsoft-agents-testing/tests/integration/data_driven/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py new file mode 100644 index 00000000..729148fc --- /dev/null +++ b/dev/microsoft-agents-testing/tests/integration/data_driven/test_data_driven_test.py @@ -0,0 +1,825 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch, call +from copy import deepcopy + +from microsoft_agents.activity import Activity +from microsoft_agents.testing.assertions import ModelAssertion +from microsoft_agents.testing.integration.core import AgentClient, ResponseClient +from microsoft_agents.testing.integration.data_driven import DataDrivenTest + + +class TestDataDrivenTestInit: + """Tests for DataDrivenTest initialization.""" + + def test_init_minimal(self): + """Test initialization with minimal required fields.""" + test_flow = {"name": "test1"} + ddt = DataDrivenTest(test_flow) + + assert ddt._name == "test1" + assert ddt._description == "" + assert ddt._input_defaults == {} + assert ddt._assertion_defaults == {} + assert ddt._sleep_defaults == {} + assert ddt._test == [] + + def test_init_with_description(self): + """Test initialization with description.""" + test_flow = {"name": "test1", "description": "Test description"} + ddt = DataDrivenTest(test_flow) + + assert ddt._name == "test1" + assert ddt._description == "Test description" + + def test_init_with_defaults(self): + """Test initialization with defaults.""" + test_flow = { + "name": "test1", + "defaults": { + "input": {"activity": {"type": "message", "locale": "en-US"}}, + "assertion": {"quantifier": "all"}, + "sleep": {"duration": 1.0}, + }, + } + ddt = DataDrivenTest(test_flow) + + assert ddt._input_defaults == { + "activity": {"type": "message", "locale": "en-US"} + } + assert ddt._assertion_defaults == {"quantifier": "all"} + assert ddt._sleep_defaults == {"duration": 1.0} + + def test_init_with_test_steps(self): + """Test initialization with test steps.""" + test_flow = { + "name": "test1", + "test": [ + {"type": "input", "activity": {"text": "Hello"}}, + {"type": "assertion", "activity": {"text": "Hi"}}, + ], + } + ddt = DataDrivenTest(test_flow) + + assert len(ddt._test) == 2 + assert ddt._test[0]["type"] == "input" + assert ddt._test[1]["type"] == "assertion" + + def test_init_with_parent_defaults(self): + """Test initialization with parent defaults.""" + parent = { + "defaults": { + "input": {"activity": {"type": "message"}}, + "assertion": {"quantifier": "one"}, + "sleep": {"duration": 0.5}, + } + } + test_flow = { + "name": "test1", + "parent": parent, + "defaults": { + "input": {"activity": {"locale": "en-US"}}, + "assertion": {"quantifier": "all"}, + }, + } + ddt = DataDrivenTest(test_flow) + + # Child defaults should override parent + assert ddt._input_defaults == { + "activity": {"type": "message", "locale": "en-US"} + } + assert ddt._assertion_defaults == {"quantifier": "all"} + assert ddt._sleep_defaults == {"duration": 0.5} + + def test_init_without_name_raises_error(self): + """Test that missing name field raises ValueError.""" + test_flow = {"description": "Test without name"} + + with pytest.raises(ValueError, match="Test flow must have a 'name' field"): + DataDrivenTest(test_flow) + + def test_init_parent_defaults_dont_mutate_original(self): + """Test that merging parent defaults doesn't mutate original dictionaries.""" + parent = { + "defaults": { + "input": {"activity": {"type": "message"}}, + } + } + test_flow = { + "name": "test1", + "parent": parent, + "defaults": { + "input": {"activity": {"locale": "en-US"}}, + }, + } + + original_parent_defaults = deepcopy(parent["defaults"]["input"]) + ddt = DataDrivenTest(test_flow) + + # Verify parent defaults weren't modified + assert parent["defaults"]["input"] == original_parent_defaults + + +class TestDataDrivenTestLoadInput: + """Tests for _load_input method.""" + + def test_load_input_basic(self): + """Test loading a basic input activity.""" + test_flow = {"name": "test1"} + ddt = DataDrivenTest(test_flow) + + input_data = {"activity": {"type": "message", "text": "Hello"}} + activity = ddt._load_input(input_data) + + assert isinstance(activity, Activity) + assert activity.type == "message" + assert activity.text == "Hello" + + def test_load_input_with_defaults(self): + """Test loading input with defaults applied.""" + test_flow = { + "name": "test1", + "defaults": {"input": {"activity": {"type": "message", "locale": "en-US"}}}, + } + ddt = DataDrivenTest(test_flow) + + input_data = {"activity": {"text": "Hello"}} + activity = ddt._load_input(input_data) + + assert activity.type == "message" + assert activity.text == "Hello" + assert activity.locale == "en-US" + + def test_load_input_override_defaults(self): + """Test that explicit input values override defaults.""" + test_flow = { + "name": "test1", + "defaults": {"input": {"activity": {"type": "message", "locale": "en-US"}}}, + } + ddt = DataDrivenTest(test_flow) + + input_data = {"activity": {"type": "event", "locale": "fr-FR"}} + activity = ddt._load_input(input_data) + + assert activity.type == "event" + assert activity.locale == "fr-FR" + + def test_load_input_empty_activity_fails(self): + """Test loading input with empty activity.""" + test_flow = {"name": "test1"} + ddt = DataDrivenTest(test_flow) + + input_data = {"activity": {}} + + with pytest.raises(Exception): + ddt._load_input(input_data) + + def test_load_input_nested_defaults(self): + """Test loading input with nested default values.""" + test_flow = { + "name": "test1", + "defaults": { + "input": {"activity": {"channelData": {"nested": {"value": 123}}}} + }, + } + ddt = DataDrivenTest(test_flow) + + input_data = {"activity": {"type": "message", "text": "Hello"}} + activity = ddt._load_input(input_data) + + assert activity.text == "Hello" + assert activity.channel_data == {"nested": {"value": 123}} + + def test_load_input_no_activity_key_raises(self): + """Test loading input when activity key is missing.""" + test_flow = {"name": "test1"} + ddt = DataDrivenTest(test_flow) + + input_data = {} + + with pytest.raises(Exception): + ddt._load_input(input_data) + + +class TestDataDrivenTestLoadAssertion: + """Tests for _load_assertion method.""" + + def test_load_assertion_basic(self): + """Test loading a basic assertion.""" + test_flow = {"name": "test1"} + ddt = DataDrivenTest(test_flow) + + assertion_data = {"activity": {"type": "message", "text": "Hello"}} + assertion = ddt._load_assertion(assertion_data) + + assert isinstance(assertion, ModelAssertion) + + def test_load_assertion_with_defaults(self): + """Test loading assertion with defaults applied.""" + test_flow = {"name": "test1", "defaults": {"assertion": {"quantifier": "one"}}} + ddt = DataDrivenTest(test_flow) + + assertion_data = {"activity": {"text": "Hello"}} + assertion = ddt._load_assertion(assertion_data) + + assert isinstance(assertion, ModelAssertion) + + def test_load_assertion_override_defaults(self): + """Test that explicit assertion values override defaults.""" + test_flow = {"name": "test1", "defaults": {"assertion": {"quantifier": "one"}}} + ddt = DataDrivenTest(test_flow) + + assertion_data = {"quantifier": "all", "activity": {"text": "Hello"}} + assertion = ddt._load_assertion(assertion_data) + + assert isinstance(assertion, ModelAssertion) + + def test_load_assertion_with_selector(self): + """Test loading assertion with selector.""" + test_flow = {"name": "test1"} + ddt = DataDrivenTest(test_flow) + + assertion_data = { + "activity": {"type": "message"}, + "selector": {"selector": {"type": "message"}}, + } + assertion = ddt._load_assertion(assertion_data) + + assert isinstance(assertion, ModelAssertion) + + def test_load_assertion_empty(self): + """Test loading empty assertion.""" + test_flow = {"name": "test1"} + ddt = DataDrivenTest(test_flow) + + assertion_data = {} + assertion = ddt._load_assertion(assertion_data) + + assert isinstance(assertion, ModelAssertion) + + +class TestDataDrivenTestSleep: + """Tests for _sleep method.""" + + @pytest.mark.asyncio + async def test_sleep_with_explicit_duration(self): + """Test sleep with explicit duration.""" + test_flow = {"name": "test1"} + ddt = DataDrivenTest(test_flow) + + sleep_data = {"duration": 0.1} + start_time = asyncio.get_event_loop().time() + await ddt._sleep(sleep_data) + elapsed = asyncio.get_event_loop().time() - start_time + + assert elapsed >= 0.1 + assert elapsed < 0.2 # Allow some margin + + @pytest.mark.asyncio + async def test_sleep_with_default_duration(self): + """Test sleep using default duration.""" + test_flow = {"name": "test1", "defaults": {"sleep": {"duration": 0.1}}} + ddt = DataDrivenTest(test_flow) + + sleep_data = {} + start_time = asyncio.get_event_loop().time() + await ddt._sleep(sleep_data) + elapsed = asyncio.get_event_loop().time() - start_time + + assert elapsed >= 0.1 + + @pytest.mark.asyncio + async def test_sleep_zero_duration(self): + """Test sleep with zero duration.""" + test_flow = {"name": "test1"} + ddt = DataDrivenTest(test_flow) + + sleep_data = {"duration": 0} + start_time = asyncio.get_event_loop().time() + await ddt._sleep(sleep_data) + elapsed = asyncio.get_event_loop().time() - start_time + + assert elapsed < 0.1 + + @pytest.mark.asyncio + async def test_sleep_no_duration_no_default(self): + """Test sleep with no duration and no default.""" + test_flow = {"name": "test1"} + ddt = DataDrivenTest(test_flow) + + sleep_data = {} + start_time = asyncio.get_event_loop().time() + await ddt._sleep(sleep_data) + elapsed = asyncio.get_event_loop().time() - start_time + + # Should default to 0 + assert elapsed < 0.1 + + @pytest.mark.asyncio + async def test_sleep_override_default(self): + """Test that explicit duration overrides default.""" + test_flow = {"name": "test1", "defaults": {"sleep": {"duration": 1.0}}} + ddt = DataDrivenTest(test_flow) + + sleep_data = {"duration": 0.05} + start_time = asyncio.get_event_loop().time() + await ddt._sleep(sleep_data) + elapsed = asyncio.get_event_loop().time() - start_time + + assert elapsed >= 0.05 + assert elapsed < 0.2 # Should not use default 1.0 + + +class TestDataDrivenTestRun: + """Tests for run method.""" + + @pytest.mark.asyncio + async def test_run_empty_test(self): + """Test running empty test.""" + test_flow = {"name": "test1", "test": []} + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[]) + + await ddt.run(agent_client, response_client) + + agent_client.send_activity.assert_not_called() + + @pytest.mark.asyncio + async def test_run_single_input(self): + """Test running test with single input.""" + test_flow = { + "name": "test1", + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}} + ], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[]) + + await ddt.run(agent_client, response_client) + + agent_client.send_activity.assert_called_once() + call_args = agent_client.send_activity.call_args[0][0] + assert isinstance(call_args, Activity) + assert call_args.text == "Hello" + + @pytest.mark.asyncio + async def test_run_input_and_assertion(self): + """Test running test with input and assertion.""" + test_flow = { + "name": "test1", + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + {"type": "assertion", "activity": {"type": "message"}}, + ], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock( + return_value=[Activity(type="message", text="Hi")] + ) + + await ddt.run(agent_client, response_client) + + agent_client.send_activity.assert_called_once() + response_client.pop.assert_called_once() + + @pytest.mark.asyncio + async def test_run_with_sleep(self): + """Test running test with sleep step.""" + test_flow = {"name": "test1", "test": [{"type": "sleep", "duration": 0.05}]} + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[]) + + start_time = asyncio.get_event_loop().time() + await ddt.run(agent_client, response_client) + elapsed = asyncio.get_event_loop().time() - start_time + + assert elapsed >= 0.05 + + @pytest.mark.asyncio + async def test_run_missing_step_type_raises_error(self): + """Test that missing step type raises ValueError.""" + test_flow = {"name": "test1", "test": [{"activity": {"text": "Hello"}}]} + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + with pytest.raises(ValueError, match="Each step must have a 'type' field"): + await ddt.run(agent_client, response_client) + + @pytest.mark.asyncio + async def test_run_multiple_steps(self): + """Test running test with multiple steps.""" + test_flow = { + "name": "test1", + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + {"type": "sleep", "duration": 0.01}, + {"type": "assertion", "activity": {"type": "message"}}, + {"type": "input", "activity": {"type": "message", "text": "Goodbye"}}, + ], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock( + return_value=[Activity(type="message", text="Hi")] + ) + + await ddt.run(agent_client, response_client) + + assert agent_client.send_activity.call_count == 2 + + @pytest.mark.asyncio + async def test_run_assertion_accumulates_responses(self): + """Test that assertion accumulates responses from previous steps.""" + test_flow = { + "name": "test1", + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + { + "type": "assertion", + "activity": {"type": "message"}, + "quantifier": "all", + }, + ], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + # Mock multiple responses + responses = [ + Activity(type="message", text="Response 1"), + Activity(type="message", text="Response 2"), + ] + response_client.pop = AsyncMock(return_value=responses) + + await ddt.run(agent_client, response_client) + + response_client.pop.assert_called_once() + + @pytest.mark.asyncio + async def test_run_assertion_fails_raises_assertion_error(self): + """Test that failing assertion raises AssertionError.""" + test_flow = { + "name": "test1", + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + {"type": "assertion", "activity": {"text": "Expected text"}}, + ], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock( + return_value=[Activity(type="message", text="Different text")] + ) + + with pytest.raises(AssertionError): + await ddt.run(agent_client, response_client) + + @pytest.mark.asyncio + async def test_run_with_defaults_applied(self): + """Test that defaults are applied during run.""" + test_flow = { + "name": "test1", + "defaults": {"input": {"activity": {"type": "message", "locale": "en-US"}}}, + "test": [{"type": "input", "activity": {"text": "Hello"}}], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + await ddt.run(agent_client, response_client) + + call_args = agent_client.send_activity.call_args[0][0] + assert call_args.type == "message" + assert call_args.text == "Hello" + assert call_args.locale == "en-US" + + @pytest.mark.asyncio + async def test_run_multiple_assertions_extend_responses(self): + """Test that multiple assertions extend the responses list.""" + test_flow = { + "name": "test1", + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}}, + {"type": "assertion", "activity": {"type": "message"}}, + {"type": "input", "activity": {"type": "message", "text": "World"}}, + {"type": "assertion", "activity": {"type": "message"}}, + ], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + # First pop returns one activity, second pop returns another + response_client.pop = AsyncMock( + side_effect=[ + [Activity(type="message", text="Response 1")], + [Activity(type="message", text="Response 2")], + ] + ) + + await ddt.run(agent_client, response_client) + + assert response_client.pop.call_count == 2 + + +class TestDataDrivenTestIntegration: + """Integration tests with realistic scenarios.""" + + @pytest.mark.asyncio + async def test_full_conversation_flow(self): + """Test a complete conversation flow.""" + test_flow = { + "name": "greeting_test", + "description": "Test greeting conversation", + "defaults": { + "input": {"activity": {"type": "message", "locale": "en-US"}}, + "assertion": {"quantifier": "all"}, + }, + "test": [ + {"type": "input", "activity": {"text": "Hello"}}, + {"type": "sleep", "duration": 0.05}, + { + "type": "assertion", + "activity": {"type": "message"}, + "selector": {"selector": {"type": "message"}}, + }, + ], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock( + return_value=[Activity(type="message", text="Hi! How can I help you?")] + ) + + await ddt.run(agent_client, response_client) + + # Verify input was sent + assert agent_client.send_activity.call_count == 1 + + # Verify assertion was checked + assert response_client.pop.call_count == 1 + + @pytest.mark.asyncio + async def test_complex_multi_turn_conversation(self): + """Test multi-turn conversation with multiple inputs and assertions.""" + test_flow = { + "name": "multi_turn_test", + "test": [ + { + "type": "input", + "activity": {"type": "message", "text": "What's the weather?"}, + }, + {"type": "assertion", "activity": {"type": "message"}}, + {"type": "sleep", "duration": 0.01}, + {"type": "input", "activity": {"type": "message", "text": "Thank you"}}, + { + "type": "assertion", + "activity": {"type": "message"}, + "quantifier": "any", + }, + ], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock( + side_effect=[ + [Activity(type="message", text="It's sunny today")], + [Activity(type="message", text="You're welcome!")], + ] + ) + + await ddt.run(agent_client, response_client) + + assert agent_client.send_activity.call_count == 2 + assert response_client.pop.call_count == 2 + + @pytest.mark.asyncio + async def test_with_parent_inheritance(self): + """Test data driven test with parent defaults inheritance.""" + parent = { + "defaults": { + "input": {"activity": {"type": "message", "locale": "en-US"}}, + "sleep": {"duration": 0.01}, + } + } + + test_flow = { + "name": "child_test", + "parent": parent, + "defaults": {"input": {"activity": {"channel_id": "test-channel"}}}, + "test": [ + {"type": "input", "activity": {"text": "Hello"}}, + {"type": "sleep"}, + {"type": "assertion", "activity": {"type": "message"}}, + ], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock( + return_value=[Activity(type="message", text="Hi")] + ) + + start_time = asyncio.get_event_loop().time() + await ddt.run(agent_client, response_client) + elapsed = asyncio.get_event_loop().time() - start_time + + # Verify inherited sleep duration was used + assert elapsed >= 0.01 + + # Verify merged defaults were applied + call_args = agent_client.send_activity.call_args[0][0] + assert call_args.type == "message" + assert call_args.locale == "en-US" + assert call_args.channel_id == "test-channel" + + +class TestDataDrivenTestEdgeCases: + """Tests for edge cases and error conditions.""" + + def test_empty_name_string_raises_error(self): + """Test that empty name string raises ValueError.""" + test_flow = {"name": ""} + + with pytest.raises(ValueError, match="Test flow must have a 'name' field"): + DataDrivenTest(test_flow) + + def test_none_name_raises_error(self): + """Test that None name raises ValueError.""" + test_flow = {"name": None} + + with pytest.raises(ValueError, match="Test flow must have a 'name' field"): + DataDrivenTest(test_flow) + + @pytest.mark.asyncio + async def test_run_unknown_step_type(self): + """Test that unknown step type is ignored (no error in current implementation).""" + test_flow = { + "name": "test1", + "test": [{"type": "unknown_type", "data": "something"}], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + + # Should complete without error (unknown types are simply skipped) + await ddt.run(agent_client, response_client) + + @pytest.mark.asyncio + async def test_run_assertion_with_no_prior_responses(self): + """Test assertion when no responses have been collected.""" + test_flow = { + "name": "test1", + "test": [{"type": "assertion", "activity": {"type": "message"}}], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock(return_value=[]) + + # Should pass because empty list matches ALL quantifier with no failures + await ddt.run(agent_client, response_client) + + def test_deep_nested_defaults(self): + """Test deeply nested default values.""" + test_flow = { + "name": "test1", + "defaults": { + "input": { + "activity": { + "channel_data": {"level1": {"level2": {"level3": "value"}}} + } + } + }, + } + ddt = DataDrivenTest(test_flow) + + assert ( + ddt._input_defaults["activity"]["channel_data"]["level1"]["level2"][ + "level3" + ] + == "value" + ) + + @pytest.mark.asyncio + async def test_load_input_preserves_original_data(self): + """Test that _load_input doesn't mutate original input data.""" + test_flow = { + "name": "test1", + "defaults": {"input": {"activity": {"type": "message"}}}, + } + ddt = DataDrivenTest(test_flow) + + original_input = {"activity": {"text": "Hello"}} + original_copy = deepcopy(original_input) + + ddt._load_input(original_input) + + # Original should be modified (update_with_defaults modifies in place) + # But let's verify the activity is still loadable + assert original_input is not None + + @pytest.mark.asyncio + async def test_run_with_special_activity_types(self): + """Test running with non-message activity types.""" + test_flow = { + "name": "test1", + "test": [ + { + "type": "input", + "activity": {"type": "event", "name": "custom_event"}, + }, + {"type": "assertion", "activity": {"type": "event"}}, + ], + } + ddt = DataDrivenTest(test_flow) + + agent_client = AsyncMock(spec=AgentClient) + response_client = AsyncMock(spec=ResponseClient) + response_client.pop = AsyncMock( + return_value=[Activity(type="event", name="response_event")] + ) + + await ddt.run(agent_client, response_client) + + call_args = agent_client.send_activity.call_args[0][0] + assert call_args.type == "event" + assert call_args.name == "custom_event" + + +class TestDataDrivenTestProperties: + """Tests for accessing test properties.""" + + def test_name_property(self): + """Test accessing the name property.""" + test_flow = {"name": "my_test"} + ddt = DataDrivenTest(test_flow) + + assert ddt._name == "my_test" + + def test_description_property(self): + """Test accessing the description property.""" + test_flow = {"name": "test1", "description": "This is a test"} + ddt = DataDrivenTest(test_flow) + + assert ddt._description == "This is a test" + + def test_defaults_properties(self): + """Test accessing defaults properties.""" + test_flow = { + "name": "test1", + "defaults": { + "input": {"activity": {"type": "message"}}, + "assertion": {"quantifier": "all"}, + "sleep": {"duration": 1.0}, + }, + } + ddt = DataDrivenTest(test_flow) + + assert ddt._input_defaults == {"activity": {"type": "message"}} + assert ddt._assertion_defaults == {"quantifier": "all"} + assert ddt._sleep_defaults == {"duration": 1.0} + + def test_test_steps_property(self): + """Test accessing test steps property.""" + test_flow = { + "name": "test1", + "test": [{"type": "input"}, {"type": "assertion"}], + } + ddt = DataDrivenTest(test_flow) + + assert len(ddt._test) == 2 + assert ddt._test[0]["type"] == "input" diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_ddt.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_ddt.py new file mode 100644 index 00000000..fe7eec0f --- /dev/null +++ b/dev/microsoft-agents-testing/tests/integration/data_driven/test_ddt.py @@ -0,0 +1,657 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +import tempfile +import json +from pathlib import Path +from unittest.mock import Mock, AsyncMock, patch, MagicMock + +from microsoft_agents.activity import Activity +from microsoft_agents.testing.integration.core import ( + Integration, + AgentClient, + ResponseClient, +) +from microsoft_agents.testing.integration.data_driven import DataDrivenTest, ddt +from microsoft_agents.testing.integration.data_driven.ddt import _add_test_method + + +class TestAddTestMethod: + """Tests for _add_test_method function.""" + + def test_add_test_method_creates_method(self): + """Test that _add_test_method creates a new test method on the class.""" + + class TestClass(Integration): + pass + + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "test_case_1" + mock_ddt.run = AsyncMock() + + _add_test_method(TestClass, mock_ddt) + + assert hasattr(TestClass, "test_data_driven__test_case_1") + method = getattr(TestClass, "test_data_driven__test_case_1") + assert callable(method) + + def test_add_test_method_replaces_slashes_in_name(self): + """Test that slashes in test name are replaced with underscores.""" + + class TestClass(Integration): + pass + + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "folder/subfolder/test_case" + mock_ddt.run = AsyncMock() + + _add_test_method(TestClass, mock_ddt) + + assert hasattr(TestClass, "test_data_driven__folder_subfolder_test_case") + assert not hasattr(TestClass, "test_data_driven__folder/subfolder/test_case") + + def test_add_test_method_replaces_dots_in_name(self): + """Test that dots in test name are replaced with underscores.""" + + class TestClass(Integration): + pass + + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "test.case.with.dots" + mock_ddt.run = AsyncMock() + + _add_test_method(TestClass, mock_ddt) + + assert hasattr(TestClass, "test_data_driven__test_case_with_dots") + + def test_add_test_method_replaces_multiple_special_chars(self): + """Test that multiple special characters are replaced.""" + + class TestClass(Integration): + pass + + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "path/to/test.case.name" + mock_ddt.run = AsyncMock() + + _add_test_method(TestClass, mock_ddt) + + assert hasattr(TestClass, "test_data_driven__path_to_test_case_name") + + @pytest.mark.asyncio + async def test_add_test_method_runs_data_driven_test(self): + """Test that the added method runs the data driven test.""" + + class TestClass(Integration): + pass + + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "test_case" + mock_ddt.run = AsyncMock() + + _add_test_method(TestClass, mock_ddt) + + test_instance = TestClass() + mock_agent_client = AsyncMock(spec=AgentClient) + mock_response_client = AsyncMock(spec=ResponseClient) + + await test_instance.test_data_driven__test_case( + mock_agent_client, mock_response_client + ) + + mock_ddt.run.assert_called_once_with(mock_agent_client, mock_response_client) + + @pytest.mark.asyncio + async def test_add_test_method_has_pytest_asyncio_mark(self): + """Test that the added method has pytest.mark.asyncio decorator.""" + + class TestClass(Integration): + pass + + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "test_case" + mock_ddt.run = AsyncMock() + + _add_test_method(TestClass, mock_ddt) + + method = getattr(TestClass, "test_data_driven__test_case") + assert hasattr(method, "pytestmark") + assert any(mark.name == "asyncio" for mark in method.pytestmark) + + def test_add_test_method_multiple_tests(self): + """Test adding multiple test methods to the same class.""" + + class TestClass(Integration): + pass + + mock_ddt1 = Mock(spec=DataDrivenTest) + mock_ddt1.name = "test_case_1" + mock_ddt1.run = AsyncMock() + + mock_ddt2 = Mock(spec=DataDrivenTest) + mock_ddt2.name = "test_case_2" + mock_ddt2.run = AsyncMock() + + _add_test_method(TestClass, mock_ddt1) + _add_test_method(TestClass, mock_ddt2) + + assert hasattr(TestClass, "test_data_driven__test_case_1") + assert hasattr(TestClass, "test_data_driven__test_case_2") + + @pytest.mark.asyncio + async def test_add_test_method_preserves_test_scope(self): + """Test that each added method maintains its own test scope.""" + + class TestClass(Integration): + pass + + mock_ddt1 = Mock(spec=DataDrivenTest) + mock_ddt1.name = "test_1" + mock_ddt1.run = AsyncMock() + + mock_ddt2 = Mock(spec=DataDrivenTest) + mock_ddt2.name = "test_2" + mock_ddt2.run = AsyncMock() + + _add_test_method(TestClass, mock_ddt1) + _add_test_method(TestClass, mock_ddt2) + + test_instance = TestClass() + mock_agent_client = AsyncMock(spec=AgentClient) + mock_response_client = AsyncMock(spec=ResponseClient) + + await test_instance.test_data_driven__test_1( + mock_agent_client, mock_response_client + ) + await test_instance.test_data_driven__test_2( + mock_agent_client, mock_response_client + ) + + # Each test should call its own run method + mock_ddt1.run.assert_called_once() + mock_ddt2.run.assert_called_once() + + def test_add_test_method_empty_name(self): + """Test adding method with empty test name.""" + + class TestClass(Integration): + pass + + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "" + mock_ddt.run = AsyncMock() + + _add_test_method(TestClass, mock_ddt) + + assert hasattr(TestClass, "test_data_driven__") + + def test_add_test_method_name_with_spaces(self): + """Test that spaces in names are preserved (converted to underscores by replace).""" + + class TestClass(Integration): + pass + + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "test with spaces" + mock_ddt.run = AsyncMock() + + _add_test_method(TestClass, mock_ddt) + + # Spaces are not replaced by the current implementation + assert hasattr(TestClass, "test_data_driven__test with spaces") + + +class TestDdtDecorator: + """Tests for ddt decorator function.""" + + def test_ddt_decorator_raises_if_no_tests(self): + """Test that ddt raises if not tests are found.""" + with pytest.raises(RuntimeError): + ddt("test_path") + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_recursive_false(self, mock_load_ddts): + """Test that ddt decorator respects recursive parameter.""" + mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] + + @ddt("test_path", recursive=False) + class TestClass(Integration): + pass + + mock_load_ddts.assert_called_once_with("test_path", recursive=False) + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_adds_test_methods(self, mock_load_ddts): + """Test that ddt decorator adds test methods for each loaded test.""" + mock_ddt1 = Mock(spec=DataDrivenTest) + mock_ddt1.name = "test_1" + mock_ddt1.run = AsyncMock() + + mock_ddt2 = Mock(spec=DataDrivenTest) + mock_ddt2.name = "test_2" + mock_ddt2.run = AsyncMock() + + mock_load_ddts.return_value = [mock_ddt1, mock_ddt2] + + @ddt("test_path") + class TestClass(Integration): + pass + + assert hasattr(TestClass, "test_data_driven__test_1") + assert hasattr(TestClass, "test_data_driven__test_2") + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_returns_same_class(self, mock_load_ddts): + """Test that ddt decorator returns the same class (modified).""" + mock_load_ddts.return_value = [DataDrivenTest({"name": "test_case"})] + + class TestClass(Integration): + pass + + decorated = ddt("test_path")(TestClass) + + assert decorated is TestClass + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_preserves_existing_methods(self, mock_load_ddts): + """Test that ddt decorator preserves existing test methods.""" + mock_load_ddts.return_value = [DataDrivenTest({"name": "new_test"})] + + @ddt("test_path") + class TestClass(Integration): + def test_existing_method(self): + pass + + assert hasattr(TestClass, "test_existing_method") + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_with_path_as_pathlib_path(self, mock_load_ddts): + """Test ddt decorator with pathlib.Path object.""" + mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] + test_path = Path("test_path") + + @ddt(str(test_path)) + class TestClass(Integration): + pass + + mock_load_ddts.assert_called_once_with(str(test_path), recursive=True) + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_multiple_classes(self, mock_load_ddts): + """Test that ddt decorator can be applied to multiple classes.""" + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "test_case" + mock_ddt.run = AsyncMock() + mock_load_ddts.return_value = [mock_ddt] + + @ddt("test_path") + class TestClass1(Integration): + pass + + @ddt("test_path") + class TestClass2(Integration): + pass + + assert hasattr(TestClass1, "test_data_driven__test_case") + assert hasattr(TestClass2, "test_data_driven__test_case") + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_with_relative_path(self, mock_load_ddts): + """Test ddt decorator with relative path.""" + mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] + + @ddt("./tests/data") + class TestClass(Integration): + pass + + mock_load_ddts.assert_called_once_with("./tests/data", recursive=True) + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_with_absolute_path(self, mock_load_ddts): + """Test ddt decorator with absolute path.""" + mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] + abs_path = Path("/absolute/path/to/tests").as_posix() + + @ddt(abs_path) + class TestClass(Integration): + pass + + mock_load_ddts.assert_called_once_with(abs_path, recursive=True) + + +class TestDdtDecoratorIntegration: + """Integration tests for ddt decorator with actual file loading.""" + + def test_ddt_decorator_loads_real_json_files(self): + """Test ddt decorator with actual JSON files.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create test file + test_data = { + "name": "real_test", + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}} + ], + } + test_file = Path(temp_dir) / "test.json" + with open(test_file, "w", encoding="utf-8") as f: + json.dump(test_data, f) + + @ddt(temp_dir, recursive=False) + class TestClass(Integration): + pass + + assert hasattr(TestClass, "test_data_driven__real_test") + + def test_ddt_decorator_loads_real_yaml_files(self): + """Test ddt decorator with actual YAML files.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create test file + yaml_content = """name: yaml_test +test: + - type: input + activity: + type: message + text: Hello +""" + test_file = Path(temp_dir) / "test.yaml" + with open(test_file, "w", encoding="utf-8") as f: + f.write(yaml_content) + + @ddt(temp_dir, recursive=False) + class TestClass(Integration): + pass + + assert hasattr(TestClass, "test_data_driven__yaml_test") + + def test_ddt_decorator_loads_multiple_files(self): + """Test ddt decorator loading multiple test files.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create multiple test files + for i in range(3): + test_data = {"name": f"test_{i}", "test": []} + test_file = Path(temp_dir) / f"test_{i}.json" + with open(test_file, "w", encoding="utf-8") as f: + json.dump(test_data, f) + + @ddt(temp_dir, recursive=False) + class TestClass(Integration): + pass + + assert hasattr(TestClass, "test_data_driven__test_0") + assert hasattr(TestClass, "test_data_driven__test_1") + assert hasattr(TestClass, "test_data_driven__test_2") + + def test_ddt_decorator_recursive_loading(self): + """Test ddt decorator with recursive directory loading.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create subdirectory + sub_dir = Path(temp_dir) / "subdir" + sub_dir.mkdir() + + # Create test in root + root_data = {"name": "root_test", "test": []} + root_file = Path(temp_dir) / "root.json" + with open(root_file, "w", encoding="utf-8") as f: + json.dump(root_data, f) + + # Create test in subdirectory + sub_data = {"name": "sub_test", "test": []} + sub_file = sub_dir / "sub.json" + with open(sub_file, "w", encoding="utf-8") as f: + json.dump(sub_data, f) + + @ddt(temp_dir, recursive=True) + class TestClass(Integration): + pass + + assert hasattr(TestClass, "test_data_driven__root_test") + assert hasattr(TestClass, "test_data_driven__sub_test") + + def test_ddt_decorator_non_recursive_skips_subdirs(self): + """Test that non-recursive mode skips subdirectories.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create subdirectory + sub_dir = Path(temp_dir) / "subdir" + sub_dir.mkdir() + + # Create test in subdirectory + sub_data = {"name": "sub_test", "test": []} + sub_file = sub_dir / "sub.json" + with open(sub_file, "w", encoding="utf-8") as f: + json.dump(sub_data, f) + + with pytest.raises(Exception): + + @ddt(temp_dir, recursive=False) + class TestClass(Integration): + pass + + @pytest.mark.asyncio + async def test_ddt_decorated_class_runs_tests(self): + """Test that decorated class can actually run the generated tests.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create test file + test_data = { + "name": "runnable_test", + "test": [ + {"type": "input", "activity": {"type": "message", "text": "Hello"}} + ], + } + test_file = Path(temp_dir) / "test.json" + with open(test_file, "w", encoding="utf-8") as f: + json.dump(test_data, f) + + @ddt(temp_dir, recursive=False) + class TestClass(Integration): + pass + + test_instance = TestClass() + mock_agent_client = AsyncMock(spec=AgentClient) + mock_response_client = AsyncMock(spec=ResponseClient) + + await test_instance.test_data_driven__runnable_test( + mock_agent_client, mock_response_client + ) + + # Verify the test ran + mock_agent_client.send_activity.assert_called_once() + + +class TestDdtDecoratorEdgeCases: + """Tests for edge cases in ddt decorator.""" + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_with_load_error(self, mock_load_ddts): + """Test ddt decorator behavior when load_ddts raises an error.""" + mock_load_ddts.side_effect = FileNotFoundError("Test files not found") + + with pytest.raises(FileNotFoundError): + + @ddt("nonexistent_path") + class TestClass(Integration): + pass + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_with_duplicate_test_names(self, mock_load_ddts): + """Test that duplicate test names overwrite previous methods.""" + mock_ddt1 = Mock(spec=DataDrivenTest) + mock_ddt1.name = "test_duplicate" + mock_ddt1.run = AsyncMock(return_value="first") + + mock_ddt2 = Mock(spec=DataDrivenTest) + mock_ddt2.name = "test_duplicate" + mock_ddt2.run = AsyncMock(return_value="second") + + mock_load_ddts.return_value = [mock_ddt1, mock_ddt2] + + @ddt("test_path") + class TestClass(Integration): + pass + + # Second test should overwrite the first + assert hasattr(TestClass, "test_data_driven__test_duplicate") + # Only one method with this name should exist + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_preserves_class_attributes(self, mock_load_ddts): + """Test that ddt decorator preserves class attributes.""" + mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] + + @ddt("test_path") + class TestClass(Integration): + class_attr = "test_value" + _service_url = "http://example.com" + + assert TestClass.class_attr == "test_value" + assert TestClass._service_url == "http://example.com" + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_preserves_class_docstring(self, mock_load_ddts): + """Test that ddt decorator preserves class docstring.""" + mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] + + @ddt("test_path") + class TestClass(Integration): + """This is a test class docstring.""" + + pass + + assert TestClass.__doc__ == "This is a test class docstring." + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_with_special_characters_in_path(self, mock_load_ddts): + """Test ddt decorator with special characters in path.""" + mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] + special_path = "test path/with spaces/and-dashes" + + @ddt(special_path) + class TestClass(Integration): + pass + + mock_load_ddts.assert_called_once_with(special_path, recursive=True) + + def test_ddt_decorator_with_test_name_collision(self): + """Test that generated test names don't collide with existing methods.""" + with tempfile.TemporaryDirectory() as temp_dir: + test_data = {"name": "existing_test", "test": []} + test_file = Path(temp_dir) / "test.json" + with open(test_file, "w", encoding="utf-8") as f: + json.dump(test_data, f) + + @ddt(temp_dir, recursive=False) + class TestClass(Integration): + def test_data_driven__existing_test(self): + """Existing method with same name.""" + return "original" + + # The decorator will overwrite the existing method + assert hasattr(TestClass, "test_data_driven__existing_test") + + +class TestDdtDecoratorWithRealIntegrationClass: + """Tests using actual Integration class features.""" + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_on_integration_subclass(self, mock_load_ddts): + """Test ddt decorator on a proper Integration subclass.""" + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "integration_test" + mock_ddt.run = AsyncMock() + mock_load_ddts.return_value = [mock_ddt] + + @ddt("test_path") + class MyIntegrationTest(Integration): + _service_url = "http://localhost:3978" + _agent_url = "http://localhost:8000" + + assert hasattr(MyIntegrationTest, "test_data_driven__integration_test") + assert MyIntegrationTest._service_url == "http://localhost:3978" + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_with_integration_fixtures(self, mock_load_ddts): + """Test that ddt-generated tests can work with Integration fixtures.""" + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "fixture_test" + mock_ddt.run = AsyncMock() + mock_load_ddts.return_value = [mock_ddt] + + @ddt("test_path") + class MyIntegrationTest(Integration): + _service_url = "http://localhost:3978" + _agent_url = "http://localhost:8000" + + # The generated method should accept agent_client and response_client parameters + import inspect + + method = getattr(MyIntegrationTest, "test_data_driven__fixture_test") + sig = inspect.signature(method) + params = list(sig.parameters.keys()) + + assert "self" in params + assert "agent_client" in params + assert "response_client" in params + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_multiple_decorators_on_same_class(self, mock_load_ddts): + """Test applying multiple ddt decorators to the same class.""" + mock_ddt1 = Mock(spec=DataDrivenTest) + mock_ddt1.name = "test_1" + mock_ddt1.run = AsyncMock() + + mock_ddt2 = Mock(spec=DataDrivenTest) + mock_ddt2.name = "test_2" + mock_ddt2.run = AsyncMock() + + mock_load_ddts.side_effect = [[mock_ddt1], [mock_ddt2]] + + @ddt("path2") + @ddt("path1") + class TestClass(Integration): + pass + + assert hasattr(TestClass, "test_data_driven__test_1") + assert hasattr(TestClass, "test_data_driven__test_2") + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_ddt_decorator_return_type(self, mock_load_ddts): + """Test that ddt decorator returns the correct type.""" + mock_load_ddts.return_value = [DataDrivenTest({"name": "test1"})] + + class TestClass(Integration): + pass + + decorated = ddt("test_path")(TestClass) + + assert isinstance(decorated, type) + assert issubclass(decorated, Integration) + + +class TestDdtDecoratorDocumentation: + """Tests related to documentation and metadata.""" + + def test_ddt_function_has_docstring(self): + """Test that ddt function has proper documentation.""" + assert ddt.__doc__ is not None + assert "data driven tests" in ddt.__doc__.lower() + + def test_add_test_method_has_docstring(self): + """Test that _add_test_method has proper documentation.""" + assert _add_test_method.__doc__ is not None + + @patch("microsoft_agents.testing.integration.data_driven.ddt.load_ddts") + def test_generated_test_methods_are_discoverable(self, mock_load_ddts): + """Test that generated test methods are discoverable by pytest.""" + mock_ddt = Mock(spec=DataDrivenTest) + mock_ddt.name = "discoverable_test" + mock_ddt.run = AsyncMock() + mock_load_ddts.return_value = [mock_ddt] + + @ddt("test_path") + class TestClass(Integration): + pass + + # Check that the method name starts with 'test_' so pytest can discover it + method_name = "test_data_driven__discoverable_test" + assert hasattr(TestClass, method_name) + assert method_name.startswith("test_") diff --git a/dev/microsoft-agents-testing/tests/integration/data_driven/test_load_ddts.py b/dev/microsoft-agents-testing/tests/integration/data_driven/test_load_ddts.py new file mode 100644 index 00000000..75c28686 --- /dev/null +++ b/dev/microsoft-agents-testing/tests/integration/data_driven/test_load_ddts.py @@ -0,0 +1,362 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +import pytest +import tempfile +import os +from pathlib import Path + +from microsoft_agents.testing.integration.data_driven import DataDrivenTest +from microsoft_agents.testing.integration.data_driven.load_ddts import load_ddts + + +class TestLoadDdts: + """Tests for load_ddts function.""" + + def test_load_ddts_from_empty_directory(self): + """Test loading from an empty directory returns empty list.""" + with tempfile.TemporaryDirectory() as temp_dir: + result = load_ddts(temp_dir, recursive=False) + assert result == [] + + def test_load_single_json_file(self): + """Test loading a single JSON test file.""" + with tempfile.TemporaryDirectory() as temp_dir: + test_data = { + "name": "test1", + "description": "Test 1", + "test": [{"type": "input", "activity": {"text": "Hello"}}], + } + + json_file = Path(temp_dir) / "test1.json" + with open(json_file, "w", encoding="utf-8") as f: + json.dump(test_data, f) + + result = load_ddts(temp_dir, recursive=False) + + assert len(result) == 1 + assert isinstance(result[0], DataDrivenTest) + assert result[0]._name == "test1" + + def test_load_single_yaml_file(self): + """Test loading a single YAML test file.""" + with tempfile.TemporaryDirectory() as temp_dir: + yaml_content = """name: test1 +description: Test 1 +test: + - type: input + activity: + text: Hello +""" + + yaml_file = Path(temp_dir) / "test1.yaml" + with open(yaml_file, "w", encoding="utf-8") as f: + f.write(yaml_content) + + result = load_ddts(temp_dir, recursive=False) + + assert len(result) == 1 + assert isinstance(result[0], DataDrivenTest) + assert result[0]._name == "test1" + + def test_load_multiple_files(self): + """Test loading multiple test files.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create JSON file + json_data = { + "name": "json_test", + "test": [{"type": "input", "activity": {"text": "Hello"}}], + } + json_file = Path(temp_dir) / "test1.json" + with open(json_file, "w", encoding="utf-8") as f: + json.dump(json_data, f) + + # Create YAML file + yaml_content = """name: yaml_test +test: + - type: input + activity: + text: World +""" + yaml_file = Path(temp_dir) / "test2.yaml" + with open(yaml_file, "w", encoding="utf-8") as f: + f.write(yaml_content) + + result = load_ddts(temp_dir, recursive=False) + + assert len(result) == 2 + names = {test._name for test in result} + assert "json_test" in names + assert "yaml_test" in names + + def test_load_recursive(self): + """Test loading files recursively from subdirectories.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create subdirectory + sub_dir = Path(temp_dir) / "subdir" + sub_dir.mkdir() + + # Create file in root + root_data = {"name": "root_test", "test": []} + root_file = Path(temp_dir) / "root.json" + with open(root_file, "w", encoding="utf-8") as f: + json.dump(root_data, f) + + # Create file in subdirectory + sub_data = {"name": "sub_test", "test": []} + sub_file = sub_dir / "sub.json" + with open(sub_file, "w", encoding="utf-8") as f: + json.dump(sub_data, f) + + # Non-recursive should find only root file + result_non_recursive = load_ddts(temp_dir, recursive=False) + assert len(result_non_recursive) == 1 + assert result_non_recursive[0]._name == "root_test" + + # Recursive should find both files + result_recursive = load_ddts(temp_dir, recursive=True) + assert len(result_recursive) == 2 + names = {test._name for test in result_recursive} + assert "root_test" in names + assert "sub_test" in names + + def test_load_with_parent_reference(self): + """Test loading files with parent references.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create parent file + parent_data = { + "name": "parent", + "defaults": { + "input": {"activity": {"type": "message", "locale": "en-US"}} + }, + } + parent_file = Path(temp_dir) / "parent.json" + with open(parent_file, "w", encoding="utf-8") as f: + json.dump(parent_data, f) + + # Create child file with parent reference + child_data = { + "name": "child", + "parent": str(parent_file), + "test": [{"type": "input", "activity": {"text": "Hello"}}], + } + child_file = Path(temp_dir) / "child.json" + with open(child_file, "w", encoding="utf-8") as f: + json.dump(child_data, f) + + result = load_ddts(temp_dir, recursive=False) + + # Should load both files + assert len(result) == 1 + + # Find the child test + child_test = next((t for t in result if t._name == "parent.child"), None) + assert child_test is not None + + # Child should have inherited defaults from parent + assert child_test._input_defaults == { + "activity": {"type": "message", "locale": "en-US"} + } + + def test_load_with_relative_parent_reference(self): + """Test loading files with relative parent references.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create parent file + parent_data = { + "name": "parent", + "defaults": {"input": {"activity": {"type": "message"}}}, + } + parent_file = Path(temp_dir) / "parent.yaml" + with open(parent_file, "w", encoding="utf-8") as f: + f.write( + "name: parent\ndefaults:\n input:\n activity:\n type: message\n" + ) + + # Create child file with relative parent reference + child_data = {"name": "child", "parent": "parent.yaml", "test": []} + child_file = Path(temp_dir) / "child.json" + with open(child_file, "w", encoding="utf-8") as f: + json.dump(child_data, f) + + # Change to temp_dir so relative path works + original_dir = os.getcwd() + try: + os.chdir(temp_dir) + result = load_ddts(temp_dir, recursive=False) + + assert len(result) == 1 + child_test = next( + (t for t in result if t._name == "parent.child"), None + ) + assert child_test is not None + finally: + os.chdir(original_dir) + + def test_load_with_nested_parent_references(self): + """Test loading files with nested parent references (grandparent -> parent -> child).""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create grandparent file + grandparent_data = { + "name": "grandparent", + "defaults": {"input": {"activity": {"type": "message"}}}, + } + grandparent_file = Path(temp_dir) / "grandparent.json" + with open(grandparent_file, "w", encoding="utf-8") as f: + json.dump(grandparent_data, f) + + # Create parent file referencing grandparent + parent_data = { + "name": "parent", + "parent": str(grandparent_file), + "defaults": {"input": {"activity": {"locale": "en-US"}}}, + } + parent_file = Path(temp_dir) / "parent.json" + with open(parent_file, "w", encoding="utf-8") as f: + json.dump(parent_data, f) + + # Create child file referencing parent + child_data = {"name": "child", "parent": str(parent_file), "test": []} + child_file = Path(temp_dir) / "child.json" + with open(child_file, "w", encoding="utf-8") as f: + json.dump(child_data, f) + + result = load_ddts(temp_dir, recursive=False) + + # Should load all three files + assert len(result) == 1 + + # Verify child has inherited all defaults + child_test = next( + (t for t in result if t._name == "grandparent.parent.child"), None + ) + assert child_test is not None + + def test_load_with_missing_parent_raises_error(self): + """Test that referencing a non-existent parent file raises an error.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create child file with non-existent parent reference + child_data = { + "name": "child", + "parent": str(Path(temp_dir) / "nonexistent.json"), + "test": [], + } + child_file = Path(temp_dir) / "child.json" + with open(child_file, "w", encoding="utf-8") as f: + json.dump(child_data, f) + + with pytest.raises(Exception): + load_ddts(temp_dir, recursive=False) + + def test_load_sets_name_from_filename_when_missing(self): + """Test that name is set from filename when not provided in test data.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create file without name field + test_data = {"test": [{"type": "input", "activity": {"text": "Hello"}}]} + test_file = Path(temp_dir) / "my_test_file.json" + with open(test_file, "w", encoding="utf-8") as f: + json.dump(test_data, f) + + result = load_ddts(temp_dir, recursive=False) + + assert len(result) == 1 + assert result[0]._name == "my_test_file" + + def test_load_uses_current_working_directory_when_path_is_none(self): + """Test that load_ddts uses current working directory when path is None.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create test file + test_data = {"name": "test", "test": []} + test_file = Path(temp_dir) / "test.json" + with open(test_file, "w", encoding="utf-8") as f: + json.dump(test_data, f) + + # Change to temp_dir and load without path + original_dir = os.getcwd() + try: + os.chdir(temp_dir) + result = load_ddts(None, recursive=False) + + assert len(result) == 1 + assert result[0]._name == "test" + finally: + os.chdir(original_dir) + + def test_load_resolves_parent_to_absolute_path(self): + """Test that parent references are resolved to absolute paths.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create parent file + parent_data = { + "name": "parent", + "defaults": {"input": {"activity": {"type": "message"}}}, + } + parent_file = Path(temp_dir) / "parent.json" + with open(parent_file, "w", encoding="utf-8") as f: + json.dump(parent_data, f) + + # Create child with parent reference + child_data = {"name": "child", "parent": str(parent_file), "test": []} + child_file = Path(temp_dir) / "child.json" + with open(child_file, "w", encoding="utf-8") as f: + json.dump(child_data, f) + + result = load_ddts(temp_dir, recursive=False) + + # Find child and verify parent is a dict (resolved) + child_test = next((t for t in result if t._name == "parent.child"), None) + assert child_test is not None + + def test_load_handles_mixed_json_and_yaml_files(self): + """Test loading both JSON and YAML files together.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create JSON parent + parent_data = { + "name": "json_parent", + "defaults": {"input": {"activity": {"type": "message"}}}, + } + parent_file = Path(temp_dir) / "parent.json" + with open(parent_file, "w", encoding="utf-8") as f: + json.dump(parent_data, f) + + # Create YAML child referencing JSON parent + yaml_content = f"""name: yaml_child +parent: {parent_file} +test: [] +""" + child_file = Path(temp_dir) / "child.yaml" + with open(child_file, "w", encoding="utf-8") as f: + f.write(yaml_content) + + result = load_ddts(temp_dir, recursive=False) + + assert len(result) == 1 + names = {test._name for test in result} + assert "json_parent.yaml_child" in names + + def test_load_with_path_as_string(self): + """Test that path parameter accepts string type.""" + with tempfile.TemporaryDirectory() as temp_dir: + test_data = {"name": "test", "test": []} + test_file = Path(temp_dir) / "test.json" + with open(test_file, "w", encoding="utf-8") as f: + json.dump(test_data, f) + + # Pass path as string instead of Path object + result = load_ddts(str(temp_dir), recursive=False) + + assert len(result) == 1 + assert result[0]._name == "test" + + def test_load_with_path_as_path_object(self): + """Test that path parameter accepts Path object.""" + with tempfile.TemporaryDirectory() as temp_dir: + test_data = {"name": "test", "test": []} + test_file = Path(temp_dir) / "test.json" + with open(test_file, "w", encoding="utf-8") as f: + json.dump(test_data, f) + + # Pass path as Path object + result = load_ddts(Path(temp_dir), recursive=False) + + assert len(result) == 1 + assert result[0]._name == "test" diff --git a/dev/microsoft-agents-testing/tests/utils/__init__.py b/dev/microsoft-agents-testing/tests/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dev/microsoft-agents-testing/tests/utils/test_populate.py b/dev/microsoft-agents-testing/tests/utils/test_populate.py new file mode 100644 index 00000000..07b99eab --- /dev/null +++ b/dev/microsoft-agents-testing/tests/utils/test_populate.py @@ -0,0 +1,358 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +from microsoft_agents.activity import Activity, ChannelAccount, ConversationAccount + +from microsoft_agents.testing.utils.populate import ( + update_with_defaults, + populate_activity, +) + + +class TestUpdateWithDefaults: + """Tests for the update_with_defaults function.""" + + def test_update_with_defaults_with_empty_original(self): + """Test that defaults are added to an empty dictionary.""" + original = {} + defaults = {"key1": "value1", "key2": "value2"} + update_with_defaults(original, defaults) + assert original == {"key1": "value1", "key2": "value2"} + + def test_update_with_defaults_with_empty_defaults(self): + """Test that original dictionary is unchanged when defaults is empty.""" + original = {"key1": "value1"} + defaults = {} + update_with_defaults(original, defaults) + assert original == {"key1": "value1"} + + def test_update_with_defaults_with_non_overlapping_keys(self): + """Test that defaults are added when keys don't overlap.""" + original = {"key1": "value1"} + defaults = {"key2": "value2", "key3": "value3"} + update_with_defaults(original, defaults) + assert original == {"key1": "value1", "key2": "value2", "key3": "value3"} + + def test_update_with_defaults_preserves_existing_values(self): + """Test that existing values in original are not overwritten.""" + original = {"key1": "original_value", "key2": "value2"} + defaults = {"key1": "default_value", "key3": "value3"} + update_with_defaults(original, defaults) + assert original == { + "key1": "original_value", + "key2": "value2", + "key3": "value3", + } + + def test_update_with_defaults_with_nested_dicts(self): + """Test that nested dictionaries are recursively updated.""" + original = {"nested": {"key1": "original"}} + defaults = {"nested": {"key1": "default", "key2": "value2"}} + update_with_defaults(original, defaults) + assert original == {"nested": {"key1": "original", "key2": "value2"}} + + def test_update_with_defaults_with_deeply_nested_dicts(self): + """Test recursive update with deeply nested structures.""" + original = {"level1": {"level2": {"key1": "original"}}} + defaults = { + "level1": { + "level2": {"key1": "default", "key2": "value2"}, + "level2b": {"key3": "value3"}, + } + } + update_with_defaults(original, defaults) + assert original == { + "level1": { + "level2": {"key1": "original", "key2": "value2"}, + "level2b": {"key3": "value3"}, + } + } + + def test_update_with_defaults_adds_nested_dict_when_missing(self): + """Test that nested dicts are added when they don't exist in original.""" + original = {"key1": "value1"} + defaults = {"nested": {"key2": "value2"}} + update_with_defaults(original, defaults) + assert original == {"key1": "value1", "nested": {"key2": "value2"}} + + def test_update_with_defaults_with_mixed_types(self): + """Test with various value types: strings, numbers, booleans, lists.""" + original = {"str": "text", "num": 42} + defaults = { + "str": "default_text", + "bool": True, + "list": [1, 2, 3], + "none": None, + } + update_with_defaults(original, defaults) + assert original == { + "str": "text", + "num": 42, + "bool": True, + "list": [1, 2, 3], + "none": None, + } + + def test_update_with_defaults_with_none_values(self): + """Test that None values in defaults are added.""" + original = {"key1": "value1"} + defaults = {"key2": None} + update_with_defaults(original, defaults) + assert original == {"key1": "value1", "key2": None} + + def test_update_with_defaults_preserves_none_in_original(self): + """Test that None values in original are preserved.""" + original = {"key1": None} + defaults = {"key1": "default_value"} + update_with_defaults(original, defaults) + assert original == {"key1": None} + + def test_update_with_defaults_with_list_values(self): + """Test that list values are not merged, only added if missing.""" + original = {"list1": [1, 2]} + defaults = {"list1": [3, 4], "list2": [5, 6]} + update_with_defaults(original, defaults) + assert original == {"list1": [1, 2], "list2": [5, 6]} + + def test_update_with_defaults_type_mismatch_original_wins(self): + """Test that when types differ, original value is preserved.""" + original = {"key1": "string_value"} + defaults = {"key1": {"nested": "dict"}} + update_with_defaults(original, defaults) + assert original == {"key1": "string_value"} + + def test_update_with_defaults_type_mismatch_defaults_dict(self): + """Test that when original is dict and default is not, original is preserved.""" + original = {"key1": {"nested": "dict"}} + defaults = {"key1": "string_value"} + update_with_defaults(original, defaults) + assert original == {"key1": {"nested": "dict"}} + + def test_update_with_defaults_modifies_in_place(self): + """Test that the function modifies the original dict in place.""" + original = {"key1": "value1"} + original_id = id(original) + defaults = {"key2": "value2"} + update_with_defaults(original, defaults) + assert id(original) == original_id + assert original == {"key1": "value1", "key2": "value2"} + + def test_update_with_defaults_with_complex_nested_structure(self): + """Test with complex real-world-like nested structure.""" + original = { + "user": {"name": "Alice", "settings": {"theme": "dark"}}, + "timestamp": "2025-01-01", + } + defaults = { + "user": { + "name": "DefaultName", + "settings": {"theme": "light", "language": "en"}, + "role": "user", + }, + "channel": "default-channel", + } + update_with_defaults(original, defaults) + assert original == { + "user": { + "name": "Alice", + "settings": {"theme": "dark", "language": "en"}, + "role": "user", + }, + "timestamp": "2025-01-01", + "channel": "default-channel", + } + + +class TestPopulateActivity: + """Tests for the populate_activity function.""" + + def test_populate_activity_with_none_values_filled(self): + """Test that None values in original are replaced with defaults.""" + original = Activity(type="message") + defaults = Activity(type="message", text="Default text") + result = populate_activity(original, defaults) + assert result.text == "Default text" + assert result.type == "message" + + def test_populate_activity_preserves_existing_values(self): + """Test that existing non-None values are preserved.""" + original = Activity(type="message", text="Original text") + defaults = Activity(type="event", text="Default text") + result = populate_activity(original, defaults) + assert result.text == "Original text" + assert result.type == "message" + + def test_populate_activity_returns_new_instance(self): + """Test that a new Activity instance is returned.""" + original = Activity(type="message", text="Original") + defaults = {"text": "Default text"} + result = populate_activity(original, defaults) + assert result is not original + assert id(result) != id(original) + + def test_populate_activity_original_unchanged(self): + """Test that the original Activity is not modified.""" + original = Activity(type="message") + defaults = Activity(type="message", text="Default text") + original_text = original.text + result = populate_activity(original, defaults) + assert original.text == original_text + assert result.text == "Default text" + + def test_populate_activity_with_dict_defaults(self): + """Test that defaults can be provided as a dictionary.""" + original = Activity(type="message") + original.channel_id = "channel" + defaults = {"text": "Default text", "channel_id": "default-channel"} + result = populate_activity(original, defaults) + assert result.text == "Default text" + assert result.channel_id == "channel" + + def test_populate_activity_with_activity_defaults(self): + """Test that defaults can be provided as an Activity object.""" + original = Activity(type="message") + defaults = Activity(type="event", text="Default text", channel_id="channel") + result = populate_activity(original, defaults) + assert result.text == "Default text" + + def test_populate_activity_with_empty_defaults(self): + """Test that original is unchanged when defaults is empty.""" + original = Activity(type="message", text="Original text") + defaults = {} + result = populate_activity(original, defaults) + assert result.text == "Original text" + assert result.type == "message" + + def test_populate_activity_with_multiple_fields(self): + """Test populating multiple None fields.""" + original = Activity( + type="message", + ) + defaults = { + "text": "Default text", + "channel_id": "default-channel", + "locale": "en-US", + } + result = populate_activity(original, defaults) + assert result.text == "Default text" + assert result.channel_id == "default-channel" + assert result.locale == "en-US" + + def test_populate_activity_with_complex_objects(self): + """Test populating with complex nested objects.""" + original = Activity(type="message") + defaults = Activity( + type="invoke", + from_property=ChannelAccount(id="bot123", name="Bot"), + conversation=ConversationAccount(id="conv123", name="Conversation"), + ) + result = populate_activity(original, defaults) + assert result.from_property is not None + assert result.from_property.id == "bot123" + assert result.conversation is not None + assert result.conversation.id == "conv123" + + def test_populate_activity_preserves_complex_objects(self): + """Test that existing complex objects are preserved.""" + original = Activity( + type="message", + from_property=ChannelAccount(id="user456", name="User"), + ) + defaults = Activity( + type="invoke", from_property=ChannelAccount(id="bot123", name="Bot") + ) + result = populate_activity(original, defaults) + assert result.from_property.id == "user456" + + def test_populate_activity_partial_defaults(self): + """Test that only specified defaults are applied.""" + original = Activity(type="message") + defaults = {"text": "Default text"} + result = populate_activity(original, defaults) + assert result.text == "Default text" + assert result.channel_id is None + + def test_populate_activity_with_zero_and_empty_string(self): + """Test that zero and empty string are considered as set values.""" + original = Activity(type="message", text="") + defaults = {"text": "Default text", "locale": "en-US"} + result = populate_activity(original, defaults) + # Empty strings should be preserved as they are not None + assert result.text == "" + assert result.locale == "en-US" + + def test_populate_activity_with_false_boolean(self): + """Test that False boolean values are preserved.""" + original = Activity(type="message") + original.history_disclosed = False + defaults = {"history_disclosed": True} + result = populate_activity(original, defaults) + # False should be preserved as it's not None + assert result.history_disclosed is False + + def test_populate_activity_with_zero_numeric(self): + """Test that numeric zero values are preserved.""" + original = Activity(type="message") + # Assuming there's a numeric field we can test + original.channel_data = {"count": 0} + defaults = {"channel_data": {"count": 10}} + result = populate_activity(original, defaults) + # Zero should be preserved + assert result.channel_data == {"count": 0} + + def test_populate_activity_defaults_from_activity_excludes_unset(self): + """Test that only explicitly set fields from Activity defaults are used.""" + original = Activity(type="message") + # Create defaults with only type set explicitly + defaults = Activity(type="event") + result = populate_activity(original, defaults) + # Since defaults Activity didn't explicitly set text, it should remain None + assert result.text is None + + def test_populate_activity_with_empty_activity_defaults(self): + """Test with an Activity that has no fields set.""" + original = Activity(type="message") + defaults = {} + result = populate_activity(original, defaults) + assert result.type == "message" + assert result.text is None + + def test_populate_activity_real_world_scenario(self): + """Test a real-world scenario of populating a bot response.""" + original = Activity( + type="message", + text="User's query result", + from_property=ChannelAccount(id="bot123"), + ) + defaults = { + "conversation": ConversationAccount(id="default-conv"), + "channel_id": "teams", + "locale": "en-US", + } + result = populate_activity(original, defaults) + assert result.text == "User's query result" + assert result.from_property.id == "bot123" + assert result.conversation.id == "default-conv" + assert result.channel_id == "teams" + assert result.locale == "en-US" + + def test_populate_activity_with_list_fields(self): + """Test populating list fields like attachments or entities.""" + original = Activity(type="message") + defaults = {"attachments": [], "entities": []} + result = populate_activity(original, defaults) + assert result.attachments == [] + assert result.entities == [] + + def test_populate_activity_preserves_empty_lists(self): + """Test that empty lists in original are preserved.""" + original = Activity(type="message", attachments=[], entities=[]) + defaults = { + "attachments": [{"type": "card"}], + "entities": [{"type": "mention"}], + } + result = populate_activity(original, defaults) + # Empty lists are not None, so they should be preserved + assert result.attachments == [] + assert result.entities == [] diff --git a/tests/activity/test_sub_channels.py b/tests/activity/test_sub_channels.py deleted file mode 100644 index 67c6d92c..00000000 --- a/tests/activity/test_sub_channels.py +++ /dev/null @@ -1,5 +0,0 @@ -from microsoft_agents.activity import Activity, ChannelId, Entity - - -class TestSubChannels: - pass