Integration testing framework

dev/README.md

@@ -1,6 +0,0 @@
-This directory contains tools to aid the developers of the Microsoft 365 Agents SDK for Python.
-
-### `benchmark`
-
-This folder contains benchmarking utilities built in Python to send concurrent requests
-to an agent.

dev/benchmark/payload.json

@@ -15,5 +15,6 @@
     "id": "user-id-0",
     "aadObjectId": "00000000-0000-0000-0000-0000000000020"
   },
-  "type": "message"
+  "type": "message",
+  "text": "Hello, Bot!"
 }

dev/benchmark/src/main.py

@@ -1,32 +1,35 @@
-import json
+import json, sys
+from io import StringIO
 import logging
 from datetime import datetime, timezone
+from contextlib import contextmanager
 
 import click
 
 from .payload_sender import create_payload_sender
 from .executor import Executor, CoroutineExecutor, ThreadExecutor
 from .aggregated_results import AggregatedResults
 from .config import BenchmarkConfig
+from .output import output_results
 
 LOG_FORMAT = "%(asctime)s: %(message)s"
 logging.basicConfig(format=LOG_FORMAT, level=logging.INFO, datefmt="%H:%M:%S")
 
 BenchmarkConfig.load_from_env()
 
-
 @click.command()
 @click.option(
     "--payload_path", "-p", default="./payload.json", help="Path to the payload file."
 )
 @click.option("--num_workers", "-n", default=1, help="Number of workers to use.")
+@click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging.")
 @click.option(
     "--async_mode",
     "-a",
     is_flag=True,
     help="Run coroutine workers rather than thread workers.",
 )
-def main(payload_path: str, num_workers: int, async_mode: bool):
+def main(payload_path: str, num_workers: int, verbose: bool, async_mode: bool):
     """Main function to run the benchmark."""
 
     with open(payload_path, "r", encoding="utf-8") as f:
@@ -35,10 +38,12 @@ def main(payload_path: str, num_workers: int, async_mode: bool):
     func = create_payload_sender(payload)
 
     executor: Executor = CoroutineExecutor() if async_mode else ThreadExecutor()
-
+    
     start_time = datetime.now(timezone.utc).timestamp()
     results = executor.run(func, num_workers=num_workers)
     end_time = datetime.now(timezone.utc).timestamp()
+    if verbose:
+        output_results(results)
 
     agg = AggregatedResults(results)
     agg.display(start_time, end_time)

dev/benchmark/src/output.py

@@ -0,0 +1,9 @@
+from .executor import ExecutionResult
+
+def output_results(results: list[ExecutionResult]) -> None:
+    """Output the results of the benchmark to the console."""
+
+    for result in results:
+        status = "Success" if result.success else "Failure"
+        print(f"Execution ID: {result.exe_id}, Duration: {result.duration:.4f} seconds, Status: {status}")
+        print(result.result)

dev/install.sh

@@ -0,0 +1 @@
+pip install -e ./microsoft-agents-testing/ --config-settings editable_mode=compat

dev/microsoft-agents-testing/README.md

@@ -0,0 +1 @@
+# Microsoft 365 Agents SDK for Python - Testing Framework

dev/microsoft-agents-testing/microsoft_agents/testing/__init__.py

@@ -0,0 +1,36 @@
+from .sdk_config import SDKConfig
+
+from .auth import (
+    generate_token,
+    generate_token_from_config
+)
+
+from .utils import (
+    populate_activity,
+    get_host_and_port
+)
+
+from .integration import (
+    Sample,
+    Environment,
+    ApplicationRunner,
+    AgentClient,
+    ResponseClient,
+    AiohttpEnvironment,
+    Integration
+)
+
+__all__ = [
+    "SDKConfig",
+    "generate_token",
+    "generate_token_from_config",
+    "Sample",
+    "Environment",
+    "ApplicationRunner",
+    "AgentClient",
+    "ResponseClient",
+    "AiohttpEnvironment",
+    "Integration",
+    "populate_activity",
+    "get_host_and_port"
+]

dev/microsoft-agents-testing/microsoft_agents/testing/auth/__init__.py

@@ -0,0 +1,6 @@
+from .generate_token import generate_token, generate_token_from_config
+
+__all__ = [
+    "generate_token",
+    "generate_token_from_config"
+]

dev/microsoft-agents-testing/microsoft_agents/testing/auth/generate_token.py

@@ -0,0 +1,47 @@
+import requests
+
+from microsoft_agents.hosting.core import AgentAuthConfiguration
+from microsoft_agents.testing.sdk_config import SDKConfig
+
+def generate_token(app_id: str, app_secret: str, tenant_id: str) -> str:
+    """Generate a token using the provided app credentials.
+
+    :param app_id: Application (client) ID.
+    :param app_secret: Application client secret.
+    :param tenant_id: Directory (tenant) ID.
+    :return: Generated access token as a string.
+    """
+
+    authority_endpoint = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
+
+    res = requests.post(
+        authority_endpoint,
+        headers={
+            "Content-Type": "application/x-www-form-urlencoded",
+        },
+        data={
+            "grant_type": "client_credentials",
+            "client_id": app_id,
+            "client_secret": app_secret,
+            "scope": f"{app_id}/.default",
+        },
+        timeout=10,
+    )
+    return res.json().get("access_token")
+
+def generate_token_from_config(sdk_config: SDKConfig) -> str:
+    """Generates a token using a provided config object.
+
+    :param config: Configuration dictionary containing connection settings.
+    :return: Generated access token as a string.
+    """
+
+    settings: AgentAuthConfiguration = sdk_config.get_connection()
+
+    app_id = settings.CLIENT_ID
+    app_secret = settings.CLIENT_SECRET
+    tenant_id = settings.TENANT_ID
+
+    if not app_id or not app_secret or not tenant_id:
+        raise ValueError("Incorrect configuration provided for token generation.")
+    return generate_token(app_id, app_secret, tenant_id)

dev/microsoft-agents-testing/microsoft_agents/testing/integration/__init__.py

@@ -0,0 +1,19 @@
+from .core import (
+    AgentClient,
+    ApplicationRunner,
+    AiohttpEnvironment,
+    ResponseClient,
+    Environment,
+    Integration,
+    Sample,
+)
+
+__all__ = [
+    "AgentClient",
+    "ApplicationRunner",
+    "AiohttpEnvironment",
+    "ResponseClient",
+    "Environment",
+    "Integration",
+    "Sample",
+]

dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/__init__.py

@@ -0,0 +1,20 @@
+from .application_runner import ApplicationRunner
+from .aiohttp import AiohttpEnvironment
+from .client import (
+    AgentClient,
+    ResponseClient,
+)
+from .environment import Environment
+from .integration import Integration
+from .sample import Sample
+
+
+__all__ = [
+    "AgentClient",
+    "ApplicationRunner",
+    "AiohttpEnvironment",
+    "ResponseClient",
+    "Environment",
+    "Integration",
+    "Sample",
+]

dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/__init__.py

@@ -0,0 +1,4 @@
+from .aiohttp_environment import AiohttpEnvironment
+from .aiohttp_runner import AiohttpRunner
+
+__all__ = ["AiohttpEnvironment", "AiohttpRunner"]

dev/microsoft-agents-testing/microsoft_agents/testing/integration/core/aiohttp/aiohttp_environment.py

@@ -0,0 +1,58 @@
+from tkinter import E
+from aiohttp.web import Request, Response, Application, run_app
+
+from microsoft_agents.hosting.aiohttp import (
+    CloudAdapter,
+    jwt_authorization_middleware,
+    start_agent_process,
+)
+from microsoft_agents.hosting.core import (
+    Authorization,
+    AgentApplication,
+    TurnState,
+    MemoryStorage,
+)
+from microsoft_agents.authentication.msal import MsalConnectionManager
+from microsoft_agents.activity import load_configuration_from_env
+
+from ..application_runner import ApplicationRunner
+from ..environment import Environment
+from .aiohttp_runner import AiohttpRunner
+
+
+class AiohttpEnvironment(Environment):
+    """An environment for aiohttp-hosted agents."""
+
+    async def init_env(self, environ_config: dict) -> None:
+        environ_config = environ_config or {}
+
+        self.config = load_configuration_from_env(environ_config)
+
+        self.storage = MemoryStorage()
+        self.connection_manager = MsalConnectionManager(**self.config)
+        self.adapter = CloudAdapter(connection_manager=self.connection_manager)
+        self.authorization = Authorization(
+            self.storage, self.connection_manager, **self.config
+        )
+
+        self.agent_application = AgentApplication[TurnState](
+            storage=self.storage,
+            adapter=self.adapter,
+            authorization=self.authorization,
+            **self.config
+        )
+
+    def create_runner(self, host: str, port: int) -> ApplicationRunner:
+
+        async def entry_point(req: Request) -> Response:
+            agent: AgentApplication = req.app["agent_app"]
+            adapter: CloudAdapter = req.app["adapter"]
+            return await start_agent_process(req, agent, adapter)
+
+        APP = Application(middlewares=[jwt_authorization_middleware])
+        APP.router.add_post("/api/messages", entry_point)
+        APP["agent_configuration"] = self.connection_manager.get_default_connection_configuration()
+        APP["agent_app"] = self.agent_application
+        APP["adapter"] = self.adapter
+
+        return AiohttpRunner(APP, host, port)