Basic stress testing and JWT/MSAL usage optimizations

dev/README.md

@@ -0,0 +1,6 @@
+This directory contains tools to aid the developers of the Microsoft 365 Agents SDK for Python.
+
+### `benchmark`
+
+This folder contains benchmarking utilities built in Python to send concurrent requests
+to an agent.

dev/benchmark/README.md

@@ -0,0 +1,107 @@
+A simple benchmarking tool.
+
+## Benchmark Python Environment Manual Setup (Windows)
+
+Currently a version of this tool that spawns async workers/coroutines instead of
+concurrent threads is not supported, so if you use a "normal" (non free-threaded) version
+of Python, you will be running with the global interpreter lock (GIL).
+
+Note: This may or may not incur significant changes in performance over using
+free-threaded concurrent tests or async workers, depending on the test scenario.
+
+Install any Python version >= 3.9. Check with:
+
+```bash
+python --version
+```
+
+Then, set up and activate the virtual environment with:
+
+```bash
+python -m venv venv
+. ./venv/Scripts/activate
+pip install -r requirements.txt
+```
+
+To activate the virtual environment, use:
+
+```bash
+. ./venv/Scripts/activate
+```
+
+To deactivate it, you may use:
+
+```bash
+deactivate
+```
+
+## Benchmark Python Environment Setup (Windows) - Free Threaded Python
+
+Traditionally, most Python versions have a global interpreter lock (GIL) which prevents
+more than 1 thread to run at the same time. With 3.13, there are free-threaded versions
+of Python which allow one to bypass this constraint. This section walks through how
+to do that on Windows. Use PowerShell.
+
+Based on: https://docs.python.org/3/using/windows.html#
+
+Go to `Microsoft Store` and install `Python Install Manager` and follow the instructions
+presented. You may have to make certain changes to alias used by your machine (that
+should be guided by the installation process).
+
+Based on: https://docs.python.org/3/whatsnew/3.13.html#free-threaded-cpython
+
+In PowerShell, install the free-threaded version of Python of your choice. In this guide
+we will install `3.14t`:
+
+```bash
+py install 3.14t
+```
+
+Then, set up and activate the virtual environment with:
+
+```bash
+python3.14t -m venv venv
+. ./venv/Scripts/activate
+pip install -r requirements.txt
+```
+
+To activate the virtual environment, use:
+
+```bash
+. ./venv/Scripts/activate
+```
+
+To deactivate it, you may use:
+
+```bash
+deactivate
+```
+
+## Benchmark Configuration
+
+If you open the `env.template` file, you will see three environmental variables to define:
+
+```bash
+TENANT_ID=
+APP_ID=
+APP_SECRET=
+```
+
+For `APP_ID` use the app Id of your ABS resource. For `APP_SECRET` set it to a secret
+for the App Registration resource tied to your ABS resource. Finally, the `TENANT_ID`
+variable should be set to the tenant Id of your ABS resource.
+
+These settings are used to generate valid tokens that are sent and validated by the
+agent you are trying to run.
+
+## Usage
+
+Running these tests requires you to have the agent running in a separate process. You
+may open a separate PowerShell window or VSCode window and run your agent there.
+
+To run the basic payload sending stress test (our only implemented test so far), use:
+
+```bash
+. ./venv/Scripts/activate # activate the virtual environment if you haven't already
+python -m src.main --num_workers=...
+```

dev/benchmark/env.template

@@ -0,0 +1,3 @@
+TENANT_ID=
+APP_ID=
+APP_SECRET=

dev/benchmark/payload.json

@@ -0,0 +1,19 @@
+{
+  "channelId": "msteams",
+  "serviceUrl": "http://localhost:49231/_connector",
+  "delivery_mode": "expectReplies",
+  "recipient": {
+    "id": "00000000-0000-0000-0000-00000000000011",
+    "name": "Test Bot"
+  },
+  "conversation": {
+    "id": "personal-chat-id",
+    "conversationType": "personal",
+    "tenantId": "00000000-0000-0000-0000-0000000000001"
+  },
+  "from": {
+    "id": "user-id-0",
+    "aadObjectId": "00000000-0000-0000-0000-0000000000020"
+  },
+  "type": "message"
+}

dev/benchmark/requirements.txt

@@ -0,0 +1,4 @@
+microsoft-agents-activity
+microsoft-agents-hosting-core
+click
+azure-identity

dev/benchmark/src/aggregated_results.py

@@ -0,0 +1,51 @@
+from .executor import ExecutionResult
+
+
+class AggregatedResults:
+    """Class to analyze execution time results."""
+
+    def __init__(self, results: list[ExecutionResult]):
+        self._results = results
+
+        self.average = sum(r.duration for r in results) / len(results) if results else 0
+        self.min = min((r.duration for r in results), default=0)
+        self.max = max((r.duration for r in results), default=0)
+        self.success_count = sum(1 for r in results if r.success)
+        self.failure_count = len(results) - self.success_count
+        self.total_time = sum(r.duration for r in results)
+
+    def display(self, start_time: float, end_time: float):
+        """Display aggregated results."""
+        print()
+        print("---- Aggregated Results ----")
+        print()
+        print(f"Average Time: {self.average:.4f} seconds")
+        print(f"Min Time:     {self.min:.4f} seconds")
+        print(f"Max Time:     {self.max:.4f} seconds")
+        print()
+        print(f"Success Rate: {self.success_count} / {len(self._results)}")
+        print()
+        print(f"Total Time:   {end_time - start_time} seconds")
+        print("----------------------------")
+        print()
+
+    def display_timeline(self):
+        """Display timeline of individual execution results."""
+        print()
+        print("---- Execution Timeline ----")
+        print(
+            "Each '.' represents 1 second of successful execution. So a line like '...' is a success that took 3 seconds (rounded up), 'x' represents a failure."
+        )
+        print()
+        for result in sorted(self._results, key=lambda r: r.exe_id):
+            c = "." if result.success else "x"
+            if c == ".":
+                duration = int(round(result.duration))
+                for _ in range(1 + duration):
+                    print(c, end="")
+                print()
+            else:
+                print(c)
+
+        print("----------------------------")
+        print()

dev/benchmark/src/config.py

@@ -0,0 +1,23 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+class BenchmarkConfig:
+    """Configuration class for benchmark settings."""
+
+    TENANT_ID: str = ""
+    APP_ID: str = ""
+    APP_SECRET: str = ""
+    AGENT_API_URL: str = ""
+
+    @classmethod
+    def load_from_env(cls) -> None:
+        """Loads configuration values from environment variables."""
+        cls.TENANT_ID = os.environ.get("TENANT_ID", "")
+        cls.APP_ID = os.environ.get("APP_ID", "")
+        cls.APP_SECRET = os.environ.get("APP_SECRET", "")
+        cls.AGENT_URL = os.environ.get(
+            "AGENT_API_URL", "http://localhost:3978/api/messages"
+        )

dev/benchmark/src/executor/__init__.py

@@ -0,0 +1,11 @@
+from .coroutine_executor import CoroutineExecutor
+from .execution_result import ExecutionResult
+from .executor import Executor
+from .thread_executor import ThreadExecutor
+
+__all__ = [
+    "CoroutineExecutor",
+    "ExecutionResult",
+    "Executor",
+    "ThreadExecutor",
+]

dev/benchmark/src/executor/coroutine_executor.py

@@ -0,0 +1,28 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import asyncio
+from typing import Callable, Awaitable, Any
+
+from .executor import Executor
+from .execution_result import ExecutionResult
+
+
+class CoroutineExecutor(Executor):
+    """An executor that runs asynchronous functions using asyncio."""
+
+    def run(
+        self, func: Callable[[], Awaitable[Any]], num_workers: int = 1
+    ) -> list[ExecutionResult]:
+        """Run the given asynchronous function using the specified number of coroutines.
+
+        :param func: An asynchronous function to be executed.
+        :param num_workers: The number of coroutines to use.
+        """
+
+        async def gather():
+            return await asyncio.gather(
+                *[self.run_func(i, func) for i in range(num_workers)]
+            )
+
+        return asyncio.run(gather())

dev/benchmark/src/executor/execution_result.py

@@ -0,0 +1,28 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from typing import Any, Optional
+from dataclasses import dataclass
+
+
+@dataclass
+class ExecutionResult:
+    """Class to represent the result of an execution."""
+
+    exe_id: int
+
+    start_time: float
+    end_time: float
+
+    result: Any = None
+    error: Optional[Exception] = None
+
+    @property
+    def success(self) -> bool:
+        """Indicate whether the execution was successful."""
+        return self.error is None
+
+    @property
+    def duration(self) -> float:
+        """Calculate the duration of the execution, in seconds."""
+        return self.end_time - self.start_time

dev/benchmark/src/executor/executor.py

@@ -0,0 +1,49 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from datetime import datetime, timezone
+from abc import ABC, abstractmethod
+from typing import Callable, Awaitable, Any
+
+from .execution_result import ExecutionResult
+
+
+class Executor(ABC):
+    """Protocol for executing asynchronous functions concurrently."""
+
+    async def run_func(
+        self, exe_id: int, func: Callable[[], Awaitable[Any]]
+    ) -> ExecutionResult:
+        """Run the given asynchronous function.
+
+        :param exe_id: An identifier for the execution instance.
+        :param func: An asynchronous function to be executed.
+        """
+
+        start_time = datetime.now(timezone.utc).timestamp()
+        try:
+            result = await func()
+            return ExecutionResult(
+                exe_id=exe_id,
+                result=result,
+                start_time=start_time,
+                end_time=datetime.now(timezone.utc).timestamp(),
+            )
+        except Exception as e:  # pylint: disable=broad-except
+            return ExecutionResult(
+                exe_id=exe_id,
+                error=e,
+                start_time=start_time,
+                end_time=datetime.now(timezone.utc).timestamp(),
+            )
+
+    @abstractmethod
+    def run(
+        self, func: Callable[[], Awaitable[Any]], num_workers: int = 1
+    ) -> list[ExecutionResult]:
+        """Run the given asynchronous function using the specified number of workers.
+
+        :param func: An asynchronous function to be executed.
+        :param num_workers: The number of concurrent workers to use.
+        """
+        raise NotImplementedError("This method should be implemented by subclasses.")

dev/benchmark/src/executor/thread_executor.py

@@ -0,0 +1,37 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import logging
+import asyncio
+from typing import Callable, Awaitable, Any
+from concurrent.futures import ThreadPoolExecutor
+
+from .executor import Executor
+from .execution_result import ExecutionResult
+
+logger = logging.getLogger(__name__)
+
+
+class ThreadExecutor(Executor):
+    """An executor that runs asynchronous functions using multiple threads."""
+
+    def run(
+        self, func: Callable[[], Awaitable[Any]], num_workers: int = 1
+    ) -> list[ExecutionResult]:
+        """Run the given asynchronous function using the specified number of threads.
+
+        :param func: An asynchronous function to be executed.
+        :param num_workers: The number of concurrent threads to use.
+        """
+
+        def _func(exe_id: int) -> ExecutionResult:
+            return asyncio.run(self.run_func(exe_id, func))
+
+        results: list[ExecutionResult] = []
+
+        with ThreadPoolExecutor(max_workers=num_workers) as executor:
+            futures = [executor.submit(_func, i) for i in range(num_workers)]
+            for future in futures:
+                results.append(future.result())
+
+        return results