diff --git a/test_samples/e2e_test/.envLocal b/test_samples/e2e_test/.envLocal new file mode 100644 index 00000000..0ba8c6de --- /dev/null +++ b/test_samples/e2e_test/.envLocal @@ -0,0 +1,10 @@ +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID= +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET= +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID= + +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_API_VERSION= +AZURE_OPENAI_CHAT_DEPLOYMENT_NAME= +AZURE_OPENAI_TEXT_DEPLOYMENT_NAME= +AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME= \ No newline at end of file diff --git a/test_samples/e2e_test/agent_bot.py b/test_samples/e2e_test/agent_bot.py new file mode 100644 index 00000000..e4d19fb5 --- /dev/null +++ b/test_samples/e2e_test/agent_bot.py @@ -0,0 +1,246 @@ +from __future__ import annotations +import re +from typing import Optional, Union +from os import environ +import json + +from microsoft.agents.hosting.core import ( + AgentApplication, + TurnState, + TurnContext, + MessageFactory, +) +from microsoft.agents.activity import ( + ActivityTypes, + InvokeResponse, + Activity, + ConversationUpdateTypes, + ChannelAccount, + Attachment, +) + +from agents import ( + Agent as OpenAIAgent, + Model, + ModelProvider, + OpenAIChatCompletionsModel, + RunConfig, + Runner, + ModelSettings, +) + +from pydantic import BaseModel, Field +from semantic_kernel import Kernel +from semantic_kernel.contents import ChatHistory +from weather.agents.weather_forecast_agent import WeatherForecastAgent + +from openai import AsyncAzureOpenAI + + +class AgentBot: + def __init__(self, client: AsyncAzureOpenAI): + self.client = client + self.multiple_message_pattern = re.compile(r"(\w+)\s+(\d+)") + self.weather_message_pattern = re.compile(r"^w: .*") + + def register_handlers(self, agent_app: AgentApplication[TurnState]): + """Register all handlers with the agent application""" + agent_app.conversation_update(ConversationUpdateTypes.MEMBERS_ADDED)( + self.on_members_added + ) + agent_app.message(self.weather_message_pattern)(self.on_weather_message) + agent_app.message(self.multiple_message_pattern)(self.on_multiple_message) + agent_app.message(re.compile(r"^poem$"))(self.on_poem_message) + agent_app.activity(ActivityTypes.message)(self.on_message) + agent_app.activity(ActivityTypes.invoke)(self.on_invoke) + agent_app.activity(ActivityTypes.message_update)(self.on_message_edit) + agent_app.activity(ActivityTypes.event)(self.on_meeting_events) + + async def on_members_added(self, context: TurnContext, _state: TurnState): + await context.send_activity(MessageFactory.text("Hello and Welcome!")) + + async def on_weather_message(self, context: TurnContext, state: TurnState): + + context.streaming_response.queue_informative_update( + "Working on a response for you" + ) + + chat_history = state.get_value( + "ConversationState.chatHistory", + lambda: ChatHistory(), + target_cls=ChatHistory, + ) + + weather_agent = WeatherForecastAgent() + + forecast_response = await weather_agent.invoke_agent_async( + context.activity.text, chat_history + ) + if forecast_response == None: + context.streaming_response.queue_text_chunk( + "Sorry, I couldn't get the weather forecast at the moment." + ) + await context.streaming_response.end_stream() + return + + if forecast_response.contentType == "AdaptiveCard": + context.streaming_response.set_attachments( + [ + Attachment( + content_type="application/vnd.microsoft.card.adaptive", + content=forecast_response.content, + ) + ] + ) + else: + context.streaming_response.queue_text_chunk(forecast_response.content) + + await context.streaming_response.end_stream() + + async def on_multiple_message(self, context: TurnContext, state: TurnState): + counter = state.get_value( + "ConversationState.counter", + default_value_factory=(lambda: 0), + target_cls=int, + ) + + match = self.multiple_message_pattern.match(context.activity.text) + if not match: + return + word = match.group(1) + count = int(match.group(2)) + for _ in range(count): + await context.send_activity(f"[{counter}] You said: {word}") + counter += 1 + + state.set_value("ConversationState.counter", counter) + + async def on_poem_message(self, context: TurnContext, state: TurnState): + try: + context.streaming_response.queue_informative_update( + "Hold on for an awesome poem about Apollo..." + ) + + stream = await self.client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": """ + You are a creative assistant who has deeply studied Greek and Roman Gods, You also know all of the Percy Jackson Series + You write poems about the Greek Gods as they are depicted in the Percy Jackson books. + You format the poems in a way that is easy to read and understand + You break your poems into stanzas + You format your poems in Markdown using double lines to separate stanzas + """, + }, + { + "role": "user", + "content": "Write a poem about the Greek God Apollo as depicted in the Percy Jackson books", + }, + ], + stream=True, + max_tokens=1000, + ) + + async for update in stream: + if len(update.choices) > 0: + delta = update.choices[0].delta + if delta.content: + context.streaming_response.queue_text_chunk(delta.content) + finally: + await context.streaming_response.end_stream() + + async def on_message(self, context: TurnContext, state: TurnState): + counter = state.get_value( + "ConversationState.counter", + default_value_factory=(lambda: 0), + target_cls=int, + ) + await context.send_activity(f"[{counter}] You said: {context.activity.text}") + counter += 1 + state.set_value("ConversationState.counter", counter) + + async def on_invoke(self, context: TurnContext, state: TurnState): + + # Simulate Teams extensions until implemented + if context.activity.name == "composeExtension/query": + invoke_response = InvokeResponse( + status=200, + body={ + "ComposeExtension": { + "type": "result", + "AttachmentLayout": "list", + "Attachments": [ + {"content_type": "test", "content_url": "example.com"} + ], + } + }, + ) + + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + elif context.activity.name == "composeExtension/queryLink": + invoke_response = InvokeResponse( + status=200, + body={ + "ChannelId": "msteams", + "ComposeExtension": { + "type": "result", + "text": "On Query Link", + }, + }, + ) + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + elif context.activity.name == "composeExtension/selectItem": + value = context.activity.value + invoke_response = InvokeResponse( + status=200, + body={ + "ChannelId": "msteams", + "ComposeExtension": { + "type": "result", + "AttachmentLayout": "list", + "Attachments": [ + { + "contenttype": "application/vnd.microsoft.card.thumbnail", + "content": { + "title": f"{value["id"]}, {value["version"]}" + }, + } + ], + }, + }, + ) + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + else: + invoke_response = InvokeResponse( + status=200, + body={"message": "Invoke received.", "data": context.activity.value}, + ) + + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + + async def on_message_edit(self, context: TurnContext, state: TurnState): + await context.send_activity(f"Message Edited: {context.activity.id}") + + async def on_meeting_events(self, context: TurnContext, state: TurnState): + if context.activity.name == "application/vnd.microsoft.meetingStart": + await context.send_activity( + f"Meeting started with ID: {context.activity.value["id"]}" + ) + elif context.activity.name == "application/vnd.microsoft.meetingEnd": + await context.send_activity( + f"Meeting ended with ID: {context.activity.value["id"]}" + ) + elif ( + context.activity.name == "application/vnd.microsoft.meetingParticipantJoin" + ): + await context.send_activity("Welcome to the meeting!") diff --git a/test_samples/e2e_test/app.py b/test_samples/e2e_test/app.py new file mode 100644 index 00000000..60dd6ca8 --- /dev/null +++ b/test_samples/e2e_test/app.py @@ -0,0 +1,90 @@ +from __future__ import annotations +import logging +from aiohttp.web import Application, Request, Response, run_app +from dotenv import load_dotenv +from os import environ, path + +from semantic_kernel import Kernel +from semantic_kernel.utils.logging import setup_logging +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from openai import AsyncAzureOpenAI + +from microsoft.agents.hosting.aiohttp import ( + CloudAdapter, + jwt_authorization_middleware, + start_agent_process, +) +from microsoft.agents.hosting.core import ( + Authorization, + AgentApplication, + TurnState, + MemoryStorage, +) +from microsoft.agents.authentication.msal import MsalConnectionManager +from microsoft.agents.activity import load_configuration_from_env, ConversationUpdateTypes, ActivityTypes +import re + +from agent_bot import AgentBot + +# Load environment variables +load_dotenv(path.join(path.dirname(__file__), ".env")) + +# Load configuration +agents_sdk_config = load_configuration_from_env(environ) + +# Initialize storage and connection manager +STORAGE = MemoryStorage() +CONNECTION_MANAGER = MsalConnectionManager(**agents_sdk_config) +ADAPTER = CloudAdapter(connection_manager=CONNECTION_MANAGER) +AUTHORIZATION = Authorization(STORAGE, CONNECTION_MANAGER, **agents_sdk_config) + +# Initialize Semantic Kernel +kernel = Kernel() + +chat_completion = AzureChatCompletion( + deployment_name=environ.get("AZURE_OPENAI_DEPLOYMENT_NAME", "gpt-4o"), + base_url=environ.get("AZURE_OPENAI_ENDPOINT"), + api_key=environ.get("AZURE_OPENAI_API_KEY"), + service_id="adaptive_card_service" +) + +kernel.add_service(chat_completion) + +# Initialize Azure OpenAI client +client = AsyncAzureOpenAI( + api_version=environ.get("AZURE_OPENAI_API_VERSION"), + azure_endpoint=environ.get("AZURE_OPENAI_ENDPOINT"), + api_key=environ.get("AZURE_OPENAI_API_KEY"), +) + +# Initialize Agent Application +AGENT_APP_INSTANCE = AgentApplication[TurnState]( + storage=STORAGE, adapter=ADAPTER, authorization=AUTHORIZATION, **agents_sdk_config +) + +# Create and configure the AgentBot +AGENT = AgentBot(client) +AGENT.register_handlers(AGENT_APP_INSTANCE) + +# Listen for incoming requests on /api/messages +async def messages(req: Request) -> Response: + agent: AgentApplication = req.app["agent_app"] + adapter: CloudAdapter = req.app["adapter"] + return await start_agent_process( + req, + agent, + adapter, + ) + +# Create the application +APP = Application(middlewares=[jwt_authorization_middleware]) +APP.router.add_post("/api/messages", messages) +APP["agent_configuration"] = CONNECTION_MANAGER.get_default_connection_configuration() +APP["agent_app"] = AGENT_APP_INSTANCE +APP["adapter"] = ADAPTER + +if __name__ == "__main__": + try: + run_app(APP, host="localhost", port=3978) + except Exception as error: + raise error diff --git a/test_samples/e2e_test/config.py b/test_samples/e2e_test/config.py new file mode 100644 index 00000000..135adc28 --- /dev/null +++ b/test_samples/e2e_test/config.py @@ -0,0 +1,18 @@ +from os import environ +from microsoft.agents.hosting.core import AuthTypes, AgentAuthConfiguration + + +class DefaultConfig(AgentAuthConfiguration): + """Agent Configuration""" + + def __init__(self) -> None: + self.AUTH_TYPE = AuthTypes.client_secret + self.TENANT_ID = "" or environ.get("TENANT_ID") + self.CLIENT_ID = "" or environ.get("CLIENT_ID") + self.CLIENT_SECRET = "" or environ.get("CLIENT_SECRET") + self.AZURE_OPENAI_API_KEY = "" or environ.get("AZURE_OPENAI_API_KEY") + self.AZURE_OPENAI_ENDPOINT = "" or environ.get("AZURE_OPENAI_ENDPOINT") + self.AZURE_OPENAI_API_VERSION = "" or environ.get( + "AZURE_OPENAI_API_VERSION", "2024-06-01" + ) + self.PORT = 3978 diff --git a/test_samples/e2e_test/requirements.txt b/test_samples/e2e_test/requirements.txt new file mode 100644 index 00000000..1a158dae --- /dev/null +++ b/test_samples/e2e_test/requirements.txt @@ -0,0 +1,11 @@ +openai +openai-agents +semantic-kernel + +-e ../../libraries/microsoft-agents-hosting-core/ editable_mode=compat +-e ../../libraries/microsoft-agents-hosting-aiohttp/ editable_mode=compat +-e ../../libraries/microsoft-agents-authentication-msal/ editable_mode=compat +-e ../../libraries/microsoft-agents-activity/ editable_mode=compat +-e ../../libraries/microsoft-agents-hosting-teams/ editable_mode=compat +-e ../../libraries/microsoft-agents-copilotstudio-client/ editable_mode=compat +-e ../../libraries/microsoft-agents-storage-blob/ editable_mode=compat \ No newline at end of file diff --git a/test_samples/e2e_test/weather/agents/weather_forecast_agent.py b/test_samples/e2e_test/weather/agents/weather_forecast_agent.py new file mode 100644 index 00000000..4f082d4f --- /dev/null +++ b/test_samples/e2e_test/weather/agents/weather_forecast_agent.py @@ -0,0 +1,96 @@ +import json +from typing import Optional +from semantic_kernel import Kernel +from semantic_kernel.connectors.ai.open_ai import OpenAIPromptExecutionSettings +from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior + +from semantic_kernel.functions import KernelArguments +from semantic_kernel.contents import ChatHistory +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from weather.agents.weather_forecast_agent_response import WeatherForecastAgentResponse +from semantic_kernel.agents import ChatCompletionAgent, ChatHistoryAgentThread + +from weather.plugins.date_time_plugin import DateTimePlugin +from weather.plugins.weather_forecast_plugin import WeatherForecastPlugin +from weather.plugins.adaptive_card_plugin import AdaptiveCardPlugin + +class WeatherForecastAgent: + + agent_name = "WeatherForecastAgent" + agent_instructions = """ + You are a friendly assistant that helps people find a weather forecast for a given time and place. + You may ask follow up questions until you have enough information to answer the customers question, + but once you have a forecast forecast, make sure to format it nicely using an adaptive card. + You should use adaptive JSON format to display the information in a visually appealing way + You should include a button for more details that points at https://www.msn.com/en-us/weather/forecast/in-{location} (replace {location} with the location the user asked about). + You should use adaptive cards version 1.5 or later. + + Respond only in JSON format with the following JSON schema: + + { + "contentType": "'Text' or 'AdaptiveCard' only", + "content": "{The content of the response, may be plain text, or JSON based adaptive card}" + } + """ + + def __init__(self): + self.kernel = Kernel() + + execution_settings = OpenAIPromptExecutionSettings() + execution_settings.function_choice_behavior = FunctionChoiceBehavior.Auto() + execution_settings.temperature = 0 + execution_settings.top_p = 1 + + self.agent = ChatCompletionAgent( + service=AzureChatCompletion(), + name=self.agent_name, + instructions=self.agent_instructions, + kernel=self.kernel, + arguments=KernelArguments( + chat_history=ChatHistory(), + settings=execution_settings, + kernel=self.kernel + ) + ) + + self.agent.kernel.add_plugin( + plugin=DateTimePlugin(), + plugin_name="datetime" + ) + self.kernel.add_plugin( + plugin=AdaptiveCardPlugin(), + plugin_name="adaptiveCard" + ) + self.kernel.add_plugin( + plugin=WeatherForecastPlugin(), + plugin_name="weatherForecast" + ) + + + async def invoke_agent_async(self, input: str, chat_history: ChatHistory) -> WeatherForecastAgentResponse: + + thread = ChatHistoryAgentThread() + + # Add user message to chat history + chat_history.add_user_message(input) + + resp: str = "" + + async for chat in self.agent.invoke(chat_history, thread=thread): + chat_history.add_message(chat.content) + resp += chat.content.content + + # if resp has a json\n prefix, remove it + if "json\n" in resp: + resp = resp.replace("json\n", "") + resp = resp.replace("```", "") + + # Format the response + resp = resp.strip() + + try: + json_node = json.loads(resp) + result = WeatherForecastAgentResponse.model_validate(json_node) + return result + except Exception as error: + return await self.invoke_agent_async("That response did not match the expected format. Please try again. Error: " + str(error), chat_history) \ No newline at end of file diff --git a/test_samples/e2e_test/weather/agents/weather_forecast_agent_response.py b/test_samples/e2e_test/weather/agents/weather_forecast_agent_response.py new file mode 100644 index 00000000..cbd8acdb --- /dev/null +++ b/test_samples/e2e_test/weather/agents/weather_forecast_agent_response.py @@ -0,0 +1,6 @@ +from typing import Union, Literal +from pydantic import BaseModel + +class WeatherForecastAgentResponse(BaseModel): + contentType: str = Literal["Text", "AdaptiveCard"] + content: Union[dict, str] \ No newline at end of file diff --git a/test_samples/e2e_test/weather/plugins/adaptive_card_plugin.py b/test_samples/e2e_test/weather/plugins/adaptive_card_plugin.py new file mode 100644 index 00000000..343ea2d2 --- /dev/null +++ b/test_samples/e2e_test/weather/plugins/adaptive_card_plugin.py @@ -0,0 +1,35 @@ +from semantic_kernel.functions import kernel_function +from semantic_kernel.connectors.ai.open_ai import OpenAIPromptExecutionSettings +from semantic_kernel.contents import ChatHistory +from os import environ, path +from semantic_kernel import Kernel + +class AdaptiveCardPlugin: + + @kernel_function() + async def get_adaptive_card_for_data( + self, + data: str, + kernel + ) -> str: + + instructions = """ + When given data about the weather forecast for a given time and place, generate an adaptive card + that displays the information in a visually appealing way. Only return the valid adaptive card + JSON string in the response. + """ + + # Set up chat + chat = ChatHistory(instructions=instructions) + chat.add_user_message(data) + + chat_completion = kernel.get_service("adaptive_card_service") + + # Get the response + result = await chat_completion.get_chat_message_contents(chat, OpenAIPromptExecutionSettings()) + + # Extract the message text (if result is a list of ChatMessageContent) + message = result[0].content if result else "No response" + + return message + \ No newline at end of file diff --git a/test_samples/e2e_test/weather/plugins/date_time_plugin.py b/test_samples/e2e_test/weather/plugins/date_time_plugin.py new file mode 100644 index 00000000..563d5a83 --- /dev/null +++ b/test_samples/e2e_test/weather/plugins/date_time_plugin.py @@ -0,0 +1,31 @@ +from semantic_kernel.functions import kernel_function +from datetime import date +from datetime import datetime + +class DateTimePlugin: + + @kernel_function( + name="today", + description="Get the current date", + ) + def today(self, formatProvider: str) -> str: + """ + Get the current date + """ + + _today = date.today() + formatted_date = _today.strftime(formatProvider) + return formatted_date + + @kernel_function( + name="now", + description="Get the current date and time in the local time zone", + ) + def now(self, formatProvider: str) -> str: + """ + Get the current date and time in the local time zone + """ + date_time = datetime.now() + formatted_date_time = date_time.strftime(formatProvider) + return formatted_date_time + diff --git a/test_samples/e2e_test/weather/plugins/weather_forecast.py b/test_samples/e2e_test/weather/plugins/weather_forecast.py new file mode 100644 index 00000000..e192e52d --- /dev/null +++ b/test_samples/e2e_test/weather/plugins/weather_forecast.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + +class WeatherForecast(BaseModel): + date: str + temperatureC: int + temperatureF: int \ No newline at end of file diff --git a/test_samples/e2e_test/weather/plugins/weather_forecast_plugin.py b/test_samples/e2e_test/weather/plugins/weather_forecast_plugin.py new file mode 100644 index 00000000..9ea1c922 --- /dev/null +++ b/test_samples/e2e_test/weather/plugins/weather_forecast_plugin.py @@ -0,0 +1,25 @@ +from semantic_kernel.functions import kernel_function +from .weather_forecast import WeatherForecast +import random +from typing import Annotated + +class WeatherForecastPlugin: + + @kernel_function( + name="get_forecast_for_date", + description="Get a weather forecast for a specific date and location" + ) + def get_forecast_for_date( + self, + date: Annotated[str, "The date for the forecast (e.g., '2025-08-01')"], + location: Annotated[str, "The location for the forecast (e.g., 'Seattle, WA'"] + ) -> Annotated[WeatherForecast, "Weather forecast object with temperature and date"]: + + temperatureC = int(random.uniform(15, 30)) + temperatureF = int((temperatureC * 9 / 5) + 32) + + return WeatherForecast( + date=date, + temperatureC=temperatureC, + temperatureF=temperatureF + ) \ No newline at end of file