From e9d5458e071d18abc837206369480bd3be1b05b0 Mon Sep 17 00:00:00 2001 From: Claire Cho Date: Fri, 1 Aug 2025 12:05:45 -0700 Subject: [PATCH 1/4] e2e --- test_samples/e2e_test/.envLocal | 7 + test_samples/e2e_test/agent_bot.py | 194 ++++++++++++++++++ test_samples/e2e_test/app.py | 89 ++++++++ test_samples/e2e_test/config.py | 18 ++ test_samples/e2e_test/requirements.txt | 3 + test_samples/e2e_test/tools/date_time_tool.py | 10 + .../e2e_test/tools/get_adaptivecard_tool.py | 0 .../e2e_test/tools/get_weather_tool.py | 22 ++ 8 files changed, 343 insertions(+) create mode 100644 test_samples/e2e_test/.envLocal create mode 100644 test_samples/e2e_test/agent_bot.py create mode 100644 test_samples/e2e_test/app.py create mode 100644 test_samples/e2e_test/config.py create mode 100644 test_samples/e2e_test/requirements.txt create mode 100644 test_samples/e2e_test/tools/date_time_tool.py create mode 100644 test_samples/e2e_test/tools/get_adaptivecard_tool.py create mode 100644 test_samples/e2e_test/tools/get_weather_tool.py diff --git a/test_samples/e2e_test/.envLocal b/test_samples/e2e_test/.envLocal new file mode 100644 index 00000000..b2d2c926 --- /dev/null +++ b/test_samples/e2e_test/.envLocal @@ -0,0 +1,7 @@ +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID= +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET= +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID= + +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_API_VERSION= \ No newline at end of file diff --git a/test_samples/e2e_test/agent_bot.py b/test_samples/e2e_test/agent_bot.py new file mode 100644 index 00000000..85695dea --- /dev/null +++ b/test_samples/e2e_test/agent_bot.py @@ -0,0 +1,194 @@ +from __future__ import annotations +import re +from typing import Optional, Union +from os import environ + +from microsoft.agents.hosting.core import ( + AgentApplication, + TurnState, + TurnContext, + MessageFactory +) +from microsoft.agents.activity import ( + ActivityTypes, + InvokeResponse, + Activity, + ConversationUpdateTypes, + ChannelAccount, + Attachment +) + +from agents import ( + Agent as OpenAIAgent, + Model, + ModelProvider, + OpenAIChatCompletionsModel, + RunConfig, + Runner, + ModelSettings +) + +from pydantic import BaseModel, Field + +from tools.get_weather_tool import get_weather +from tools.date_time_tool import get_date + +from openai import AsyncAzureOpenAI + + +class AgentBot: + def __init__(self, client: AsyncAzureOpenAI): + self.client = client + self.multiple_message_pattern = re.compile(r"(\w+)\s+(\d+)") + self.weather_message_pattern = re.compile(r"^w: .*") + + def register_handlers(self, agent_app: AgentApplication[TurnState]): + """Register all handlers with the agent application""" + agent_app.conversation_update(ConversationUpdateTypes.MEMBERS_ADDED)(self.on_members_added) + agent_app.message(self.weather_message_pattern)(self.on_weather_message) + agent_app.message(self.multiple_message_pattern)(self.on_multiple_message) + agent_app.message(re.compile(r"^poem$"))(self.on_poem_message) + agent_app.activity(ActivityTypes.message)(self.on_message) + agent_app.activity(ActivityTypes.invoke)(self.on_invoke) + + async def on_members_added(self, context: TurnContext, _state: TurnState): + await context.send_activity(MessageFactory.text("Hello and Welcome!")) + + async def on_weather_message(self, context: TurnContext, state: TurnState): + + class WeatherForecastAgentResponse(BaseModel): + contentType: str = Field(pattern=r"^(Text|AdaptiveCard)$") + content: Union[dict, str] + context.streaming_response.queue_informative_update("Working on a response for you") + + agent = OpenAIAgent( + name="WeatherAgent", + model_settings=ModelSettings(temperature=0, top_p=1), + instructions="""" + You are a friendly assistant that helps people find a weather forecast for a given time and place. + You may ask follow up questions until you have enough information to answer the customers question, + but once you have a forecast forecast, make sure to format it nicely using an adaptive card. + You should use adaptive JSON format to display the information in a visually appealing way + You should include a button for more details that points at https://www.msn.com/en-us/weather/forecast/in-{location} (replace {location} with the location the user asked about). + You should use adaptive cards version 1.5 or later. + + Respond in JSON format with the following JSON schema: + + { + "contentType": "'Text' or 'AdaptiveCard' only", + "content": "{The content of the response, may be plain text, or JSON based adaptive card}" + } + + Do not include "json" in front of the JSON response. + """, + tools=[get_weather, get_date], + ) + + class CustomModelProvider(ModelProvider): + def get_model(self, model_name: Optional[str]) -> Model: + return OpenAIChatCompletionsModel( + model=model_name or "gpt-4o", + openai_client=self.client, + ) + + custom_model_provider = CustomModelProvider() + + phrase = context.activity.text.split("w: ", 1)[-1].strip() + + response = await Runner.run( + agent, + phrase, + run_config=RunConfig( + model_provider=custom_model_provider, + tracing_disabled=True, + ), + ) + + if "json\n" in response.final_output: + response.final_output = response.final_output.split("json\n", 1)[-1] + + try: + llm_response = WeatherForecastAgentResponse.model_validate_json( + response.final_output + ) + except: + llm_response = response.final_output + + if type(llm_response) is str: + activity = MessageFactory.text(llm_response) + await context.streaming_response.queue_text_chunk(activity) + else: + activity = [ + Attachment( + content_type="application/vnd.microsoft.card.adaptive", + content=llm_response.content, + ) + ] + await context.streaming_response.set_attachments(activity) + + await context.streaming_response.end_stream() + + async def on_multiple_message(self, context: TurnContext, state: TurnState): + counter = state.get_value("ConversationState.counter", default_value_factory=(lambda: 0), target_cls=int) + + match = self.multiple_message_pattern.match(context.activity.text) + if not match: + return + word = match.group(1) + count = int(match.group(2)) + for _ in range(count): + await context.send_activity(f"[{counter}] You said: {word}") + counter += 1 + + state.set_value("ConversationState.counter", counter) + + async def on_poem_message(self, context: TurnContext, state: TurnState): + try: + context.streaming_response.queue_informative_update( + "Hold on for an awesome poem about Apollo..." + ) + + stream = await self.client.chat.completions.create( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": """ + You are a creative assistant who has deeply studied Greek and Roman Gods, You also know all of the Percy Jackson Series + You write poems about the Greek Gods as they are depicted in the Percy Jackson books. + You format the poems in a way that is easy to read and understand + You break your poems into stanzas + You format your poems in Markdown using double lines to separate stanzas + """ + }, + { + "role": "user", + "content": "Write a poem about the Greek God Apollo as depicted in the Percy Jackson books" + } + ], + stream=True, + max_tokens=1000 + ) + + async for update in stream: + if len(update.choices) > 0: + delta = update.choices[0].delta + if delta.content: + context.streaming_response.queue_text_chunk(delta.content) + finally: + await context.streaming_response.end_stream() + + async def on_message(self, context: TurnContext, state: TurnState): + counter = state.get_value("ConversationState.counter", default_value_factory=(lambda: 0), target_cls=int) + await context.send_activity(f"[{counter}] You said: {context.activity.text}") + counter += 1 + state.set_value("ConversationState.counter", counter) + + async def on_invoke(self, context: TurnContext, state: TurnState): + invoke_response = InvokeResponse( + status=200, body={"message": "Invoke received.", "data": context.activity.value} + ) + + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) diff --git a/test_samples/e2e_test/app.py b/test_samples/e2e_test/app.py new file mode 100644 index 00000000..da3d9f03 --- /dev/null +++ b/test_samples/e2e_test/app.py @@ -0,0 +1,89 @@ +from __future__ import annotations +import logging +from aiohttp.web import Application, Request, Response, run_app +from dotenv import load_dotenv +from os import environ, path + +from semantic_kernel import Kernel +from semantic_kernel.utils.logging import setup_logging +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from openai import AsyncAzureOpenAI + +from microsoft.agents.hosting.aiohttp import ( + CloudAdapter, + jwt_authorization_middleware, + start_agent_process, +) +from microsoft.agents.hosting.core import ( + Authorization, + AgentApplication, + TurnState, + MemoryStorage, +) +from microsoft.agents.authentication.msal import MsalConnectionManager +from microsoft.agents.activity import load_configuration_from_env, ConversationUpdateTypes, ActivityTypes +import re + +from agent_bot import AgentBot + +# Load environment variables +load_dotenv(path.join(path.dirname(__file__), ".env")) + +# Load configuration +agents_sdk_config = load_configuration_from_env(environ) + +# Initialize storage and connection manager +STORAGE = MemoryStorage() +CONNECTION_MANAGER = MsalConnectionManager(**agents_sdk_config) +ADAPTER = CloudAdapter(connection_manager=CONNECTION_MANAGER) +AUTHORIZATION = Authorization(STORAGE, CONNECTION_MANAGER, **agents_sdk_config) + +# Initialize Semantic Kernel +kernel = Kernel() +chat_completion = AzureChatCompletion( + deployment_name=environ.get("AZURE_OPENAI_DEPLOYMENT_NAME", "gpt-4o"), + base_url=environ.get("AZURE_OPENAI_ENDPOINT"), + api_key=environ.get("AZURE_OPENAI_API_KEY"), +) +kernel.add_service(chat_completion) +setup_logging() +logging.getLogger("kernel").setLevel(logging.DEBUG) + +# Initialize Azure OpenAI client +client = AsyncAzureOpenAI( + api_version=environ.get("AZURE_OPENAI_API_VERSION"), + azure_endpoint=environ.get("AZURE_OPENAI_ENDPOINT"), + api_key=environ.get("AZURE_OPENAI_API_KEY"), +) + +# Initialize Agent Application +AGENT_APP_INSTANCE = AgentApplication[TurnState]( + storage=STORAGE, adapter=ADAPTER, authorization=AUTHORIZATION, **agents_sdk_config +) + +# Create and configure the AgentBot +AGENT = AgentBot(client) +AGENT.register_handlers(AGENT_APP_INSTANCE) + +# Listen for incoming requests on /api/messages +async def messages(req: Request) -> Response: + agent: AgentApplication = req.app["agent_app"] + adapter: CloudAdapter = req.app["adapter"] + return await start_agent_process( + req, + agent, + adapter, + ) + +# Create the application +APP = Application(middlewares=[jwt_authorization_middleware]) +APP.router.add_post("/api/messages", messages) +APP["agent_configuration"] = CONNECTION_MANAGER.get_default_connection_configuration() +APP["agent_app"] = AGENT_APP_INSTANCE +APP["adapter"] = ADAPTER + +if __name__ == "__main__": + try: + run_app(APP, host="localhost", port=3978) + except Exception as error: + raise error diff --git a/test_samples/e2e_test/config.py b/test_samples/e2e_test/config.py new file mode 100644 index 00000000..135adc28 --- /dev/null +++ b/test_samples/e2e_test/config.py @@ -0,0 +1,18 @@ +from os import environ +from microsoft.agents.hosting.core import AuthTypes, AgentAuthConfiguration + + +class DefaultConfig(AgentAuthConfiguration): + """Agent Configuration""" + + def __init__(self) -> None: + self.AUTH_TYPE = AuthTypes.client_secret + self.TENANT_ID = "" or environ.get("TENANT_ID") + self.CLIENT_ID = "" or environ.get("CLIENT_ID") + self.CLIENT_SECRET = "" or environ.get("CLIENT_SECRET") + self.AZURE_OPENAI_API_KEY = "" or environ.get("AZURE_OPENAI_API_KEY") + self.AZURE_OPENAI_ENDPOINT = "" or environ.get("AZURE_OPENAI_ENDPOINT") + self.AZURE_OPENAI_API_VERSION = "" or environ.get( + "AZURE_OPENAI_API_VERSION", "2024-06-01" + ) + self.PORT = 3978 diff --git a/test_samples/e2e_test/requirements.txt b/test_samples/e2e_test/requirements.txt new file mode 100644 index 00000000..7cc2f377 --- /dev/null +++ b/test_samples/e2e_test/requirements.txt @@ -0,0 +1,3 @@ +openai +openai-agents +semantic-kernel \ No newline at end of file diff --git a/test_samples/e2e_test/tools/date_time_tool.py b/test_samples/e2e_test/tools/date_time_tool.py new file mode 100644 index 00000000..22986b30 --- /dev/null +++ b/test_samples/e2e_test/tools/date_time_tool.py @@ -0,0 +1,10 @@ +from agents import function_tool +from datetime import datetime + + +@function_tool +def get_date() -> str: + """ + A function tool that returns the current date and time. + """ + return datetime.now().isoformat() diff --git a/test_samples/e2e_test/tools/get_adaptivecard_tool.py b/test_samples/e2e_test/tools/get_adaptivecard_tool.py new file mode 100644 index 00000000..e69de29b diff --git a/test_samples/e2e_test/tools/get_weather_tool.py b/test_samples/e2e_test/tools/get_weather_tool.py new file mode 100644 index 00000000..f274f157 --- /dev/null +++ b/test_samples/e2e_test/tools/get_weather_tool.py @@ -0,0 +1,22 @@ +import random +from pydantic import BaseModel + +from agents import function_tool + + +class Weather(BaseModel): + city: str + temperature: str + conditions: str + date: str + + +@function_tool +def get_weather(city: str, date: str) -> Weather: + temperature = random.randint(8, 21) + return Weather( + city=city, + temperature=f"{temperature}C", + conditions="Sunny with wind.", + date=date, + ) From 9967c27aeb8ffddd02af39ee3b5443e9ba5eb8f0 Mon Sep 17 00:00:00 2001 From: Claire Cho Date: Tue, 5 Aug 2025 11:17:36 -0700 Subject: [PATCH 2/4] weather agent works --- test_samples/e2e_test/agent_bot.py | 97 ++++++------------- test_samples/e2e_test/app.py | 5 +- test_samples/e2e_test/requirements.txt | 10 +- test_samples/e2e_test/tools/date_time_tool.py | 10 -- .../e2e_test/tools/get_adaptivecard_tool.py | 0 .../e2e_test/tools/get_weather_tool.py | 22 ----- .../weather/agents/weather_forecast_agent.py | 96 ++++++++++++++++++ .../agents/weather_forecast_agent_response.py | 6 ++ .../weather/plugins/adaptive_card_plugin.py | 35 +++++++ .../weather/plugins/date_time_plugin.py | 31 ++++++ .../weather/plugins/weather_forecast.py | 6 ++ .../plugins/weather_forecast_plugin.py | 25 +++++ 12 files changed, 238 insertions(+), 105 deletions(-) delete mode 100644 test_samples/e2e_test/tools/date_time_tool.py delete mode 100644 test_samples/e2e_test/tools/get_adaptivecard_tool.py delete mode 100644 test_samples/e2e_test/tools/get_weather_tool.py create mode 100644 test_samples/e2e_test/weather/agents/weather_forecast_agent.py create mode 100644 test_samples/e2e_test/weather/agents/weather_forecast_agent_response.py create mode 100644 test_samples/e2e_test/weather/plugins/adaptive_card_plugin.py create mode 100644 test_samples/e2e_test/weather/plugins/date_time_plugin.py create mode 100644 test_samples/e2e_test/weather/plugins/weather_forecast.py create mode 100644 test_samples/e2e_test/weather/plugins/weather_forecast_plugin.py diff --git a/test_samples/e2e_test/agent_bot.py b/test_samples/e2e_test/agent_bot.py index 85695dea..f9852f92 100644 --- a/test_samples/e2e_test/agent_bot.py +++ b/test_samples/e2e_test/agent_bot.py @@ -15,7 +15,7 @@ Activity, ConversationUpdateTypes, ChannelAccount, - Attachment + Attachment, ) from agents import ( @@ -29,13 +29,12 @@ ) from pydantic import BaseModel, Field - -from tools.get_weather_tool import get_weather -from tools.date_time_tool import get_date +from semantic_kernel import Kernel +from semantic_kernel.contents import ChatHistory +from weather.agents.weather_forecast_agent import WeatherForecastAgent from openai import AsyncAzureOpenAI - class AgentBot: def __init__(self, client: AsyncAzureOpenAI): self.client = client @@ -55,77 +54,35 @@ async def on_members_added(self, context: TurnContext, _state: TurnState): await context.send_activity(MessageFactory.text("Hello and Welcome!")) async def on_weather_message(self, context: TurnContext, state: TurnState): - - class WeatherForecastAgentResponse(BaseModel): - contentType: str = Field(pattern=r"^(Text|AdaptiveCard)$") - content: Union[dict, str] + context.streaming_response.queue_informative_update("Working on a response for you") - - agent = OpenAIAgent( - name="WeatherAgent", - model_settings=ModelSettings(temperature=0, top_p=1), - instructions="""" - You are a friendly assistant that helps people find a weather forecast for a given time and place. - You may ask follow up questions until you have enough information to answer the customers question, - but once you have a forecast forecast, make sure to format it nicely using an adaptive card. - You should use adaptive JSON format to display the information in a visually appealing way - You should include a button for more details that points at https://www.msn.com/en-us/weather/forecast/in-{location} (replace {location} with the location the user asked about). - You should use adaptive cards version 1.5 or later. - - Respond in JSON format with the following JSON schema: - - { - "contentType": "'Text' or 'AdaptiveCard' only", - "content": "{The content of the response, may be plain text, or JSON based adaptive card}" - } - - Do not include "json" in front of the JSON response. - """, - tools=[get_weather, get_date], - ) - - class CustomModelProvider(ModelProvider): - def get_model(self, model_name: Optional[str]) -> Model: - return OpenAIChatCompletionsModel( - model=model_name or "gpt-4o", - openai_client=self.client, - ) - custom_model_provider = CustomModelProvider() - - phrase = context.activity.text.split("w: ", 1)[-1].strip() + chat_history = state.get_value( + "ConversationState.chatHistory", + lambda: ChatHistory(), + target_cls=ChatHistory + ) + + weather_agent = WeatherForecastAgent() - response = await Runner.run( - agent, - phrase, - run_config=RunConfig( - model_provider=custom_model_provider, - tracing_disabled=True, - ), - ) + forecast_response = await weather_agent.invoke_agent_async(context.activity.text, chat_history) + if forecast_response == None: + context.streaming_response.queue_text_chunk("Sorry, I couldn't get the weather forecast at the moment.") + await context.streaming_response.end_stream() + return - if "json\n" in response.final_output: - response.final_output = response.final_output.split("json\n", 1)[-1] - - try: - llm_response = WeatherForecastAgentResponse.model_validate_json( - response.final_output + if forecast_response.contentType == "AdaptiveCard": + context.streaming_response.set_attachments( + [ + Attachment( + content_type="application/vnd.microsoft.card.adaptive", + content=forecast_response.content + ) + ] ) - except: - llm_response = response.final_output - - if type(llm_response) is str: - activity = MessageFactory.text(llm_response) - await context.streaming_response.queue_text_chunk(activity) else: - activity = [ - Attachment( - content_type="application/vnd.microsoft.card.adaptive", - content=llm_response.content, - ) - ] - await context.streaming_response.set_attachments(activity) - + context.streaming_response.queue_text_chunk(forecast_response.content) + await context.streaming_response.end_stream() async def on_multiple_message(self, context: TurnContext, state: TurnState): diff --git a/test_samples/e2e_test/app.py b/test_samples/e2e_test/app.py index da3d9f03..60dd6ca8 100644 --- a/test_samples/e2e_test/app.py +++ b/test_samples/e2e_test/app.py @@ -40,14 +40,15 @@ # Initialize Semantic Kernel kernel = Kernel() + chat_completion = AzureChatCompletion( deployment_name=environ.get("AZURE_OPENAI_DEPLOYMENT_NAME", "gpt-4o"), base_url=environ.get("AZURE_OPENAI_ENDPOINT"), api_key=environ.get("AZURE_OPENAI_API_KEY"), + service_id="adaptive_card_service" ) + kernel.add_service(chat_completion) -setup_logging() -logging.getLogger("kernel").setLevel(logging.DEBUG) # Initialize Azure OpenAI client client = AsyncAzureOpenAI( diff --git a/test_samples/e2e_test/requirements.txt b/test_samples/e2e_test/requirements.txt index 7cc2f377..1a158dae 100644 --- a/test_samples/e2e_test/requirements.txt +++ b/test_samples/e2e_test/requirements.txt @@ -1,3 +1,11 @@ openai openai-agents -semantic-kernel \ No newline at end of file +semantic-kernel + +-e ../../libraries/microsoft-agents-hosting-core/ editable_mode=compat +-e ../../libraries/microsoft-agents-hosting-aiohttp/ editable_mode=compat +-e ../../libraries/microsoft-agents-authentication-msal/ editable_mode=compat +-e ../../libraries/microsoft-agents-activity/ editable_mode=compat +-e ../../libraries/microsoft-agents-hosting-teams/ editable_mode=compat +-e ../../libraries/microsoft-agents-copilotstudio-client/ editable_mode=compat +-e ../../libraries/microsoft-agents-storage-blob/ editable_mode=compat \ No newline at end of file diff --git a/test_samples/e2e_test/tools/date_time_tool.py b/test_samples/e2e_test/tools/date_time_tool.py deleted file mode 100644 index 22986b30..00000000 --- a/test_samples/e2e_test/tools/date_time_tool.py +++ /dev/null @@ -1,10 +0,0 @@ -from agents import function_tool -from datetime import datetime - - -@function_tool -def get_date() -> str: - """ - A function tool that returns the current date and time. - """ - return datetime.now().isoformat() diff --git a/test_samples/e2e_test/tools/get_adaptivecard_tool.py b/test_samples/e2e_test/tools/get_adaptivecard_tool.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test_samples/e2e_test/tools/get_weather_tool.py b/test_samples/e2e_test/tools/get_weather_tool.py deleted file mode 100644 index f274f157..00000000 --- a/test_samples/e2e_test/tools/get_weather_tool.py +++ /dev/null @@ -1,22 +0,0 @@ -import random -from pydantic import BaseModel - -from agents import function_tool - - -class Weather(BaseModel): - city: str - temperature: str - conditions: str - date: str - - -@function_tool -def get_weather(city: str, date: str) -> Weather: - temperature = random.randint(8, 21) - return Weather( - city=city, - temperature=f"{temperature}C", - conditions="Sunny with wind.", - date=date, - ) diff --git a/test_samples/e2e_test/weather/agents/weather_forecast_agent.py b/test_samples/e2e_test/weather/agents/weather_forecast_agent.py new file mode 100644 index 00000000..4f082d4f --- /dev/null +++ b/test_samples/e2e_test/weather/agents/weather_forecast_agent.py @@ -0,0 +1,96 @@ +import json +from typing import Optional +from semantic_kernel import Kernel +from semantic_kernel.connectors.ai.open_ai import OpenAIPromptExecutionSettings +from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior + +from semantic_kernel.functions import KernelArguments +from semantic_kernel.contents import ChatHistory +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from weather.agents.weather_forecast_agent_response import WeatherForecastAgentResponse +from semantic_kernel.agents import ChatCompletionAgent, ChatHistoryAgentThread + +from weather.plugins.date_time_plugin import DateTimePlugin +from weather.plugins.weather_forecast_plugin import WeatherForecastPlugin +from weather.plugins.adaptive_card_plugin import AdaptiveCardPlugin + +class WeatherForecastAgent: + + agent_name = "WeatherForecastAgent" + agent_instructions = """ + You are a friendly assistant that helps people find a weather forecast for a given time and place. + You may ask follow up questions until you have enough information to answer the customers question, + but once you have a forecast forecast, make sure to format it nicely using an adaptive card. + You should use adaptive JSON format to display the information in a visually appealing way + You should include a button for more details that points at https://www.msn.com/en-us/weather/forecast/in-{location} (replace {location} with the location the user asked about). + You should use adaptive cards version 1.5 or later. + + Respond only in JSON format with the following JSON schema: + + { + "contentType": "'Text' or 'AdaptiveCard' only", + "content": "{The content of the response, may be plain text, or JSON based adaptive card}" + } + """ + + def __init__(self): + self.kernel = Kernel() + + execution_settings = OpenAIPromptExecutionSettings() + execution_settings.function_choice_behavior = FunctionChoiceBehavior.Auto() + execution_settings.temperature = 0 + execution_settings.top_p = 1 + + self.agent = ChatCompletionAgent( + service=AzureChatCompletion(), + name=self.agent_name, + instructions=self.agent_instructions, + kernel=self.kernel, + arguments=KernelArguments( + chat_history=ChatHistory(), + settings=execution_settings, + kernel=self.kernel + ) + ) + + self.agent.kernel.add_plugin( + plugin=DateTimePlugin(), + plugin_name="datetime" + ) + self.kernel.add_plugin( + plugin=AdaptiveCardPlugin(), + plugin_name="adaptiveCard" + ) + self.kernel.add_plugin( + plugin=WeatherForecastPlugin(), + plugin_name="weatherForecast" + ) + + + async def invoke_agent_async(self, input: str, chat_history: ChatHistory) -> WeatherForecastAgentResponse: + + thread = ChatHistoryAgentThread() + + # Add user message to chat history + chat_history.add_user_message(input) + + resp: str = "" + + async for chat in self.agent.invoke(chat_history, thread=thread): + chat_history.add_message(chat.content) + resp += chat.content.content + + # if resp has a json\n prefix, remove it + if "json\n" in resp: + resp = resp.replace("json\n", "") + resp = resp.replace("```", "") + + # Format the response + resp = resp.strip() + + try: + json_node = json.loads(resp) + result = WeatherForecastAgentResponse.model_validate(json_node) + return result + except Exception as error: + return await self.invoke_agent_async("That response did not match the expected format. Please try again. Error: " + str(error), chat_history) \ No newline at end of file diff --git a/test_samples/e2e_test/weather/agents/weather_forecast_agent_response.py b/test_samples/e2e_test/weather/agents/weather_forecast_agent_response.py new file mode 100644 index 00000000..cbd8acdb --- /dev/null +++ b/test_samples/e2e_test/weather/agents/weather_forecast_agent_response.py @@ -0,0 +1,6 @@ +from typing import Union, Literal +from pydantic import BaseModel + +class WeatherForecastAgentResponse(BaseModel): + contentType: str = Literal["Text", "AdaptiveCard"] + content: Union[dict, str] \ No newline at end of file diff --git a/test_samples/e2e_test/weather/plugins/adaptive_card_plugin.py b/test_samples/e2e_test/weather/plugins/adaptive_card_plugin.py new file mode 100644 index 00000000..343ea2d2 --- /dev/null +++ b/test_samples/e2e_test/weather/plugins/adaptive_card_plugin.py @@ -0,0 +1,35 @@ +from semantic_kernel.functions import kernel_function +from semantic_kernel.connectors.ai.open_ai import OpenAIPromptExecutionSettings +from semantic_kernel.contents import ChatHistory +from os import environ, path +from semantic_kernel import Kernel + +class AdaptiveCardPlugin: + + @kernel_function() + async def get_adaptive_card_for_data( + self, + data: str, + kernel + ) -> str: + + instructions = """ + When given data about the weather forecast for a given time and place, generate an adaptive card + that displays the information in a visually appealing way. Only return the valid adaptive card + JSON string in the response. + """ + + # Set up chat + chat = ChatHistory(instructions=instructions) + chat.add_user_message(data) + + chat_completion = kernel.get_service("adaptive_card_service") + + # Get the response + result = await chat_completion.get_chat_message_contents(chat, OpenAIPromptExecutionSettings()) + + # Extract the message text (if result is a list of ChatMessageContent) + message = result[0].content if result else "No response" + + return message + \ No newline at end of file diff --git a/test_samples/e2e_test/weather/plugins/date_time_plugin.py b/test_samples/e2e_test/weather/plugins/date_time_plugin.py new file mode 100644 index 00000000..563d5a83 --- /dev/null +++ b/test_samples/e2e_test/weather/plugins/date_time_plugin.py @@ -0,0 +1,31 @@ +from semantic_kernel.functions import kernel_function +from datetime import date +from datetime import datetime + +class DateTimePlugin: + + @kernel_function( + name="today", + description="Get the current date", + ) + def today(self, formatProvider: str) -> str: + """ + Get the current date + """ + + _today = date.today() + formatted_date = _today.strftime(formatProvider) + return formatted_date + + @kernel_function( + name="now", + description="Get the current date and time in the local time zone", + ) + def now(self, formatProvider: str) -> str: + """ + Get the current date and time in the local time zone + """ + date_time = datetime.now() + formatted_date_time = date_time.strftime(formatProvider) + return formatted_date_time + diff --git a/test_samples/e2e_test/weather/plugins/weather_forecast.py b/test_samples/e2e_test/weather/plugins/weather_forecast.py new file mode 100644 index 00000000..e192e52d --- /dev/null +++ b/test_samples/e2e_test/weather/plugins/weather_forecast.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + +class WeatherForecast(BaseModel): + date: str + temperatureC: int + temperatureF: int \ No newline at end of file diff --git a/test_samples/e2e_test/weather/plugins/weather_forecast_plugin.py b/test_samples/e2e_test/weather/plugins/weather_forecast_plugin.py new file mode 100644 index 00000000..9ea1c922 --- /dev/null +++ b/test_samples/e2e_test/weather/plugins/weather_forecast_plugin.py @@ -0,0 +1,25 @@ +from semantic_kernel.functions import kernel_function +from .weather_forecast import WeatherForecast +import random +from typing import Annotated + +class WeatherForecastPlugin: + + @kernel_function( + name="get_forecast_for_date", + description="Get a weather forecast for a specific date and location" + ) + def get_forecast_for_date( + self, + date: Annotated[str, "The date for the forecast (e.g., '2025-08-01')"], + location: Annotated[str, "The location for the forecast (e.g., 'Seattle, WA'"] + ) -> Annotated[WeatherForecast, "Weather forecast object with temperature and date"]: + + temperatureC = int(random.uniform(15, 30)) + temperatureF = int((temperatureC * 9 / 5) + 32) + + return WeatherForecast( + date=date, + temperatureC=temperatureC, + temperatureF=temperatureF + ) \ No newline at end of file From 66c7a7f4076a90da2f226f3c1fd5496838cd3fe2 Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 6 Aug 2025 16:23:53 -0700 Subject: [PATCH 3/4] teams activities - kinda hacky though --- test_samples/e2e_test/.envLocal | 5 +- test_samples/e2e_test/agent_bot.py | 164 +++++++++++++++++++++++------ 2 files changed, 135 insertions(+), 34 deletions(-) diff --git a/test_samples/e2e_test/.envLocal b/test_samples/e2e_test/.envLocal index b2d2c926..0ba8c6de 100644 --- a/test_samples/e2e_test/.envLocal +++ b/test_samples/e2e_test/.envLocal @@ -4,4 +4,7 @@ CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID= AZURE_OPENAI_ENDPOINT= AZURE_OPENAI_API_KEY= -AZURE_OPENAI_API_VERSION= \ No newline at end of file +AZURE_OPENAI_API_VERSION= +AZURE_OPENAI_CHAT_DEPLOYMENT_NAME= +AZURE_OPENAI_TEXT_DEPLOYMENT_NAME= +AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME= \ No newline at end of file diff --git a/test_samples/e2e_test/agent_bot.py b/test_samples/e2e_test/agent_bot.py index f9852f92..16048487 100644 --- a/test_samples/e2e_test/agent_bot.py +++ b/test_samples/e2e_test/agent_bot.py @@ -2,19 +2,20 @@ import re from typing import Optional, Union from os import environ +import json from microsoft.agents.hosting.core import ( AgentApplication, TurnState, TurnContext, - MessageFactory + MessageFactory, ) from microsoft.agents.activity import ( ActivityTypes, InvokeResponse, Activity, ConversationUpdateTypes, - ChannelAccount, + ChannelAccount, Attachment, ) @@ -25,7 +26,7 @@ OpenAIChatCompletionsModel, RunConfig, Runner, - ModelSettings + ModelSettings, ) from pydantic import BaseModel, Field @@ -35,48 +36,59 @@ from openai import AsyncAzureOpenAI + class AgentBot: def __init__(self, client: AsyncAzureOpenAI): self.client = client self.multiple_message_pattern = re.compile(r"(\w+)\s+(\d+)") self.weather_message_pattern = re.compile(r"^w: .*") - + def register_handlers(self, agent_app: AgentApplication[TurnState]): """Register all handlers with the agent application""" - agent_app.conversation_update(ConversationUpdateTypes.MEMBERS_ADDED)(self.on_members_added) + agent_app.conversation_update(ConversationUpdateTypes.MEMBERS_ADDED)( + self.on_members_added + ) agent_app.message(self.weather_message_pattern)(self.on_weather_message) agent_app.message(self.multiple_message_pattern)(self.on_multiple_message) agent_app.message(re.compile(r"^poem$"))(self.on_poem_message) agent_app.activity(ActivityTypes.message)(self.on_message) agent_app.activity(ActivityTypes.invoke)(self.on_invoke) - + agent_app.activity(ActivityTypes.message_update)(self.on_message_edit) + agent_app.activity(ActivityTypes.event)(self.on_meeting_events) + async def on_members_added(self, context: TurnContext, _state: TurnState): await context.send_activity(MessageFactory.text("Hello and Welcome!")) async def on_weather_message(self, context: TurnContext, state: TurnState): - - context.streaming_response.queue_informative_update("Working on a response for you") + + context.streaming_response.queue_informative_update( + "Working on a response for you" + ) chat_history = state.get_value( - "ConversationState.chatHistory", - lambda: ChatHistory(), - target_cls=ChatHistory - ) - + "ConversationState.chatHistory", + lambda: ChatHistory(), + target_cls=ChatHistory, + ) + weather_agent = WeatherForecastAgent() - forecast_response = await weather_agent.invoke_agent_async(context.activity.text, chat_history) + forecast_response = await weather_agent.invoke_agent_async( + context.activity.text, chat_history + ) if forecast_response == None: - context.streaming_response.queue_text_chunk("Sorry, I couldn't get the weather forecast at the moment.") + context.streaming_response.queue_text_chunk( + "Sorry, I couldn't get the weather forecast at the moment." + ) await context.streaming_response.end_stream() return - + if forecast_response.contentType == "AdaptiveCard": context.streaming_response.set_attachments( [ Attachment( content_type="application/vnd.microsoft.card.adaptive", - content=forecast_response.content + content=forecast_response.content, ) ] ) @@ -86,7 +98,11 @@ async def on_weather_message(self, context: TurnContext, state: TurnState): await context.streaming_response.end_stream() async def on_multiple_message(self, context: TurnContext, state: TurnState): - counter = state.get_value("ConversationState.counter", default_value_factory=(lambda: 0), target_cls=int) + counter = state.get_value( + "ConversationState.counter", + default_value_factory=(lambda: 0), + target_cls=int, + ) match = self.multiple_message_pattern.match(context.activity.text) if not match: @@ -102,8 +118,8 @@ async def on_multiple_message(self, context: TurnContext, state: TurnState): async def on_poem_message(self, context: TurnContext, state: TurnState): try: context.streaming_response.queue_informative_update( - "Hold on for an awesome poem about Apollo..." - ) + "Hold on for an awesome poem about Apollo..." + ) stream = await self.client.chat.completions.create( model="gpt-4o", @@ -116,17 +132,17 @@ async def on_poem_message(self, context: TurnContext, state: TurnState): You format the poems in a way that is easy to read and understand You break your poems into stanzas You format your poems in Markdown using double lines to separate stanzas - """ + """, }, { "role": "user", - "content": "Write a poem about the Greek God Apollo as depicted in the Percy Jackson books" - } + "content": "Write a poem about the Greek God Apollo as depicted in the Percy Jackson books", + }, ], stream=True, - max_tokens=1000 + max_tokens=1000, ) - + async for update in stream: if len(update.choices) > 0: delta = update.choices[0].delta @@ -136,16 +152,98 @@ async def on_poem_message(self, context: TurnContext, state: TurnState): await context.streaming_response.end_stream() async def on_message(self, context: TurnContext, state: TurnState): - counter = state.get_value("ConversationState.counter", default_value_factory=(lambda: 0), target_cls=int) + counter = state.get_value( + "ConversationState.counter", + default_value_factory=(lambda: 0), + target_cls=int, + ) await context.send_activity(f"[{counter}] You said: {context.activity.text}") counter += 1 state.set_value("ConversationState.counter", counter) + # This needs a rework async def on_invoke(self, context: TurnContext, state: TurnState): - invoke_response = InvokeResponse( - status=200, body={"message": "Invoke received.", "data": context.activity.value} - ) - - await context.send_activity( - Activity(type=ActivityTypes.invoke_response, value=invoke_response) - ) + + # Simulate Teams extensions until implemented + if context.activity.name == "composeExtension/query": + invoke_response = InvokeResponse( + status=200, + body={ + "ComposeExtension": { + "type": "result", + "AttachmentLayout": "list", + "Attachments": [ + {"content_type": "test", "content_url": "example.com"} + ], + }, + "message": "Invoke received.", + "data": context.activity.value, + }, + ) + + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + elif context.activity.name == "composeExtension/queryLink": + invoke_response = InvokeResponse( + status=200, + body={ + "ChannelId": "msteams", + "ComposeExtension": { + "type": "result", + "text": "On Query Link", + }, + }, + ) + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + elif context.activity.name == "composeExtension/selectItem": + value = context.activity.value + invoke_response = InvokeResponse( + status=200, + body={ + "ChannelId": "msteams", + "ComposeExtension": { + "type": "result", + "AttachmentLayout": "list", + "Attachments": [ + { + "contenttype": "application/vnd.microsoft.card.thumbnail", + "content": { + "title": f"{value["id"]}, {value["version"]}" + }, + } + ], + }, + }, + ) + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + else: + invoke_response = InvokeResponse( + status=200, + body={"message": "Invoke received.", "data": context.activity.value}, + ) + + await context.send_activity( + Activity(type=ActivityTypes.invoke_response, value=invoke_response) + ) + + async def on_message_edit(self, context: TurnContext, state: TurnState): + await context.send_activity(f"Message Edited: {context.activity.id}") + + async def on_meeting_events(self, context: TurnContext, state: TurnState): + if context.activity.name == "application/vnd.microsoft.meetingStart": + await context.send_activity( + f"Meeting started with ID: {context.activity.value["id"]}" + ) + elif context.activity.name == "application/vnd.microsoft.meetingEnd": + await context.send_activity( + f"Meeting ended with ID: {context.activity.value["id"]}" + ) + elif ( + context.activity.name == "application/vnd.microsoft.meetingParticipantJoin" + ): + await context.send_activity("Welcome to the meeting!") From 124dc7b6434a804b6754ab69927f2167828c59bd Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 6 Aug 2025 16:28:11 -0700 Subject: [PATCH 4/4] ctrl-z --- test_samples/e2e_test/agent_bot.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test_samples/e2e_test/agent_bot.py b/test_samples/e2e_test/agent_bot.py index 16048487..e4d19fb5 100644 --- a/test_samples/e2e_test/agent_bot.py +++ b/test_samples/e2e_test/agent_bot.py @@ -161,7 +161,6 @@ async def on_message(self, context: TurnContext, state: TurnState): counter += 1 state.set_value("ConversationState.counter", counter) - # This needs a rework async def on_invoke(self, context: TurnContext, state: TurnState): # Simulate Teams extensions until implemented @@ -175,9 +174,7 @@ async def on_invoke(self, context: TurnContext, state: TurnState): "Attachments": [ {"content_type": "test", "content_url": "example.com"} ], - }, - "message": "Invoke received.", - "data": context.activity.value, + } }, )