From 405bd9765670ebf1cf3eb2a601a7309c056d5484 Mon Sep 17 00:00:00 2001 From: Dwij Patel Date: Thu, 20 Mar 2025 03:24:15 +0530 Subject: [PATCH 1/5] Added langchain callback handler support --- agentops/integrations/langchain/README.md | 158 +++ .../langchain/callback_handler.py | 1246 +++++++++++++++++ agentops/semconv/langchain_attributes.py | 37 + tests/handlers/conftest.py | 154 ++ tests/handlers/test_langchain_callback.py | 923 ++++++++++++ 5 files changed, 2518 insertions(+) create mode 100644 agentops/integrations/langchain/README.md create mode 100644 agentops/integrations/langchain/callback_handler.py create mode 100644 agentops/semconv/langchain_attributes.py create mode 100644 tests/handlers/conftest.py create mode 100644 tests/handlers/test_langchain_callback.py diff --git a/agentops/integrations/langchain/README.md b/agentops/integrations/langchain/README.md new file mode 100644 index 000000000..cad95559e --- /dev/null +++ b/agentops/integrations/langchain/README.md @@ -0,0 +1,158 @@ +# Langchain Callback Handler for AgentOps + +This module provides OpenTelemetry-based callback handlers for Langchain that integrate with AgentOps for comprehensive tracing and monitoring of Langchain applications. + +## Features + +- **Comprehensive Event Tracking**: Monitors all major Langchain operations: + - LLM calls (including streaming responses) + - Chat model interactions + - Chain executions + - Tool usage + - Retriever operations + - Agent actions + - Retry attempts + +- **Dual Mode Support**: + - Synchronous operations (`LangchainCallbackHandler`) + - Asynchronous operations (`AsyncLangchainCallbackHandler`) + +- **Detailed Span Attributes**: + - Operation inputs and outputs + - Token usage statistics + - Error details and stack traces + - Model information + - Tool parameters and results + +## Usage + +### Basic Usage + +```python +from agentops.integrations.langchain.callback_handler import LangchainCallbackHandler + +# Initialize the handler +handler = LangchainCallbackHandler( + api_key="your-api-key" +) + +# Use with Langchain +from langchain.llms import OpenAI +from langchain.chains import LLMChain + +llm = OpenAI( + callbacks=[handler], + temperature=0.7 +) + +chain = LLMChain( + llm=llm, + prompt=your_prompt, + callbacks=[handler] +) +``` + +### Async Usage + +```python +from agentops.integrations.langchain.callback_handler import AsyncLangchainCallbackHandler + +# Initialize the async handler +handler = AsyncLangchainCallbackHandler( + api_key="your-api-key" +) + +# Use with async Langchain +from langchain.llms import OpenAI +from langchain.chains import LLMChain + +llm = OpenAI( + callbacks=[handler], + temperature=0.7 +) + +chain = LLMChain( + llm=llm, + prompt=your_prompt, + callbacks=[handler] +) +``` + +## Span Types and Attributes + +### LLM Spans +- **Name**: "llm" +- **Attributes**: + - `run_id`: Unique identifier for the LLM run + - `model`: Name of the LLM model + - `prompt`: Input prompt + - `response`: Generated response + - `token_usage`: Token usage statistics + +### Chat Model Spans +- **Name**: "chat_model" +- **Attributes**: + - `run_id`: Unique identifier for the chat run + - `model`: Name of the chat model + - `messages`: Conversation history + - `response`: Generated response + +### Chain Spans +- **Name**: "chain" +- **Attributes**: + - `run_id`: Unique identifier for the chain run + - `chain_name`: Name of the chain + - `inputs`: Chain input parameters + - `outputs`: Chain output results + +### Tool Spans +- **Name**: "tool" +- **Attributes**: + - `run_id`: Unique identifier for the tool run + - `tool_name`: Name of the tool + - `tool_input`: Tool input parameters + - `tool_output`: Tool output results + +### Retriever Spans +- **Name**: "retriever" +- **Attributes**: + - `run_id`: Unique identifier for the retriever run + - `retriever_name`: Name of the retriever + - `query`: Search query + - `documents`: Retrieved documents + +### Agent Spans +- **Name**: "agent_action" +- **Attributes**: + - `run_id`: Unique identifier for the agent run + - `agent_name`: Name of the agent + - `tool_input`: Tool input parameters + - `tool_log`: Agent's reasoning log + +## Error Handling + +The handler provides comprehensive error tracking: +- Records error type and message +- Preserves error context and stack traces +- Updates span status to ERROR +- Maintains error details in span attributes + +## Testing + +The handler includes comprehensive tests in `test_callback_handler.py` that verify: +- Basic functionality of both sync and async handlers +- Error handling and recovery +- Span creation and attribute recording +- Token usage tracking +- Streaming response handling +- Chain and tool execution tracking +- Agent action monitoring + +To run the tests: +```bash +pytest tests/handlers/langchain/test_callback_handler.py -v +``` + +## Contributing + +Contributions are welcome! Please feel free to submit a Pull Request. \ No newline at end of file diff --git a/agentops/integrations/langchain/callback_handler.py b/agentops/integrations/langchain/callback_handler.py new file mode 100644 index 000000000..35bff44eb --- /dev/null +++ b/agentops/integrations/langchain/callback_handler.py @@ -0,0 +1,1246 @@ +"""Langchain callback handler using OpenTelemetry. + +This module provides callback handlers for Langchain that integrate with OpenTelemetry +for tracing and monitoring. It supports both synchronous and asynchronous operations, +tracking various events in the Langchain execution flow including LLM calls, tool usage, +chain execution, and agent actions. +""" + +from typing import Dict, Any, List, Optional, Sequence, Union +from uuid import UUID +import logging +import os +from collections import defaultdict + +from tenacity import RetryCallState +from langchain_core.agents import AgentFinish, AgentAction +from langchain_core.documents import Document +from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResult, Generation +from langchain_core.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler +from langchain_core.messages import BaseMessage, AIMessage, AIMessageChunk +from opentelemetry.trace import Status, StatusCode + +from agentops.sdk.decorators.utility import _create_as_current_span +from agentops.semconv import SpanKind +from agentops.semconv.langchain_attributes import LangchainAttributes +from agentops.logging import logger + +def get_model_from_kwargs(kwargs: Any) -> str: + """Extract model name from kwargs. + + This function attempts to get the model name from the invocation parameters + in the kwargs dictionary. It checks for both 'model' and '_type' keys in the + invocation_params dictionary. + + Args: + kwargs: Dictionary containing invocation parameters + + Returns: + str: The model name if found, otherwise 'unknown_model' + """ + if "model" in kwargs.get("invocation_params", {}): + return kwargs["invocation_params"]["model"] + elif "_type" in kwargs.get("invocation_params", {}): + return kwargs["invocation_params"]["_type"] + return "unknown_model" + +def _create_span_attributes( + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[List[str]] = None, + metadata: Optional[Dict[str, Any]] = None, + **kwargs: Any, +) -> Dict[str, Any]: + """Create common span attributes. + + This function creates a standardized set of attributes for OpenTelemetry spans. + It includes common attributes like run_id, parent_run_id, tags, and metadata, + along with any additional attributes passed in kwargs. + + Args: + run_id: Unique identifier for the current run + parent_run_id: Optional identifier for the parent run + tags: Optional list of tags for the span + metadata: Optional dictionary of metadata + **kwargs: Additional attributes to include + + Returns: + Dict[str, Any]: Dictionary of span attributes + """ + return { + LangchainAttributes.RUN_ID: str(run_id), + LangchainAttributes.PARENT_RUN_ID: str(parent_run_id) if parent_run_id else "", + LangchainAttributes.TAGS: str(tags or []), + LangchainAttributes.METADATA: str(metadata or {}), + **kwargs, + } + +def _handle_span_error(span: Any, error: Exception, **kwargs: Any) -> None: + """Handle span error consistently. + + This function provides a standardized way to handle errors in spans, + setting appropriate error attributes and status codes. + + Args: + span: The OpenTelemetry span to update + error: The exception that occurred + **kwargs: Additional context for the error + """ + span.set_status(Status(StatusCode.ERROR)) + span.set_attribute(LangchainAttributes.ERROR_TYPE, type(error).__name__) + span.set_attribute(LangchainAttributes.ERROR_MESSAGE, str(error)) + span.set_attribute(LangchainAttributes.ERROR_DETAILS, { + "run_id": kwargs.get("run_id"), + "parent_run_id": kwargs.get("parent_run_id"), + "kwargs": str(kwargs) + }) + span.end() + +class BaseLangchainHandler: + """Base class for Langchain handlers with common functionality. + + This class provides shared functionality for both synchronous and asynchronous + Langchain callback handlers, including initialization, span tracking, and + common utility methods. + """ + + def __init__( + self, + api_key: Optional[str] = None, + endpoint: Optional[str] = None, + max_wait_time: Optional[int] = None, + max_queue_size: Optional[int] = None, + default_tags: List[str] = None, + ): + """Initialize the handler with configuration options. + + Args: + api_key: Optional API key for AgentOps + endpoint: Optional endpoint URL for AgentOps + max_wait_time: Optional maximum wait time for operations + max_queue_size: Optional maximum size of the operation queue + default_tags: Optional list of default tags for spans + """ + # Set up logging + logging_level = os.getenv("AGENTOPS_LOGGING_LEVEL") + log_levels = { + "CRITICAL": logging.CRITICAL, + "ERROR": logging.ERROR, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "DEBUG": logging.DEBUG, + } + logger.setLevel(log_levels.get(logging_level or "INFO", "INFO")) + + # Initialize AgentOps client + from agentops import Client + self.client = Client() + self.client.configure( + api_key=api_key or "test-api-key", + endpoint=endpoint, + max_wait_time=max_wait_time, + max_queue_size=max_queue_size, + default_tags=default_tags or ["langchain", "sync" if isinstance(self, LangchainCallbackHandler) else "async"], + ) + self.client.init() + + # Initialize span tracking + self._llm_spans: Dict[str, Any] = {} + self._tool_spans: Dict[str, Any] = {} + self._chain_spans: Dict[str, Any] = {} + self._retriever_spans: Dict[str, Any] = {} + self._agent_actions: Dict[UUID, List[Any]] = defaultdict(list) + + def _handle_llm_response(self, span: Any, response: LLMResult) -> None: + """Handle LLM response and set appropriate attributes. + + This method processes an LLM response and updates the span with relevant + information including the response text, token usage, and other metadata. + + Args: + span: The OpenTelemetry span to update + response: The LLM response to process + """ + if not hasattr(response, "generations"): + return + + for generation_list in response.generations: + for generation in generation_list: + if isinstance(generation, Generation): + if generation.text: + span.set_attribute(LangchainAttributes.RESPONSE, generation.text) + elif hasattr(generation, "message"): + if isinstance(generation.message, AIMessage) and generation.message.content: + span.set_attribute(LangchainAttributes.RESPONSE, generation.message.content) + elif isinstance(generation.message, AIMessageChunk) and generation.message.content: + current_completion = span.get_attribute(LangchainAttributes.RESPONSE) or "" + span.set_attribute(LangchainAttributes.RESPONSE, current_completion + generation.message.content) + + # Handle token usage + if hasattr(response, "llm_output") and isinstance(response.llm_output, dict): + token_usage = response.llm_output.get("token_usage", {}) + if isinstance(token_usage, dict): + span.set_attribute(LangchainAttributes.PROMPT_TOKENS, token_usage.get("prompt_tokens", 0)) + span.set_attribute(LangchainAttributes.COMPLETION_TOKENS, token_usage.get("completion_tokens", 0)) + span.set_attribute(LangchainAttributes.TOTAL_TOKENS, token_usage.get("total_tokens", 0)) + + def _handle_agent_finish(self, run_id: UUID, finish: AgentFinish) -> None: + """Handle agent finish event and update spans. + + This method processes the completion of an agent's task, updating the + relevant spans with final outputs and status. + + Args: + run_id: Unique identifier for the agent run + finish: The AgentFinish event containing final outputs + """ + agent_spans = self._agent_actions.get(run_id, []) + if not agent_spans: + return + + last_span = agent_spans[-1] + last_span.set_attribute(LangchainAttributes.OUTPUTS, str(finish.return_values)) + last_span.set_attribute(LangchainAttributes.TOOL_LOG, finish.log) + last_span.set_status(Status(StatusCode.OK)) + last_span.end() + + # Record all agent actions + for span in agent_spans[:-1]: + span.set_status(Status(StatusCode.OK)) + span.end() + + self._agent_actions.pop(run_id, None) + + @property + def current_session_ids(self) -> List[str]: + """Get current session IDs. + + Returns: + List[str]: List of current session IDs from the AgentOps client + """ + return self.client.current_session_ids + +class LangchainCallbackHandler(BaseLangchainHandler, BaseCallbackHandler): + """Callback handler for Langchain using OpenTelemetry. + + This class implements the synchronous callback interface for Langchain, + tracking various events in the execution flow and creating appropriate + OpenTelemetry spans for monitoring and debugging. + """ + + def on_llm_start( + self, + serialized: Dict[str, Any], + prompts: List[str], + **kwargs: Any, + ) -> None: + """Handle LLM start event. + + This method is called when an LLM operation begins. It creates a new span + to track the operation and stores relevant information about the model and + input prompts. + + Args: + serialized: Serialized information about the LLM + prompts: List of input prompts + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="llm", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + model=get_model_from_kwargs(kwargs), + prompt=prompts[0] if prompts else "", + **kwargs, + ), + ) as span: + self._llm_spans[run_id] = span + + except Exception as e: + logger.error(f"Error in on_llm_start: {str(e)}") + + def on_llm_end( + self, + response: LLMResult, + **kwargs: Any, + ) -> None: + """Handle LLM end event. + + This method is called when an LLM operation completes. It processes the + response, updates the span with results, and handles any errors that + occurred during the operation. + + Args: + response: The LLM result containing generations and metadata + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + span = self._llm_spans.get(run_id) + if not span: + return + + try: + self._handle_llm_response(span, response) + span.set_status(Status(StatusCode.OK)) + except Exception as e: + logger.error(f"Error in on_llm_end: {str(e)}") + _handle_span_error(span, e, **kwargs) + finally: + span.end() + self._llm_spans.pop(run_id, None) + + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle LLM error event. + + This method is called when an error occurs during an LLM operation. + It updates the span with error information and marks it as failed. + + Args: + error: The exception that occurred + run_id: Unique identifier for the LLM run + **kwargs: Additional error context + """ + if str(run_id) in self._llm_spans: + span = self._llm_spans[str(run_id)] + _handle_span_error(span, error, run_id=run_id, **kwargs) + + def on_chat_model_start( + self, + serialized: Dict[str, Any], + messages: List[List[BaseMessage]], + **kwargs: Any, + ) -> None: + """Handle chat model start event. + + This method is called when a chat model operation begins. It creates a + new span to track the operation and stores information about the model + and input messages. + + Args: + serialized: Serialized information about the chat model + messages: List of message lists containing the conversation + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + parsed_messages = [ + {"role": message.type, "content": message.content} + for message in messages[0] + if message.type in ["system", "human"] + ] + + with _create_as_current_span( + name="chat_model", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + model=get_model_from_kwargs(kwargs), + messages=str(parsed_messages), + **kwargs, + ), + ) as span: + self._llm_spans[run_id] = span + + except Exception as e: + logger.error(f"Error in on_chat_model_start: {str(e)}") + + def on_chain_start( + self, + serialized: Dict[str, Any], + inputs: Dict[str, Any], + **kwargs: Any, + ) -> None: + """Handle chain start event. + + This method is called when a Langchain chain operation begins. It creates + a new span to track the chain execution and stores information about the + chain and its inputs. + + Args: + serialized: Serialized information about the chain + inputs: Dictionary of input values for the chain + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="chain", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + chain_name=serialized.get("name", "unknown"), + inputs=str(inputs or {}), + **kwargs, + ), + ) as span: + self._chain_spans[run_id] = span + + except Exception as e: + logger.error(f"Error in on_chain_start: {str(e)}") + + def on_chain_end( + self, + outputs: Dict[str, Any], + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle chain end event. + + This method is called when a Langchain chain operation completes. It + updates the span with the chain's outputs and marks it as successful. + + Args: + outputs: Dictionary of output values from the chain + run_id: Unique identifier for the chain run + **kwargs: Additional arguments + """ + if str(run_id) in self._chain_spans: + span = self._chain_spans[str(run_id)] + span.set_attribute(LangchainAttributes.CHAIN_OUTPUTS, str(outputs)) + span.end() + + def on_chain_error( + self, + error: BaseException, + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle chain error event. + + This method is called when an error occurs during a chain operation. + It updates the span with error information and marks it as failed. + + Args: + error: The exception that occurred + run_id: Unique identifier for the chain run + **kwargs: Additional error context + """ + if str(run_id) in self._chain_spans: + span = self._chain_spans[str(run_id)] + _handle_span_error(span, error, run_id=run_id, **kwargs) + + def on_tool_start( + self, + serialized: Dict[str, Any], + input_str: str, + **kwargs: Any, + ) -> None: + """Handle tool start event. + + This method is called when a tool operation begins. It creates a new span + to track the tool execution and stores information about the tool and its + inputs. + + Args: + serialized: Serialized information about the tool + input_str: String input for the tool + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="tool", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + tool_name=serialized.get("name", "unknown"), + tool_input=input_str, + inputs=str(kwargs.get("inputs", {})), + **kwargs, + ), + ) as span: + self._tool_spans[run_id] = span + + except Exception as e: + logger.error(f"Error in on_tool_start: {str(e)}") + + def on_tool_end( + self, + output: str, + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle tool end event. + + This method is called when a tool operation completes. It updates the span + with the tool's output and handles any errors that occurred during the + operation. + + Args: + output: String output from the tool + run_id: Unique identifier for the tool run + **kwargs: Additional arguments + """ + if str(run_id) in self._tool_spans: + span = self._tool_spans[str(run_id)] + span.set_attribute(LangchainAttributes.TOOL_OUTPUT, output) + + if kwargs.get("name") == "_Exception": + _handle_span_error(span, Exception(output), run_id=run_id, **kwargs) + else: + span.end() + + def on_tool_error( + self, + error: BaseException, + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle tool error event. + + This method is called when an error occurs during a tool operation. + It updates the span with error information and marks it as failed. + + Args: + error: The exception that occurred + run_id: Unique identifier for the tool run + **kwargs: Additional error context + """ + if str(run_id) in self._tool_spans: + span = self._tool_spans[str(run_id)] + _handle_span_error(span, error, run_id=run_id, **kwargs) + + def on_retriever_start( + self, + serialized: Dict[str, Any], + query: str, + **kwargs: Any, + ) -> None: + """Handle retriever start event. + + This method is called when a retriever operation begins. It creates a new + span to track the retrieval process and stores information about the + retriever and the query. + + Args: + serialized: Serialized information about the retriever + query: The search query string + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="retriever", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + retriever_name=serialized.get("name", "unknown"), + query=query, + **kwargs, + ), + ) as span: + self._retriever_spans[run_id] = span + + except Exception as e: + logger.error(f"Error in on_retriever_start: {str(e)}") + + def on_retriever_end( + self, + documents: Sequence[Document], + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle retriever end event. + + This method is called when a retriever operation completes. It updates + the span with the retrieved documents and marks it as successful. + + Args: + documents: Sequence of retrieved documents + run_id: Unique identifier for the retriever run + **kwargs: Additional arguments + """ + if str(run_id) in self._retriever_spans: + span = self._retriever_spans[str(run_id)] + span.set_attribute(LangchainAttributes.RETRIEVER_DOCUMENTS, str(documents)) + span.end() + + def on_retriever_error( + self, + error: BaseException, + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle retriever error event. + + This method is called when an error occurs during a retriever operation. + It updates the span with error information and marks it as failed. + + Args: + error: The exception that occurred + run_id: Unique identifier for the retriever run + **kwargs: Additional error context + """ + if str(run_id) in self._retriever_spans: + span = self._retriever_spans[str(run_id)] + _handle_span_error(span, error, run_id=run_id, **kwargs) + + def on_agent_action( + self, + action: AgentAction, + **kwargs: Any, + ) -> None: + """Handle agent action event. + + This method is called when an agent performs an action. It creates a new + span to track the action and stores information about the tool being used + and its inputs. + + Args: + action: The agent action containing tool and input information + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="agent_action", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + agent_name=action.tool, + tool_input=action.tool_input, + tool_log=action.log, + **kwargs, + ), + ) as span: + self._agent_actions[run_id].append(span) + + except Exception as e: + logger.error(f"Error in on_agent_action: {str(e)}") + + def on_agent_finish( + self, + finish: AgentFinish, + **kwargs: Any, + ) -> None: + """Handle agent finish event. + + This method is called when an agent completes its task. It updates all + relevant spans with final outputs and marks them as successful. + + Args: + finish: The AgentFinish event containing final outputs + **kwargs: Additional arguments including run_id + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + self._handle_agent_finish(run_id, finish) + except Exception as e: + logger.error(f"Error in on_agent_finish: {str(e)}") + + def on_retry( + self, + retry_state: RetryCallState, + **kwargs: Any, + ) -> None: + """Handle retry event. + + This method is called when an operation is being retried. It creates a + new span to track the retry attempt and stores information about the + error that caused the retry. + + Args: + retry_state: State information about the retry attempt + **kwargs: Additional arguments including run_id + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="retry", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + retry_attempt=retry_state.attempt_number, + error_type=type(retry_state.outcome.exception()).__name__, + error_message=str(retry_state.outcome.exception()), + **kwargs, + ), + ) as span: + span.set_status(Status(StatusCode.ERROR)) + span.end() + + except Exception as e: + logger.error(f"Error in on_retry: {str(e)}") + + def on_llm_new_token( + self, + token: str, + **kwargs: Any, + ) -> None: + """Handle new LLM token event. + + This method is called when a new token is generated during streaming + LLM responses. It updates the span with the accumulated response text. + + Args: + token: The new token generated + **kwargs: Additional arguments including run_id + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + span = self._llm_spans.get(run_id) + if not span: + return + + try: + current_completion = span.get_attribute(LangchainAttributes.RESPONSE) or "" + span.set_attribute(LangchainAttributes.RESPONSE, current_completion + token) + except Exception as e: + logger.error(f"Error in on_llm_new_token: {str(e)}") + + +class AsyncLangchainCallbackHandler(BaseLangchainHandler, AsyncCallbackHandler): + """Async callback handler for Langchain using OpenTelemetry. + + This class implements the asynchronous callback interface for Langchain, + providing the same functionality as the synchronous handler but with + async/await support for better performance in asynchronous environments. + """ + + async def on_llm_start( + self, + serialized: Dict[str, Any], + prompts: List[str], + **kwargs: Any, + ) -> None: + """Handle LLM start event asynchronously. + + This method is called when an LLM operation begins. It creates a new span + to track the operation and stores relevant information about the model and + input prompts. + + Args: + serialized: Serialized information about the LLM + prompts: List of input prompts + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="llm", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + model=get_model_from_kwargs(kwargs), + prompt=prompts[0] if prompts else "", + **kwargs, + ), + ) as span: + self._llm_spans[run_id] = span + + except Exception as e: + logger.error(f"Error in on_llm_start: {str(e)}") + + async def on_llm_end( + self, + response: LLMResult, + **kwargs: Any, + ) -> None: + """Handle LLM end event asynchronously. + + This method is called when an LLM operation completes. It processes the + response, updates the span with results, and handles any errors that + occurred during the operation. + + Args: + response: The LLM result containing generations and metadata + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + span = self._llm_spans.get(run_id) + if not span: + return + + try: + self._handle_llm_response(span, response) + span.set_status(Status(StatusCode.OK)) + except Exception as e: + logger.error(f"Error in on_llm_end: {str(e)}") + _handle_span_error(span, e, **kwargs) + finally: + span.end() + self._llm_spans.pop(run_id, None) + + async def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle LLM error event asynchronously. + + This method is called when an error occurs during an LLM operation. + It updates the span with error information and marks it as failed. + + Args: + error: The exception that occurred + run_id: Unique identifier for the LLM run + **kwargs: Additional error context + """ + if str(run_id) in self._llm_spans: + span = self._llm_spans[str(run_id)] + _handle_span_error(span, error, run_id=run_id, **kwargs) + + async def on_chat_model_start( + self, + serialized: Dict[str, Any], + messages: List[List[BaseMessage]], + **kwargs: Any, + ) -> None: + """Handle chat model start event asynchronously. + + This method is called when a chat model operation begins. It creates a + new span to track the operation and stores information about the model + and input messages. + + Args: + serialized: Serialized information about the chat model + messages: List of message lists containing the conversation + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + parsed_messages = [ + {"role": message.type, "content": message.content} + for message in messages[0] + if message.type in ["system", "human"] + ] + + with _create_as_current_span( + name="chat_model", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + model=get_model_from_kwargs(kwargs), + messages=str(parsed_messages), + **kwargs, + ), + ) as span: + self._llm_spans[run_id] = span + + except Exception as e: + logger.error(f"Error in on_chat_model_start: {str(e)}") + + async def on_chain_start( + self, + serialized: Dict[str, Any], + inputs: Dict[str, Any], + **kwargs: Any, + ) -> None: + """Handle chain start event asynchronously. + + This method is called when a Langchain chain operation begins. It creates + a new span to track the chain execution and stores information about the + chain and its inputs. + + Args: + serialized: Serialized information about the chain + inputs: Dictionary of input values for the chain + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="chain", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + chain_name=serialized.get("name", "unknown"), + inputs=str(inputs or {}), + **kwargs, + ), + ) as span: + self._chain_spans[run_id] = span + + except Exception as e: + logger.error(f"Error in on_chain_start: {str(e)}") + + async def on_chain_end( + self, + outputs: Dict[str, Any], + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle chain end event asynchronously. + + This method is called when a Langchain chain operation completes. It + updates the span with the chain's outputs and marks it as successful. + + Args: + outputs: Dictionary of output values from the chain + run_id: Unique identifier for the chain run + **kwargs: Additional arguments + """ + if str(run_id) in self._chain_spans: + span = self._chain_spans[str(run_id)] + span.set_attribute(LangchainAttributes.CHAIN_OUTPUTS, str(outputs)) + span.end() + + async def on_chain_error( + self, + error: BaseException, + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle chain error event asynchronously. + + This method is called when an error occurs during a chain operation. + It updates the span with error information and marks it as failed. + + Args: + error: The exception that occurred + run_id: Unique identifier for the chain run + **kwargs: Additional error context + """ + if str(run_id) in self._chain_spans: + span = self._chain_spans[str(run_id)] + _handle_span_error(span, error, run_id=run_id, **kwargs) + + async def on_tool_start( + self, + serialized: Dict[str, Any], + input_str: str, + **kwargs: Any, + ) -> None: + """Handle tool start event asynchronously. + + This method is called when a tool operation begins. It creates a new span + to track the tool execution and stores information about the tool and its + inputs. + + Args: + serialized: Serialized information about the tool + input_str: String input for the tool + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="tool", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + tool_name=serialized.get("name", "unknown"), + tool_input=input_str, + inputs=str(kwargs.get("inputs", {})), + **kwargs, + ), + ) as span: + self._tool_spans[run_id] = span + + except Exception as e: + logger.error(f"Error in on_tool_start: {str(e)}") + + async def on_tool_end( + self, + output: str, + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle tool end event asynchronously. + + This method is called when a tool operation completes. It updates the span + with the tool's output and handles any errors that occurred during the + operation. + + Args: + output: String output from the tool + run_id: Unique identifier for the tool run + **kwargs: Additional arguments + """ + if str(run_id) in self._tool_spans: + span = self._tool_spans[str(run_id)] + span.set_attribute(LangchainAttributes.TOOL_OUTPUT, output) + + if kwargs.get("name") == "_Exception": + _handle_span_error(span, Exception(output), run_id=run_id, **kwargs) + else: + span.end() + + async def on_tool_error( + self, + error: BaseException, + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle tool error event asynchronously. + + This method is called when an error occurs during a tool operation. + It updates the span with error information and marks it as failed. + + Args: + error: The exception that occurred + run_id: Unique identifier for the tool run + **kwargs: Additional error context + """ + if str(run_id) in self._tool_spans: + span = self._tool_spans[str(run_id)] + _handle_span_error(span, error, run_id=run_id, **kwargs) + + async def on_retriever_start( + self, + serialized: Dict[str, Any], + query: str, + **kwargs: Any, + ) -> None: + """Handle retriever start event asynchronously. + + This method is called when a retriever operation begins. It creates a new + span to track the retrieval process and stores information about the + retriever and the query. + + Args: + serialized: Serialized information about the retriever + query: The search query string + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="retriever", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + retriever_name=serialized.get("name", "unknown"), + query=query, + **kwargs, + ), + ) as span: + self._retriever_spans[run_id] = span + + except Exception as e: + logger.error(f"Error in on_retriever_start: {str(e)}") + + async def on_retriever_end( + self, + documents: Sequence[Document], + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle retriever end event asynchronously. + + This method is called when a retriever operation completes. It updates + the span with the retrieved documents and marks it as successful. + + Args: + documents: Sequence of retrieved documents + run_id: Unique identifier for the retriever run + **kwargs: Additional arguments + """ + if str(run_id) in self._retriever_spans: + span = self._retriever_spans[str(run_id)] + span.set_attribute(LangchainAttributes.RETRIEVER_DOCUMENTS, str(documents)) + span.end() + + async def on_retriever_error( + self, + error: BaseException, + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Handle retriever error event asynchronously. + + This method is called when an error occurs during a retriever operation. + It updates the span with error information and marks it as failed. + + Args: + error: The exception that occurred + run_id: Unique identifier for the retriever run + **kwargs: Additional error context + """ + if str(run_id) in self._retriever_spans: + span = self._retriever_spans[str(run_id)] + _handle_span_error(span, error, run_id=run_id, **kwargs) + + async def on_agent_action( + self, + action: AgentAction, + **kwargs: Any, + ) -> None: + """Handle agent action event asynchronously. + + This method is called when an agent performs an action. It creates a new + span to track the action and stores information about the tool being used + and its inputs. + + Args: + action: The agent action containing tool and input information + **kwargs: Additional arguments including run_id and metadata + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="agent_action", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + agent_name=action.tool, + tool_input=action.tool_input, + tool_log=action.log, + **kwargs, + ), + ) as span: + self._agent_actions[run_id].append(span) + + except Exception as e: + logger.error(f"Error in on_agent_action: {str(e)}") + + async def on_agent_finish( + self, + finish: AgentFinish, + **kwargs: Any, + ) -> None: + """Handle agent finish event asynchronously. + + This method is called when an agent completes its task. It updates all + relevant spans with final outputs and marks them as successful. + + Args: + finish: The AgentFinish event containing final outputs + **kwargs: Additional arguments including run_id + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + self._handle_agent_finish(run_id, finish) + except Exception as e: + logger.error(f"Error in on_agent_finish: {str(e)}") + + async def on_retry( + self, + retry_state: RetryCallState, + **kwargs: Any, + ) -> None: + """Handle retry event asynchronously. + + This method is called when an operation is being retried. It creates a + new span to track the retry attempt and stores information about the + error that caused the retry. + + Args: + retry_state: State information about the retry attempt + **kwargs: Additional arguments including run_id + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + try: + with _create_as_current_span( + name="retry", + kind=SpanKind.INTERNAL, + attributes=_create_span_attributes( + run_id=run_id, + retry_attempt=retry_state.attempt_number, + error_type=type(retry_state.outcome.exception()).__name__, + error_message=str(retry_state.outcome.exception()), + **kwargs, + ), + ) as span: + span.set_status(Status(StatusCode.ERROR)) + span.end() + + except Exception as e: + logger.error(f"Error in on_retry: {str(e)}") + + async def on_llm_new_token( + self, + token: str, + **kwargs: Any, + ) -> None: + """Handle new LLM token event asynchronously. + + This method is called when a new token is generated during streaming + LLM responses. It updates the span with the accumulated response text. + + Args: + token: The new token generated + **kwargs: Additional arguments including run_id + """ + run_id = kwargs.get("run_id") + if not run_id: + return + + span = self._llm_spans.get(run_id) + if not span: + return + + try: + current_completion = span.get_attribute(LangchainAttributes.RESPONSE) or "" + span.set_attribute(LangchainAttributes.RESPONSE, current_completion + token) + except Exception as e: + logger.error(f"Error in on_llm_new_token: {str(e)}") \ No newline at end of file diff --git a/agentops/semconv/langchain_attributes.py b/agentops/semconv/langchain_attributes.py new file mode 100644 index 000000000..4682218e8 --- /dev/null +++ b/agentops/semconv/langchain_attributes.py @@ -0,0 +1,37 @@ +"""Semantic conventions for Langchain integration. + +This module defines the semantic conventions used for Langchain spans and attributes +in OpenTelemetry traces. These conventions ensure consistent attribute naming +across the Langchain integration. +""" + +class LangchainAttributes: + """Semantic conventions for Langchain spans and attributes.""" + + # Run identifiers + RUN_ID = "langchain.run.id" + PARENT_RUN_ID = "langchain.parent_run.id" + TAGS = "langchain.tags" + METADATA = "langchain.metadata" + + # Response attributes + RESPONSE = "langchain.response" + PROMPT_TOKENS = "langchain.prompt_tokens" + COMPLETION_TOKENS = "langchain.completion_tokens" + TOTAL_TOKENS = "langchain.total_tokens" + + # Chain attributes + CHAIN_OUTPUTS = "langchain.chain.outputs" + + # Tool attributes + TOOL_OUTPUT = "langchain.tool.output" + TOOL_LOG = "langchain.tool.log" + OUTPUTS = "langchain.outputs" + + # Retriever attributes + RETRIEVER_DOCUMENTS = "langchain.retriever.documents" + + # Error attributes + ERROR_TYPE = "langchain.error.type" + ERROR_MESSAGE = "langchain.error.message" + ERROR_DETAILS = "langchain.error.details" \ No newline at end of file diff --git a/tests/handlers/conftest.py b/tests/handlers/conftest.py new file mode 100644 index 000000000..a4635c339 --- /dev/null +++ b/tests/handlers/conftest.py @@ -0,0 +1,154 @@ +"""Test fixtures for integrations.""" + +import os +import re +import uuid +from collections import defaultdict +from unittest import mock + +import pytest +import requests_mock +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider, ReadableSpan +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.trace import Status, StatusCode, SpanKind + +import agentops +from agentops.config import Config +from tests.fixtures.client import * # noqa +from tests.unit.sdk.instrumentation_tester import InstrumentationTester + + +@pytest.fixture +def api_key() -> str: + """Standard API key for testing""" + return "test-api-key" + + +@pytest.fixture +def endpoint() -> str: + """Base API URL""" + return Config().endpoint + + +@pytest.fixture(autouse=True) +def mock_req(endpoint, api_key): + """ + Mocks AgentOps backend API requests. + """ + with requests_mock.Mocker(real_http=False) as m: + # Map session IDs to their JWTs + m.post(endpoint + "/v3/auth/token", json={"token": str(uuid.uuid4()), + "project_id": "test-project-id", "api_key": api_key}) + yield m + + +@pytest.fixture +def noinstrument(): + # Tells the client to not instrument LLM calls + yield + + +@pytest.fixture +def mock_config(mocker): + """Mock the Client.configure method""" + return mocker.patch("agentops.client.Client.configure") + + +@pytest.fixture +def instrumentation(): + """Fixture for the instrumentation tester.""" + tester = InstrumentationTester() + yield tester + tester.reset() + + +@pytest.fixture +def tracer_provider(): + """Create a tracer provider with memory exporter for testing.""" + provider = TracerProvider() + exporter = InMemorySpanExporter() + processor = SimpleSpanProcessor(exporter) + provider.add_span_processor(processor) + trace.set_tracer_provider(provider) + return provider, exporter + + +@pytest.fixture +def mock_span(): + """Create a mock span for testing.""" + span = mock.Mock(spec=ReadableSpan) + span.name = "test_span" + span.kind = SpanKind.INTERNAL + span.attributes = {} + span.status = Status(StatusCode.OK) + span.parent = None + span.context = mock.Mock() + span.context.trace_id = 0x1234567890abcdef1234567890abcdef + span.context.span_id = 0x1234567890abcdef + return span + + +@pytest.fixture +def test_run_ids(): + """Create test run IDs for callback testing.""" + return { + "run_id": uuid.uuid4(), + "parent_run_id": uuid.uuid4(), + } + + +@pytest.fixture +def test_llm_inputs(): + """Create test LLM inputs for callback testing.""" + return { + "serialized": {"name": "test-model"}, + "prompts": ["test prompt"], + "metadata": {"test": "metadata"}, + } + + +@pytest.fixture +def test_chain_inputs(): + """Create test chain inputs for callback testing.""" + return { + "serialized": {"name": "test-chain"}, + "inputs": {"test": "input"}, + } + + +@pytest.fixture +def test_tool_inputs(): + """Create test tool inputs for callback testing.""" + return { + "serialized": {"name": "test-tool"}, + "input_str": "test input", + } + + +@pytest.fixture +def test_agent_inputs(): + """Create test agent inputs for callback testing.""" + return { + "action": mock.Mock( + tool="test-tool", + tool_input="test input", + log="test log", + ), + "finish": mock.Mock( + return_values={"output": "test output"}, + log="test log", + ), + } + + +@pytest.fixture +def test_retry_state(): + """Create test retry state for callback testing.""" + return type("RetryState", (), { + "attempt_number": 2, + "outcome": type("Outcome", (), { + "exception": lambda: Exception("test retry error"), + })(), + })() \ No newline at end of file diff --git a/tests/handlers/test_langchain_callback.py b/tests/handlers/test_langchain_callback.py new file mode 100644 index 000000000..eab8918d0 --- /dev/null +++ b/tests/handlers/test_langchain_callback.py @@ -0,0 +1,923 @@ +"""Test suite for Langchain callback handlers. + +This test suite verifies the functionality of both synchronous and asynchronous +Langchain callback handlers. It tests the following aspects: + +1. Basic Functionality: + - Handler initialization and configuration + - Span creation and management + - Attribute recording + - Error handling + +2. LLM Operations: + - LLM start/end events + - Token streaming + - Error handling + - Response processing + +3. Chat Model Operations: + - Chat model start/end events + - Message handling + - Response processing + +4. Chain Operations: + - Chain start/end events + - Input/output handling + - Error propagation + +5. Tool Operations: + - Tool start/end events + - Input/output recording + - Error handling + +6. Retriever Operations: + - Retriever start/end events + - Query handling + - Document processing + +7. Agent Operations: + - Agent action events + - Tool usage tracking + - Finish event handling + +8. Error Scenarios: + - Exception handling + - Error propagation + - Span error status + +9. Async Functionality: + - Async handler initialization + - Async event handling + - Async error handling + +10. Edge Cases: + - Missing run IDs + - Invalid inputs + - Stream handling + - Retry scenarios + +The tests use mock objects to simulate Langchain operations and verify that +the handlers correctly create and manage OpenTelemetry spans with appropriate +attributes and error handling. +""" + +import asyncio +from typing import Dict, Any, List +from uuid import UUID, uuid4 +import pytest +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider, ReadableSpan +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.trace import Status, StatusCode, SpanKind +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +from langchain_core.agents import AgentFinish, AgentAction +from langchain_core.documents import Document +from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResult, Generation +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, AIMessageChunk +from tenacity import RetryCallState + +from agentops import init +from agentops.sdk.core import TracingCore +from agentops.integrations.langchain.callback_handler import ( + LangchainCallbackHandler, + AsyncLangchainCallbackHandler, +) +from agentops.semconv import SpanKind +from agentops.semconv.span_attributes import SpanAttributes +from agentops.semconv.langchain_attributes import LangchainAttributes + + +pytestmark = pytest.mark.asyncio + + +def get_model_from_kwargs(kwargs: dict) -> str: + """Extract model name from kwargs.""" + if "model" in kwargs.get("invocation_params", {}): + return kwargs["invocation_params"]["model"] + elif "_type" in kwargs.get("invocation_params", {}): + return kwargs["invocation_params"]["_type"] + return "unknown_model" + + +@contextmanager +def _create_as_current_span( + name: str, + kind: SpanKind, + attributes: Dict[str, Any] = None, +): + """Create a span and set it as the current span.""" + tracer = trace.get_tracer(__name__) + with tracer.start_as_current_span( + name, + kind=kind, + attributes=attributes or {}, + ) as span: + yield span + + +@pytest.fixture(autouse=True) +def setup_agentops(): + """Initialize AgentOps client for testing.""" + init(api_key="test-api-key") + yield + # Cleanup will be handled by the test framework + + +@pytest.fixture +def tracer_provider(): + """Create a tracer provider with an in-memory exporter for testing.""" + provider = TracerProvider() + exporter = InMemorySpanExporter() + processor = SimpleSpanProcessor(exporter) + provider.add_span_processor(processor) + return provider, exporter + + +@pytest.fixture +def tracing_core(): + """Initialize TracingCore for testing.""" + core = TracingCore.get_instance() + core.initialize( + service_name="test_service", + ) + yield core + core.shutdown() + + +@pytest.fixture +def mock_client(): + """Create a mock AgentOps client.""" + with patch("agentops.Client") as mock: + client_instance = MagicMock() + mock.return_value = client_instance + client_instance.configure.return_value = None + client_instance.init.return_value = None + client_instance.current_session_ids = ["test-session-id"] + yield client_instance + + +@pytest.fixture +def callback_handler(mock_client): + """Create a callback handler with mocked client.""" + return LangchainCallbackHandler() + + +@pytest.fixture +def async_callback_handler(mock_client): + """Create an async callback handler with mocked client.""" + return AsyncLangchainCallbackHandler() + + +@pytest.fixture +def test_run_ids(): + """Generate test run IDs.""" + return { + "run_id": UUID("12345678-1234-5678-1234-567812345678"), + "parent_run_id": UUID("87654321-4321-8765-4321-876543210987"), + } + + +@pytest.fixture +def test_llm_inputs(): + """Generate test LLM inputs.""" + return { + "serialized": {"name": "test-llm"}, + "prompts": ["test prompt"], + "invocation_params": {"model": "test-model"}, + } + + +@pytest.fixture +def test_chain_inputs(): + """Generate test chain inputs.""" + return { + "serialized": {"name": "test-chain"}, + "inputs": {"input": "test input"}, + } + + +@pytest.fixture +def test_tool_inputs(): + """Generate test tool inputs.""" + return { + "serialized": {"name": "test-tool"}, + "input_str": "test input", + "inputs": {"input": "test input"}, + } + + +@pytest.fixture +def test_agent_inputs(): + """Generate test agent inputs.""" + return { + "action": AgentAction( + tool="test-tool", + tool_input="test input", + log="test log", + ), + "finish": AgentFinish( + return_values={"output": "test output"}, + log="test log", + ), + } + + +@pytest.fixture +def test_retry_state(): + """Generate test retry state.""" + state = MagicMock(spec=RetryCallState) + state.attempt_number = 1 + state.outcome = MagicMock() + state.outcome.exception.return_value = Exception("test error") + return state + + +def test_llm_events(callback_handler, test_run_ids, test_llm_inputs): + """Test LLM events.""" + # Test LLM start + callback_handler.on_llm_start( + **test_llm_inputs, + **test_run_ids, + ) + + # Test LLM end + response = LLMResult( + generations=[[Generation(text="test response")]], + llm_output={"token_usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}}, + ) + callback_handler.on_llm_end( + response=response, + **test_run_ids, + ) + + +def test_chain_events(callback_handler, test_run_ids, test_chain_inputs): + """Test chain events.""" + # Test chain start + callback_handler.on_chain_start( + **test_chain_inputs, + **test_run_ids, + ) + + # Test chain end + callback_handler.on_chain_end( + outputs={"output": "test output"}, + **test_run_ids, + ) + + +def test_tool_events(callback_handler, test_run_ids, test_tool_inputs): + """Test tool events.""" + # Test tool start + callback_handler.on_tool_start( + **test_tool_inputs, + **test_run_ids, + ) + + # Test tool end + callback_handler.on_tool_end( + output="test output", + **test_run_ids, + ) + + +def test_agent_events(callback_handler, test_run_ids, test_agent_inputs): + """Test agent events.""" + # Test agent action + callback_handler.on_agent_action( + action=test_agent_inputs["action"], + **test_run_ids, + ) + + # Test agent finish + callback_handler.on_agent_finish( + finish=test_agent_inputs["finish"], + **test_run_ids, + ) + + +def test_retry_events(callback_handler, test_run_ids, test_retry_state): + """Test retry events.""" + callback_handler.on_retry( + retry_state=test_retry_state, + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_llm_events(async_callback_handler, test_run_ids, test_llm_inputs): + """Test async LLM events.""" + # Test LLM start + await async_callback_handler.on_llm_start( + **test_llm_inputs, + **test_run_ids, + ) + + # Test LLM end + response = LLMResult( + generations=[[Generation(text="test response")]], + llm_output={"token_usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}}, + ) + await async_callback_handler.on_llm_end( + response=response, + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_chain_events(async_callback_handler, test_run_ids, test_chain_inputs): + """Test async chain events.""" + # Test chain start + await async_callback_handler.on_chain_start( + **test_chain_inputs, + **test_run_ids, + ) + + # Test chain end + await async_callback_handler.on_chain_end( + outputs={"output": "test output"}, + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_tool_events(async_callback_handler, test_run_ids, test_tool_inputs): + """Test async tool events.""" + # Test tool start + await async_callback_handler.on_tool_start( + **test_tool_inputs, + **test_run_ids, + ) + + # Test tool end + await async_callback_handler.on_tool_end( + output="test output", + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_agent_events(async_callback_handler, test_run_ids, test_agent_inputs): + """Test async agent events.""" + # Test agent action + await async_callback_handler.on_agent_action( + action=test_agent_inputs["action"], + **test_run_ids, + ) + + # Test agent finish + await async_callback_handler.on_agent_finish( + finish=test_agent_inputs["finish"], + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_retry_events(async_callback_handler, test_run_ids, test_retry_state): + """Test async retry events.""" + await async_callback_handler.on_retry( + retry_state=test_retry_state, + **test_run_ids, + ) + + +@pytest.fixture +def test_llm_responses(): + """Generate test LLM responses.""" + return { + "text_response": LLMResult( + generations=[[Generation(text="test response")]], + llm_output={"token_usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}}, + ), + "message_response": LLMResult( + generations=[[ChatGenerationChunk(message=AIMessageChunk(content="test message"))]], + llm_output={"token_usage": {"prompt_tokens": 5, "completion_tokens": 15, "total_tokens": 20}}, + ), + "empty_response": LLMResult( + generations=[], + llm_output=None, + ), + "error_response": LLMResult( + generations=[[Generation(text="error response")]], + llm_output={"error": "test error"}, + ), + } + + +@pytest.fixture +def test_chain_outputs(): + """Generate test chain outputs.""" + return { + "simple_output": {"output": "test output"}, + "complex_output": {"output": {"nested": "value", "list": [1, 2, 3]}}, + "error_output": {"error": "test error"}, + } + + +@pytest.fixture +def test_tool_outputs(): + """Generate test tool outputs.""" + return { + "success_output": "test output", + "exception_output": "_Exception", + "error_output": "error: test error", + } + + +@pytest.fixture +def test_agent_sequences(): + """Generate test agent action sequences.""" + return { + "single_action": [ + AgentAction(tool="tool1", tool_input="input1", log="log1"), + AgentFinish(return_values={"output": "output1"}, log="finish1"), + ], + "multiple_actions": [ + AgentAction(tool="tool1", tool_input="input1", log="log1"), + AgentAction(tool="tool2", tool_input="input2", log="log2"), + AgentFinish(return_values={"output": "output2"}, log="finish2"), + ], + "error_action": [ + AgentAction(tool="error_tool", tool_input="error_input", log="error_log"), + AgentFinish(return_values={"error": "test error"}, log="error_finish"), + ], + } + + +def test_llm_events_with_different_response_types(callback_handler, test_run_ids, test_llm_inputs, test_llm_responses): + """Test LLM event handling with various response types and scenarios. + + This test verifies that the handler correctly processes: + - LLM start events with different input configurations + - LLM end events with different response types: + * Text-based responses + * Message-based responses (using AIMessageChunk) + * Empty responses + * Error responses + - Streaming token updates + - Error handling scenarios + """ + # Test LLM start + callback_handler.on_llm_start( + **test_llm_inputs, + **test_run_ids, + ) + + # Test different response types + callback_handler.on_llm_end( + response=test_llm_responses["text_response"], + **test_run_ids, + ) + + # Test message-based response + callback_handler.on_llm_end( + response=test_llm_responses["message_response"], + **test_run_ids, + ) + + # Test empty response + callback_handler.on_llm_end( + response=test_llm_responses["empty_response"], + **test_run_ids, + ) + + # Test error response + callback_handler.on_llm_end( + response=test_llm_responses["error_response"], + **test_run_ids, + ) + + # Test streaming tokens + callback_handler.on_llm_new_token( + token="test", + **test_run_ids, + ) + callback_handler.on_llm_new_token( + token=" token", + **test_run_ids, + ) + + # Test LLM error + callback_handler.on_llm_error( + error=Exception("test error"), + **test_run_ids, + ) + + +def test_chain_events_with_metadata_and_outputs(callback_handler, test_run_ids, test_chain_inputs, test_chain_outputs): + """Test chain event handling with metadata and various output formats. + + This test verifies that the handler correctly processes: + - Chain start events with metadata + - Chain end events with different output formats: + * Simple key-value outputs + * Complex nested outputs + * Error outputs + - Chain error handling + """ + # Test chain start with metadata + callback_handler.on_chain_start( + **test_chain_inputs, + metadata={"test": "metadata"}, + **test_run_ids, + ) + + # Test different output types + callback_handler.on_chain_end( + outputs=test_chain_outputs["simple_output"], + **test_run_ids, + ) + + callback_handler.on_chain_end( + outputs=test_chain_outputs["complex_output"], + **test_run_ids, + ) + + # Test chain error + callback_handler.on_chain_error( + error=Exception("test error"), + **test_run_ids, + ) + + +def test_tool_events_with_exceptions_and_errors(callback_handler, test_run_ids, test_tool_inputs, test_tool_outputs): + """Test tool event handling with various input/output types and error scenarios. + + This test verifies that the handler correctly processes: + - Tool start events with different input configurations + - Tool end events with different output types: + * Successful outputs + * Exception outputs + * Error outputs + - Tool error handling + """ + # Test tool start with different inputs + callback_handler.on_tool_start( + **{k: v for k, v in test_tool_inputs.items() if k != 'inputs'}, + **test_run_ids, + ) + + # Test different output types + callback_handler.on_tool_end( + output=test_tool_outputs["success_output"], + **test_run_ids, + ) + + # Test exception tool + callback_handler.on_tool_end( + output=test_tool_outputs["exception_output"], + name="_Exception", + **test_run_ids, + ) + + # Test tool error + callback_handler.on_tool_error( + error=Exception("test error"), + **test_run_ids, + ) + + +def test_agent_events_with_action_sequences(callback_handler, test_run_ids, test_agent_sequences): + """Test agent event handling with different action sequences and scenarios. + + This test verifies that the handler correctly processes: + - Single action sequences (action + finish) + - Multiple action sequences (multiple actions + finish) + - Error action sequences + - Different types of agent actions and finishes + """ + # Test single action sequence + for action in test_agent_sequences["single_action"]: + if isinstance(action, AgentAction): + callback_handler.on_agent_action( + action=action, + **test_run_ids, + ) + else: + callback_handler.on_agent_finish( + finish=action, + **test_run_ids, + ) + + # Test multiple actions sequence + for action in test_agent_sequences["multiple_actions"]: + if isinstance(action, AgentAction): + callback_handler.on_agent_action( + action=action, + **test_run_ids, + ) + else: + callback_handler.on_agent_finish( + finish=action, + **test_run_ids, + ) + + # Test error sequence + for action in test_agent_sequences["error_action"]: + if isinstance(action, AgentAction): + callback_handler.on_agent_action( + action=action, + **test_run_ids, + ) + else: + callback_handler.on_agent_finish( + finish=action, + **test_run_ids, + ) + + +def test_retriever_events_with_documents(callback_handler, test_run_ids): + """Test retriever event handling with document processing. + + This test verifies that the handler correctly processes: + - Retriever start events with query information + - Retriever end events with document results + - Retriever error handling + """ + # Test retriever start + callback_handler.on_retriever_start( + serialized={"name": "test-retriever"}, + query="test query", + **test_run_ids, + ) + + # Test retriever end + callback_handler.on_retriever_end( + documents=[Document(page_content="test content")], + **test_run_ids, + ) + + # Test retriever error + callback_handler.on_retriever_error( + error=Exception("test error"), + **test_run_ids, + ) + + +def test_retry_events_with_different_states(callback_handler, test_run_ids, test_retry_state): + """Test retry event handling with different retry states and error types. + + This test verifies that the handler correctly processes: + - Retry events with standard retry states + - Retry events with different error types + - Retry state information tracking + """ + # Test retry with different states + callback_handler.on_retry( + retry_state=test_retry_state, + **test_run_ids, + ) + + # Test retry with different error types + error_state = MagicMock(spec=RetryCallState) + error_state.attempt_number = 2 + error_state.outcome = MagicMock() + error_state.outcome.exception.return_value = ValueError("test error") + callback_handler.on_retry( + retry_state=error_state, + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_llm_events_with_different_response_types(async_callback_handler, test_run_ids, test_llm_inputs, test_llm_responses): + """Test async LLM event handling with various response types and scenarios. + + This test verifies that the async handler correctly processes: + - Async LLM start events with different input configurations + - Async LLM end events with different response types: + * Text-based responses + * Message-based responses (using AIMessageChunk) + * Empty responses + * Error responses + - Async streaming token updates + - Async error handling scenarios + """ + # Test LLM start + await async_callback_handler.on_llm_start( + **test_llm_inputs, + **test_run_ids, + ) + + # Test different response types + await async_callback_handler.on_llm_end( + response=test_llm_responses["text_response"], + **test_run_ids, + ) + + # Test message-based response + await async_callback_handler.on_llm_end( + response=test_llm_responses["message_response"], + **test_run_ids, + ) + + # Test empty response + await async_callback_handler.on_llm_end( + response=test_llm_responses["empty_response"], + **test_run_ids, + ) + + # Test error response + await async_callback_handler.on_llm_end( + response=test_llm_responses["error_response"], + **test_run_ids, + ) + + # Test streaming tokens + await async_callback_handler.on_llm_new_token( + token="test", + **test_run_ids, + ) + await async_callback_handler.on_llm_new_token( + token=" token", + **test_run_ids, + ) + + # Test LLM error + await async_callback_handler.on_llm_error( + error=Exception("test error"), + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_chain_events_with_metadata_and_outputs(async_callback_handler, test_run_ids, test_chain_inputs, test_chain_outputs): + """Test async chain event handling with metadata and various output formats. + + This test verifies that the async handler correctly processes: + - Async chain start events with metadata + - Async chain end events with different output formats: + * Simple key-value outputs + * Complex nested outputs + * Error outputs + - Async chain error handling + """ + # Test chain start with metadata + await async_callback_handler.on_chain_start( + **test_chain_inputs, + metadata={"test": "metadata"}, + **test_run_ids, + ) + + # Test different output types + await async_callback_handler.on_chain_end( + outputs=test_chain_outputs["simple_output"], + **test_run_ids, + ) + + await async_callback_handler.on_chain_end( + outputs=test_chain_outputs["complex_output"], + **test_run_ids, + ) + + # Test chain error + await async_callback_handler.on_chain_error( + error=Exception("test error"), + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_tool_events_with_exceptions_and_errors(async_callback_handler, test_run_ids, test_tool_inputs, test_tool_outputs): + """Test async tool event handling with various input/output types and error scenarios. + + This test verifies that the async handler correctly processes: + - Async tool start events with different input configurations + - Async tool end events with different output types: + * Successful outputs + * Exception outputs + * Error outputs + - Async tool error handling + """ + # Test tool start with different inputs + await async_callback_handler.on_tool_start( + **{k: v for k, v in test_tool_inputs.items() if k != 'inputs'}, + **test_run_ids, + ) + + # Test different output types + await async_callback_handler.on_tool_end( + output=test_tool_outputs["success_output"], + **test_run_ids, + ) + + # Test exception tool + await async_callback_handler.on_tool_end( + output=test_tool_outputs["exception_output"], + name="_Exception", + **test_run_ids, + ) + + # Test tool error + await async_callback_handler.on_tool_error( + error=Exception("test error"), + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_agent_events_with_action_sequences(async_callback_handler, test_run_ids, test_agent_sequences): + """Test async agent event handling with different action sequences and scenarios. + + This test verifies that the async handler correctly processes: + - Async single action sequences (action + finish) + - Async multiple action sequences (multiple actions + finish) + - Async error action sequences + - Async different types of agent actions and finishes + """ + # Test single action sequence + for action in test_agent_sequences["single_action"]: + if isinstance(action, AgentAction): + await async_callback_handler.on_agent_action( + action=action, + **test_run_ids, + ) + else: + await async_callback_handler.on_agent_finish( + finish=action, + **test_run_ids, + ) + + # Test multiple actions sequence + for action in test_agent_sequences["multiple_actions"]: + if isinstance(action, AgentAction): + await async_callback_handler.on_agent_action( + action=action, + **test_run_ids, + ) + else: + await async_callback_handler.on_agent_finish( + finish=action, + **test_run_ids, + ) + + # Test error sequence + for action in test_agent_sequences["error_action"]: + if isinstance(action, AgentAction): + await async_callback_handler.on_agent_action( + action=action, + **test_run_ids, + ) + else: + await async_callback_handler.on_agent_finish( + finish=action, + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_retriever_events_with_documents(async_callback_handler, test_run_ids): + """Test async retriever event handling with document processing. + + This test verifies that the async handler correctly processes: + - Async retriever start events with query information + - Async retriever end events with document results + - Async retriever error handling + """ + # Test retriever start + await async_callback_handler.on_retriever_start( + serialized={"name": "test-retriever"}, + query="test query", + **test_run_ids, + ) + + # Test retriever end + await async_callback_handler.on_retriever_end( + documents=[Document(page_content="test content")], + **test_run_ids, + ) + + # Test retriever error + await async_callback_handler.on_retriever_error( + error=Exception("test error"), + **test_run_ids, + ) + + +@pytest.mark.asyncio +async def test_async_retry_events_with_different_states(async_callback_handler, test_run_ids, test_retry_state): + """Test async retry event handling with different retry states and error types. + + This test verifies that the async handler correctly processes: + - Async retry events with standard retry states + - Async retry events with different error types + - Async retry state information tracking + """ + # Test retry with different states + await async_callback_handler.on_retry( + retry_state=test_retry_state, + **test_run_ids, + ) + + # Test retry with different error types + error_state = MagicMock(spec=RetryCallState) + error_state.attempt_number = 2 + error_state.outcome = MagicMock() + error_state.outcome.exception.return_value = ValueError("test error") + await async_callback_handler.on_retry( + retry_state=error_state, + **test_run_ids, + ) \ No newline at end of file From 009e413cd527f423b1864a2f90a0f4e457b0af8a Mon Sep 17 00:00:00 2001 From: Dwij Patel Date: Sat, 22 Mar 2025 05:36:00 +0530 Subject: [PATCH 2/5] Updated Langchain callback handler and related documentation. --- agentops/integrations/langchain/README.md | 158 --- .../langchain/callback_handler.py | 1246 ----------------- agentops/sdk/callbacks/langchain/README.md | 59 + agentops/sdk/callbacks/langchain/__init__.py | 15 + agentops/sdk/callbacks/langchain/callback.py | 896 ++++++++++++ agentops/sdk/callbacks/langchain/utils.py | 84 ++ agentops/semconv/__init__.py | 6 +- agentops/semconv/langchain.py | 54 + agentops/semconv/langchain_attributes.py | 37 - agentops/semconv/span_kinds.py | 3 +- .../openai_assistants_example.ipynb | 880 +++++++++++- tests/handlers/conftest.py | 154 -- tests/handlers/test_langchain_callback.py | 923 ------------ 13 files changed, 1919 insertions(+), 2596 deletions(-) delete mode 100644 agentops/integrations/langchain/README.md delete mode 100644 agentops/integrations/langchain/callback_handler.py create mode 100644 agentops/sdk/callbacks/langchain/README.md create mode 100644 agentops/sdk/callbacks/langchain/__init__.py create mode 100644 agentops/sdk/callbacks/langchain/callback.py create mode 100644 agentops/sdk/callbacks/langchain/utils.py create mode 100644 agentops/semconv/langchain.py delete mode 100644 agentops/semconv/langchain_attributes.py delete mode 100644 tests/handlers/conftest.py delete mode 100644 tests/handlers/test_langchain_callback.py diff --git a/agentops/integrations/langchain/README.md b/agentops/integrations/langchain/README.md deleted file mode 100644 index cad95559e..000000000 --- a/agentops/integrations/langchain/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# Langchain Callback Handler for AgentOps - -This module provides OpenTelemetry-based callback handlers for Langchain that integrate with AgentOps for comprehensive tracing and monitoring of Langchain applications. - -## Features - -- **Comprehensive Event Tracking**: Monitors all major Langchain operations: - - LLM calls (including streaming responses) - - Chat model interactions - - Chain executions - - Tool usage - - Retriever operations - - Agent actions - - Retry attempts - -- **Dual Mode Support**: - - Synchronous operations (`LangchainCallbackHandler`) - - Asynchronous operations (`AsyncLangchainCallbackHandler`) - -- **Detailed Span Attributes**: - - Operation inputs and outputs - - Token usage statistics - - Error details and stack traces - - Model information - - Tool parameters and results - -## Usage - -### Basic Usage - -```python -from agentops.integrations.langchain.callback_handler import LangchainCallbackHandler - -# Initialize the handler -handler = LangchainCallbackHandler( - api_key="your-api-key" -) - -# Use with Langchain -from langchain.llms import OpenAI -from langchain.chains import LLMChain - -llm = OpenAI( - callbacks=[handler], - temperature=0.7 -) - -chain = LLMChain( - llm=llm, - prompt=your_prompt, - callbacks=[handler] -) -``` - -### Async Usage - -```python -from agentops.integrations.langchain.callback_handler import AsyncLangchainCallbackHandler - -# Initialize the async handler -handler = AsyncLangchainCallbackHandler( - api_key="your-api-key" -) - -# Use with async Langchain -from langchain.llms import OpenAI -from langchain.chains import LLMChain - -llm = OpenAI( - callbacks=[handler], - temperature=0.7 -) - -chain = LLMChain( - llm=llm, - prompt=your_prompt, - callbacks=[handler] -) -``` - -## Span Types and Attributes - -### LLM Spans -- **Name**: "llm" -- **Attributes**: - - `run_id`: Unique identifier for the LLM run - - `model`: Name of the LLM model - - `prompt`: Input prompt - - `response`: Generated response - - `token_usage`: Token usage statistics - -### Chat Model Spans -- **Name**: "chat_model" -- **Attributes**: - - `run_id`: Unique identifier for the chat run - - `model`: Name of the chat model - - `messages`: Conversation history - - `response`: Generated response - -### Chain Spans -- **Name**: "chain" -- **Attributes**: - - `run_id`: Unique identifier for the chain run - - `chain_name`: Name of the chain - - `inputs`: Chain input parameters - - `outputs`: Chain output results - -### Tool Spans -- **Name**: "tool" -- **Attributes**: - - `run_id`: Unique identifier for the tool run - - `tool_name`: Name of the tool - - `tool_input`: Tool input parameters - - `tool_output`: Tool output results - -### Retriever Spans -- **Name**: "retriever" -- **Attributes**: - - `run_id`: Unique identifier for the retriever run - - `retriever_name`: Name of the retriever - - `query`: Search query - - `documents`: Retrieved documents - -### Agent Spans -- **Name**: "agent_action" -- **Attributes**: - - `run_id`: Unique identifier for the agent run - - `agent_name`: Name of the agent - - `tool_input`: Tool input parameters - - `tool_log`: Agent's reasoning log - -## Error Handling - -The handler provides comprehensive error tracking: -- Records error type and message -- Preserves error context and stack traces -- Updates span status to ERROR -- Maintains error details in span attributes - -## Testing - -The handler includes comprehensive tests in `test_callback_handler.py` that verify: -- Basic functionality of both sync and async handlers -- Error handling and recovery -- Span creation and attribute recording -- Token usage tracking -- Streaming response handling -- Chain and tool execution tracking -- Agent action monitoring - -To run the tests: -```bash -pytest tests/handlers/langchain/test_callback_handler.py -v -``` - -## Contributing - -Contributions are welcome! Please feel free to submit a Pull Request. \ No newline at end of file diff --git a/agentops/integrations/langchain/callback_handler.py b/agentops/integrations/langchain/callback_handler.py deleted file mode 100644 index 35bff44eb..000000000 --- a/agentops/integrations/langchain/callback_handler.py +++ /dev/null @@ -1,1246 +0,0 @@ -"""Langchain callback handler using OpenTelemetry. - -This module provides callback handlers for Langchain that integrate with OpenTelemetry -for tracing and monitoring. It supports both synchronous and asynchronous operations, -tracking various events in the Langchain execution flow including LLM calls, tool usage, -chain execution, and agent actions. -""" - -from typing import Dict, Any, List, Optional, Sequence, Union -from uuid import UUID -import logging -import os -from collections import defaultdict - -from tenacity import RetryCallState -from langchain_core.agents import AgentFinish, AgentAction -from langchain_core.documents import Document -from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResult, Generation -from langchain_core.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler -from langchain_core.messages import BaseMessage, AIMessage, AIMessageChunk -from opentelemetry.trace import Status, StatusCode - -from agentops.sdk.decorators.utility import _create_as_current_span -from agentops.semconv import SpanKind -from agentops.semconv.langchain_attributes import LangchainAttributes -from agentops.logging import logger - -def get_model_from_kwargs(kwargs: Any) -> str: - """Extract model name from kwargs. - - This function attempts to get the model name from the invocation parameters - in the kwargs dictionary. It checks for both 'model' and '_type' keys in the - invocation_params dictionary. - - Args: - kwargs: Dictionary containing invocation parameters - - Returns: - str: The model name if found, otherwise 'unknown_model' - """ - if "model" in kwargs.get("invocation_params", {}): - return kwargs["invocation_params"]["model"] - elif "_type" in kwargs.get("invocation_params", {}): - return kwargs["invocation_params"]["_type"] - return "unknown_model" - -def _create_span_attributes( - run_id: UUID, - parent_run_id: Optional[UUID] = None, - tags: Optional[List[str]] = None, - metadata: Optional[Dict[str, Any]] = None, - **kwargs: Any, -) -> Dict[str, Any]: - """Create common span attributes. - - This function creates a standardized set of attributes for OpenTelemetry spans. - It includes common attributes like run_id, parent_run_id, tags, and metadata, - along with any additional attributes passed in kwargs. - - Args: - run_id: Unique identifier for the current run - parent_run_id: Optional identifier for the parent run - tags: Optional list of tags for the span - metadata: Optional dictionary of metadata - **kwargs: Additional attributes to include - - Returns: - Dict[str, Any]: Dictionary of span attributes - """ - return { - LangchainAttributes.RUN_ID: str(run_id), - LangchainAttributes.PARENT_RUN_ID: str(parent_run_id) if parent_run_id else "", - LangchainAttributes.TAGS: str(tags or []), - LangchainAttributes.METADATA: str(metadata or {}), - **kwargs, - } - -def _handle_span_error(span: Any, error: Exception, **kwargs: Any) -> None: - """Handle span error consistently. - - This function provides a standardized way to handle errors in spans, - setting appropriate error attributes and status codes. - - Args: - span: The OpenTelemetry span to update - error: The exception that occurred - **kwargs: Additional context for the error - """ - span.set_status(Status(StatusCode.ERROR)) - span.set_attribute(LangchainAttributes.ERROR_TYPE, type(error).__name__) - span.set_attribute(LangchainAttributes.ERROR_MESSAGE, str(error)) - span.set_attribute(LangchainAttributes.ERROR_DETAILS, { - "run_id": kwargs.get("run_id"), - "parent_run_id": kwargs.get("parent_run_id"), - "kwargs": str(kwargs) - }) - span.end() - -class BaseLangchainHandler: - """Base class for Langchain handlers with common functionality. - - This class provides shared functionality for both synchronous and asynchronous - Langchain callback handlers, including initialization, span tracking, and - common utility methods. - """ - - def __init__( - self, - api_key: Optional[str] = None, - endpoint: Optional[str] = None, - max_wait_time: Optional[int] = None, - max_queue_size: Optional[int] = None, - default_tags: List[str] = None, - ): - """Initialize the handler with configuration options. - - Args: - api_key: Optional API key for AgentOps - endpoint: Optional endpoint URL for AgentOps - max_wait_time: Optional maximum wait time for operations - max_queue_size: Optional maximum size of the operation queue - default_tags: Optional list of default tags for spans - """ - # Set up logging - logging_level = os.getenv("AGENTOPS_LOGGING_LEVEL") - log_levels = { - "CRITICAL": logging.CRITICAL, - "ERROR": logging.ERROR, - "INFO": logging.INFO, - "WARNING": logging.WARNING, - "DEBUG": logging.DEBUG, - } - logger.setLevel(log_levels.get(logging_level or "INFO", "INFO")) - - # Initialize AgentOps client - from agentops import Client - self.client = Client() - self.client.configure( - api_key=api_key or "test-api-key", - endpoint=endpoint, - max_wait_time=max_wait_time, - max_queue_size=max_queue_size, - default_tags=default_tags or ["langchain", "sync" if isinstance(self, LangchainCallbackHandler) else "async"], - ) - self.client.init() - - # Initialize span tracking - self._llm_spans: Dict[str, Any] = {} - self._tool_spans: Dict[str, Any] = {} - self._chain_spans: Dict[str, Any] = {} - self._retriever_spans: Dict[str, Any] = {} - self._agent_actions: Dict[UUID, List[Any]] = defaultdict(list) - - def _handle_llm_response(self, span: Any, response: LLMResult) -> None: - """Handle LLM response and set appropriate attributes. - - This method processes an LLM response and updates the span with relevant - information including the response text, token usage, and other metadata. - - Args: - span: The OpenTelemetry span to update - response: The LLM response to process - """ - if not hasattr(response, "generations"): - return - - for generation_list in response.generations: - for generation in generation_list: - if isinstance(generation, Generation): - if generation.text: - span.set_attribute(LangchainAttributes.RESPONSE, generation.text) - elif hasattr(generation, "message"): - if isinstance(generation.message, AIMessage) and generation.message.content: - span.set_attribute(LangchainAttributes.RESPONSE, generation.message.content) - elif isinstance(generation.message, AIMessageChunk) and generation.message.content: - current_completion = span.get_attribute(LangchainAttributes.RESPONSE) or "" - span.set_attribute(LangchainAttributes.RESPONSE, current_completion + generation.message.content) - - # Handle token usage - if hasattr(response, "llm_output") and isinstance(response.llm_output, dict): - token_usage = response.llm_output.get("token_usage", {}) - if isinstance(token_usage, dict): - span.set_attribute(LangchainAttributes.PROMPT_TOKENS, token_usage.get("prompt_tokens", 0)) - span.set_attribute(LangchainAttributes.COMPLETION_TOKENS, token_usage.get("completion_tokens", 0)) - span.set_attribute(LangchainAttributes.TOTAL_TOKENS, token_usage.get("total_tokens", 0)) - - def _handle_agent_finish(self, run_id: UUID, finish: AgentFinish) -> None: - """Handle agent finish event and update spans. - - This method processes the completion of an agent's task, updating the - relevant spans with final outputs and status. - - Args: - run_id: Unique identifier for the agent run - finish: The AgentFinish event containing final outputs - """ - agent_spans = self._agent_actions.get(run_id, []) - if not agent_spans: - return - - last_span = agent_spans[-1] - last_span.set_attribute(LangchainAttributes.OUTPUTS, str(finish.return_values)) - last_span.set_attribute(LangchainAttributes.TOOL_LOG, finish.log) - last_span.set_status(Status(StatusCode.OK)) - last_span.end() - - # Record all agent actions - for span in agent_spans[:-1]: - span.set_status(Status(StatusCode.OK)) - span.end() - - self._agent_actions.pop(run_id, None) - - @property - def current_session_ids(self) -> List[str]: - """Get current session IDs. - - Returns: - List[str]: List of current session IDs from the AgentOps client - """ - return self.client.current_session_ids - -class LangchainCallbackHandler(BaseLangchainHandler, BaseCallbackHandler): - """Callback handler for Langchain using OpenTelemetry. - - This class implements the synchronous callback interface for Langchain, - tracking various events in the execution flow and creating appropriate - OpenTelemetry spans for monitoring and debugging. - """ - - def on_llm_start( - self, - serialized: Dict[str, Any], - prompts: List[str], - **kwargs: Any, - ) -> None: - """Handle LLM start event. - - This method is called when an LLM operation begins. It creates a new span - to track the operation and stores relevant information about the model and - input prompts. - - Args: - serialized: Serialized information about the LLM - prompts: List of input prompts - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="llm", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - model=get_model_from_kwargs(kwargs), - prompt=prompts[0] if prompts else "", - **kwargs, - ), - ) as span: - self._llm_spans[run_id] = span - - except Exception as e: - logger.error(f"Error in on_llm_start: {str(e)}") - - def on_llm_end( - self, - response: LLMResult, - **kwargs: Any, - ) -> None: - """Handle LLM end event. - - This method is called when an LLM operation completes. It processes the - response, updates the span with results, and handles any errors that - occurred during the operation. - - Args: - response: The LLM result containing generations and metadata - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - span = self._llm_spans.get(run_id) - if not span: - return - - try: - self._handle_llm_response(span, response) - span.set_status(Status(StatusCode.OK)) - except Exception as e: - logger.error(f"Error in on_llm_end: {str(e)}") - _handle_span_error(span, e, **kwargs) - finally: - span.end() - self._llm_spans.pop(run_id, None) - - def on_llm_error( - self, - error: BaseException, - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle LLM error event. - - This method is called when an error occurs during an LLM operation. - It updates the span with error information and marks it as failed. - - Args: - error: The exception that occurred - run_id: Unique identifier for the LLM run - **kwargs: Additional error context - """ - if str(run_id) in self._llm_spans: - span = self._llm_spans[str(run_id)] - _handle_span_error(span, error, run_id=run_id, **kwargs) - - def on_chat_model_start( - self, - serialized: Dict[str, Any], - messages: List[List[BaseMessage]], - **kwargs: Any, - ) -> None: - """Handle chat model start event. - - This method is called when a chat model operation begins. It creates a - new span to track the operation and stores information about the model - and input messages. - - Args: - serialized: Serialized information about the chat model - messages: List of message lists containing the conversation - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - parsed_messages = [ - {"role": message.type, "content": message.content} - for message in messages[0] - if message.type in ["system", "human"] - ] - - with _create_as_current_span( - name="chat_model", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - model=get_model_from_kwargs(kwargs), - messages=str(parsed_messages), - **kwargs, - ), - ) as span: - self._llm_spans[run_id] = span - - except Exception as e: - logger.error(f"Error in on_chat_model_start: {str(e)}") - - def on_chain_start( - self, - serialized: Dict[str, Any], - inputs: Dict[str, Any], - **kwargs: Any, - ) -> None: - """Handle chain start event. - - This method is called when a Langchain chain operation begins. It creates - a new span to track the chain execution and stores information about the - chain and its inputs. - - Args: - serialized: Serialized information about the chain - inputs: Dictionary of input values for the chain - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="chain", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - chain_name=serialized.get("name", "unknown"), - inputs=str(inputs or {}), - **kwargs, - ), - ) as span: - self._chain_spans[run_id] = span - - except Exception as e: - logger.error(f"Error in on_chain_start: {str(e)}") - - def on_chain_end( - self, - outputs: Dict[str, Any], - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle chain end event. - - This method is called when a Langchain chain operation completes. It - updates the span with the chain's outputs and marks it as successful. - - Args: - outputs: Dictionary of output values from the chain - run_id: Unique identifier for the chain run - **kwargs: Additional arguments - """ - if str(run_id) in self._chain_spans: - span = self._chain_spans[str(run_id)] - span.set_attribute(LangchainAttributes.CHAIN_OUTPUTS, str(outputs)) - span.end() - - def on_chain_error( - self, - error: BaseException, - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle chain error event. - - This method is called when an error occurs during a chain operation. - It updates the span with error information and marks it as failed. - - Args: - error: The exception that occurred - run_id: Unique identifier for the chain run - **kwargs: Additional error context - """ - if str(run_id) in self._chain_spans: - span = self._chain_spans[str(run_id)] - _handle_span_error(span, error, run_id=run_id, **kwargs) - - def on_tool_start( - self, - serialized: Dict[str, Any], - input_str: str, - **kwargs: Any, - ) -> None: - """Handle tool start event. - - This method is called when a tool operation begins. It creates a new span - to track the tool execution and stores information about the tool and its - inputs. - - Args: - serialized: Serialized information about the tool - input_str: String input for the tool - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="tool", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - tool_name=serialized.get("name", "unknown"), - tool_input=input_str, - inputs=str(kwargs.get("inputs", {})), - **kwargs, - ), - ) as span: - self._tool_spans[run_id] = span - - except Exception as e: - logger.error(f"Error in on_tool_start: {str(e)}") - - def on_tool_end( - self, - output: str, - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle tool end event. - - This method is called when a tool operation completes. It updates the span - with the tool's output and handles any errors that occurred during the - operation. - - Args: - output: String output from the tool - run_id: Unique identifier for the tool run - **kwargs: Additional arguments - """ - if str(run_id) in self._tool_spans: - span = self._tool_spans[str(run_id)] - span.set_attribute(LangchainAttributes.TOOL_OUTPUT, output) - - if kwargs.get("name") == "_Exception": - _handle_span_error(span, Exception(output), run_id=run_id, **kwargs) - else: - span.end() - - def on_tool_error( - self, - error: BaseException, - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle tool error event. - - This method is called when an error occurs during a tool operation. - It updates the span with error information and marks it as failed. - - Args: - error: The exception that occurred - run_id: Unique identifier for the tool run - **kwargs: Additional error context - """ - if str(run_id) in self._tool_spans: - span = self._tool_spans[str(run_id)] - _handle_span_error(span, error, run_id=run_id, **kwargs) - - def on_retriever_start( - self, - serialized: Dict[str, Any], - query: str, - **kwargs: Any, - ) -> None: - """Handle retriever start event. - - This method is called when a retriever operation begins. It creates a new - span to track the retrieval process and stores information about the - retriever and the query. - - Args: - serialized: Serialized information about the retriever - query: The search query string - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="retriever", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - retriever_name=serialized.get("name", "unknown"), - query=query, - **kwargs, - ), - ) as span: - self._retriever_spans[run_id] = span - - except Exception as e: - logger.error(f"Error in on_retriever_start: {str(e)}") - - def on_retriever_end( - self, - documents: Sequence[Document], - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle retriever end event. - - This method is called when a retriever operation completes. It updates - the span with the retrieved documents and marks it as successful. - - Args: - documents: Sequence of retrieved documents - run_id: Unique identifier for the retriever run - **kwargs: Additional arguments - """ - if str(run_id) in self._retriever_spans: - span = self._retriever_spans[str(run_id)] - span.set_attribute(LangchainAttributes.RETRIEVER_DOCUMENTS, str(documents)) - span.end() - - def on_retriever_error( - self, - error: BaseException, - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle retriever error event. - - This method is called when an error occurs during a retriever operation. - It updates the span with error information and marks it as failed. - - Args: - error: The exception that occurred - run_id: Unique identifier for the retriever run - **kwargs: Additional error context - """ - if str(run_id) in self._retriever_spans: - span = self._retriever_spans[str(run_id)] - _handle_span_error(span, error, run_id=run_id, **kwargs) - - def on_agent_action( - self, - action: AgentAction, - **kwargs: Any, - ) -> None: - """Handle agent action event. - - This method is called when an agent performs an action. It creates a new - span to track the action and stores information about the tool being used - and its inputs. - - Args: - action: The agent action containing tool and input information - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="agent_action", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - agent_name=action.tool, - tool_input=action.tool_input, - tool_log=action.log, - **kwargs, - ), - ) as span: - self._agent_actions[run_id].append(span) - - except Exception as e: - logger.error(f"Error in on_agent_action: {str(e)}") - - def on_agent_finish( - self, - finish: AgentFinish, - **kwargs: Any, - ) -> None: - """Handle agent finish event. - - This method is called when an agent completes its task. It updates all - relevant spans with final outputs and marks them as successful. - - Args: - finish: The AgentFinish event containing final outputs - **kwargs: Additional arguments including run_id - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - self._handle_agent_finish(run_id, finish) - except Exception as e: - logger.error(f"Error in on_agent_finish: {str(e)}") - - def on_retry( - self, - retry_state: RetryCallState, - **kwargs: Any, - ) -> None: - """Handle retry event. - - This method is called when an operation is being retried. It creates a - new span to track the retry attempt and stores information about the - error that caused the retry. - - Args: - retry_state: State information about the retry attempt - **kwargs: Additional arguments including run_id - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="retry", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - retry_attempt=retry_state.attempt_number, - error_type=type(retry_state.outcome.exception()).__name__, - error_message=str(retry_state.outcome.exception()), - **kwargs, - ), - ) as span: - span.set_status(Status(StatusCode.ERROR)) - span.end() - - except Exception as e: - logger.error(f"Error in on_retry: {str(e)}") - - def on_llm_new_token( - self, - token: str, - **kwargs: Any, - ) -> None: - """Handle new LLM token event. - - This method is called when a new token is generated during streaming - LLM responses. It updates the span with the accumulated response text. - - Args: - token: The new token generated - **kwargs: Additional arguments including run_id - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - span = self._llm_spans.get(run_id) - if not span: - return - - try: - current_completion = span.get_attribute(LangchainAttributes.RESPONSE) or "" - span.set_attribute(LangchainAttributes.RESPONSE, current_completion + token) - except Exception as e: - logger.error(f"Error in on_llm_new_token: {str(e)}") - - -class AsyncLangchainCallbackHandler(BaseLangchainHandler, AsyncCallbackHandler): - """Async callback handler for Langchain using OpenTelemetry. - - This class implements the asynchronous callback interface for Langchain, - providing the same functionality as the synchronous handler but with - async/await support for better performance in asynchronous environments. - """ - - async def on_llm_start( - self, - serialized: Dict[str, Any], - prompts: List[str], - **kwargs: Any, - ) -> None: - """Handle LLM start event asynchronously. - - This method is called when an LLM operation begins. It creates a new span - to track the operation and stores relevant information about the model and - input prompts. - - Args: - serialized: Serialized information about the LLM - prompts: List of input prompts - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="llm", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - model=get_model_from_kwargs(kwargs), - prompt=prompts[0] if prompts else "", - **kwargs, - ), - ) as span: - self._llm_spans[run_id] = span - - except Exception as e: - logger.error(f"Error in on_llm_start: {str(e)}") - - async def on_llm_end( - self, - response: LLMResult, - **kwargs: Any, - ) -> None: - """Handle LLM end event asynchronously. - - This method is called when an LLM operation completes. It processes the - response, updates the span with results, and handles any errors that - occurred during the operation. - - Args: - response: The LLM result containing generations and metadata - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - span = self._llm_spans.get(run_id) - if not span: - return - - try: - self._handle_llm_response(span, response) - span.set_status(Status(StatusCode.OK)) - except Exception as e: - logger.error(f"Error in on_llm_end: {str(e)}") - _handle_span_error(span, e, **kwargs) - finally: - span.end() - self._llm_spans.pop(run_id, None) - - async def on_llm_error( - self, - error: BaseException, - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle LLM error event asynchronously. - - This method is called when an error occurs during an LLM operation. - It updates the span with error information and marks it as failed. - - Args: - error: The exception that occurred - run_id: Unique identifier for the LLM run - **kwargs: Additional error context - """ - if str(run_id) in self._llm_spans: - span = self._llm_spans[str(run_id)] - _handle_span_error(span, error, run_id=run_id, **kwargs) - - async def on_chat_model_start( - self, - serialized: Dict[str, Any], - messages: List[List[BaseMessage]], - **kwargs: Any, - ) -> None: - """Handle chat model start event asynchronously. - - This method is called when a chat model operation begins. It creates a - new span to track the operation and stores information about the model - and input messages. - - Args: - serialized: Serialized information about the chat model - messages: List of message lists containing the conversation - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - parsed_messages = [ - {"role": message.type, "content": message.content} - for message in messages[0] - if message.type in ["system", "human"] - ] - - with _create_as_current_span( - name="chat_model", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - model=get_model_from_kwargs(kwargs), - messages=str(parsed_messages), - **kwargs, - ), - ) as span: - self._llm_spans[run_id] = span - - except Exception as e: - logger.error(f"Error in on_chat_model_start: {str(e)}") - - async def on_chain_start( - self, - serialized: Dict[str, Any], - inputs: Dict[str, Any], - **kwargs: Any, - ) -> None: - """Handle chain start event asynchronously. - - This method is called when a Langchain chain operation begins. It creates - a new span to track the chain execution and stores information about the - chain and its inputs. - - Args: - serialized: Serialized information about the chain - inputs: Dictionary of input values for the chain - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="chain", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - chain_name=serialized.get("name", "unknown"), - inputs=str(inputs or {}), - **kwargs, - ), - ) as span: - self._chain_spans[run_id] = span - - except Exception as e: - logger.error(f"Error in on_chain_start: {str(e)}") - - async def on_chain_end( - self, - outputs: Dict[str, Any], - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle chain end event asynchronously. - - This method is called when a Langchain chain operation completes. It - updates the span with the chain's outputs and marks it as successful. - - Args: - outputs: Dictionary of output values from the chain - run_id: Unique identifier for the chain run - **kwargs: Additional arguments - """ - if str(run_id) in self._chain_spans: - span = self._chain_spans[str(run_id)] - span.set_attribute(LangchainAttributes.CHAIN_OUTPUTS, str(outputs)) - span.end() - - async def on_chain_error( - self, - error: BaseException, - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle chain error event asynchronously. - - This method is called when an error occurs during a chain operation. - It updates the span with error information and marks it as failed. - - Args: - error: The exception that occurred - run_id: Unique identifier for the chain run - **kwargs: Additional error context - """ - if str(run_id) in self._chain_spans: - span = self._chain_spans[str(run_id)] - _handle_span_error(span, error, run_id=run_id, **kwargs) - - async def on_tool_start( - self, - serialized: Dict[str, Any], - input_str: str, - **kwargs: Any, - ) -> None: - """Handle tool start event asynchronously. - - This method is called when a tool operation begins. It creates a new span - to track the tool execution and stores information about the tool and its - inputs. - - Args: - serialized: Serialized information about the tool - input_str: String input for the tool - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="tool", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - tool_name=serialized.get("name", "unknown"), - tool_input=input_str, - inputs=str(kwargs.get("inputs", {})), - **kwargs, - ), - ) as span: - self._tool_spans[run_id] = span - - except Exception as e: - logger.error(f"Error in on_tool_start: {str(e)}") - - async def on_tool_end( - self, - output: str, - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle tool end event asynchronously. - - This method is called when a tool operation completes. It updates the span - with the tool's output and handles any errors that occurred during the - operation. - - Args: - output: String output from the tool - run_id: Unique identifier for the tool run - **kwargs: Additional arguments - """ - if str(run_id) in self._tool_spans: - span = self._tool_spans[str(run_id)] - span.set_attribute(LangchainAttributes.TOOL_OUTPUT, output) - - if kwargs.get("name") == "_Exception": - _handle_span_error(span, Exception(output), run_id=run_id, **kwargs) - else: - span.end() - - async def on_tool_error( - self, - error: BaseException, - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle tool error event asynchronously. - - This method is called when an error occurs during a tool operation. - It updates the span with error information and marks it as failed. - - Args: - error: The exception that occurred - run_id: Unique identifier for the tool run - **kwargs: Additional error context - """ - if str(run_id) in self._tool_spans: - span = self._tool_spans[str(run_id)] - _handle_span_error(span, error, run_id=run_id, **kwargs) - - async def on_retriever_start( - self, - serialized: Dict[str, Any], - query: str, - **kwargs: Any, - ) -> None: - """Handle retriever start event asynchronously. - - This method is called when a retriever operation begins. It creates a new - span to track the retrieval process and stores information about the - retriever and the query. - - Args: - serialized: Serialized information about the retriever - query: The search query string - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="retriever", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - retriever_name=serialized.get("name", "unknown"), - query=query, - **kwargs, - ), - ) as span: - self._retriever_spans[run_id] = span - - except Exception as e: - logger.error(f"Error in on_retriever_start: {str(e)}") - - async def on_retriever_end( - self, - documents: Sequence[Document], - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle retriever end event asynchronously. - - This method is called when a retriever operation completes. It updates - the span with the retrieved documents and marks it as successful. - - Args: - documents: Sequence of retrieved documents - run_id: Unique identifier for the retriever run - **kwargs: Additional arguments - """ - if str(run_id) in self._retriever_spans: - span = self._retriever_spans[str(run_id)] - span.set_attribute(LangchainAttributes.RETRIEVER_DOCUMENTS, str(documents)) - span.end() - - async def on_retriever_error( - self, - error: BaseException, - *, - run_id: UUID, - **kwargs: Any, - ) -> None: - """Handle retriever error event asynchronously. - - This method is called when an error occurs during a retriever operation. - It updates the span with error information and marks it as failed. - - Args: - error: The exception that occurred - run_id: Unique identifier for the retriever run - **kwargs: Additional error context - """ - if str(run_id) in self._retriever_spans: - span = self._retriever_spans[str(run_id)] - _handle_span_error(span, error, run_id=run_id, **kwargs) - - async def on_agent_action( - self, - action: AgentAction, - **kwargs: Any, - ) -> None: - """Handle agent action event asynchronously. - - This method is called when an agent performs an action. It creates a new - span to track the action and stores information about the tool being used - and its inputs. - - Args: - action: The agent action containing tool and input information - **kwargs: Additional arguments including run_id and metadata - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="agent_action", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - agent_name=action.tool, - tool_input=action.tool_input, - tool_log=action.log, - **kwargs, - ), - ) as span: - self._agent_actions[run_id].append(span) - - except Exception as e: - logger.error(f"Error in on_agent_action: {str(e)}") - - async def on_agent_finish( - self, - finish: AgentFinish, - **kwargs: Any, - ) -> None: - """Handle agent finish event asynchronously. - - This method is called when an agent completes its task. It updates all - relevant spans with final outputs and marks them as successful. - - Args: - finish: The AgentFinish event containing final outputs - **kwargs: Additional arguments including run_id - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - self._handle_agent_finish(run_id, finish) - except Exception as e: - logger.error(f"Error in on_agent_finish: {str(e)}") - - async def on_retry( - self, - retry_state: RetryCallState, - **kwargs: Any, - ) -> None: - """Handle retry event asynchronously. - - This method is called when an operation is being retried. It creates a - new span to track the retry attempt and stores information about the - error that caused the retry. - - Args: - retry_state: State information about the retry attempt - **kwargs: Additional arguments including run_id - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - try: - with _create_as_current_span( - name="retry", - kind=SpanKind.INTERNAL, - attributes=_create_span_attributes( - run_id=run_id, - retry_attempt=retry_state.attempt_number, - error_type=type(retry_state.outcome.exception()).__name__, - error_message=str(retry_state.outcome.exception()), - **kwargs, - ), - ) as span: - span.set_status(Status(StatusCode.ERROR)) - span.end() - - except Exception as e: - logger.error(f"Error in on_retry: {str(e)}") - - async def on_llm_new_token( - self, - token: str, - **kwargs: Any, - ) -> None: - """Handle new LLM token event asynchronously. - - This method is called when a new token is generated during streaming - LLM responses. It updates the span with the accumulated response text. - - Args: - token: The new token generated - **kwargs: Additional arguments including run_id - """ - run_id = kwargs.get("run_id") - if not run_id: - return - - span = self._llm_spans.get(run_id) - if not span: - return - - try: - current_completion = span.get_attribute(LangchainAttributes.RESPONSE) or "" - span.set_attribute(LangchainAttributes.RESPONSE, current_completion + token) - except Exception as e: - logger.error(f"Error in on_llm_new_token: {str(e)}") \ No newline at end of file diff --git a/agentops/sdk/callbacks/langchain/README.md b/agentops/sdk/callbacks/langchain/README.md new file mode 100644 index 000000000..971e2a9e0 --- /dev/null +++ b/agentops/sdk/callbacks/langchain/README.md @@ -0,0 +1,59 @@ +# AgentOps LangChain Callback Handler + +This callback handler enables seamless integration between LangChain and AgentOps for tracing and monitoring LLM applications. + +## Features + +- **Complete Coverage**: Supports all LangChain callback methods +- **Session Tracking**: Creates a session span that serves as the root for all operations +- **Proper Hierarchy**: Maintains parent-child relationships between operations +- **Complete Instrumentation**: Tracks LLMs, chains, tools, and agent actions +- **Error Tracking**: Records errors from LLMs, chains, and tools +- **Streaming Support**: Handles token streaming for real-time insights +- **Attribute Capture**: Records inputs, outputs, and metadata for all operations +- **Error Resilience**: Handles errors gracefully to ensure spans are always properly closed + +## Supported Callbacks + +The handler implements all LangChain callback methods: + +| Method | Description | Span Kind | Attributes | +|--------|-------------|-----------|------------| +| `on_llm_start` | Start of an LLM call | `llm` | Model, prompts, parameters | +| `on_llm_end` | End of an LLM call | `llm` | Completions, token usage | +| `on_llm_new_token` | Streaming token received | N/A | Token count, last token | +| `on_llm_error` | LLM call error | `llm` | Error details | +| `on_chat_model_start` | Start of a chat model call | `llm` | Model, messages, parameters | +| `on_chain_start` | Start of a chain | `task` | Chain type, inputs | +| `on_chain_end` | End of a chain | `task` | Outputs | +| `on_chain_error` | Chain execution error | `task` | Error details | +| `on_tool_start` | Start of a tool call | `tool` | Tool name, input | +| `on_tool_end` | End of a tool call | `tool` | Output | +| `on_tool_error` | Tool execution error | `tool` | Error details | +| `on_agent_action` | Agent taking an action | `agent` | Tool, input, log | +| `on_agent_finish` | Agent completing a task | `agent` | Output, log | +| `on_text` | Arbitrary text event | `text` | Text content | + +All spans have appropriate attributes such as: +- Model information for LLM spans +- Input/output for all operations +- Tool names and types +- Chain types and configurations +- Error details for failed operations + +## Troubleshooting + +If you're not seeing data in AgentOps: + +1. Check that your API key is correctly configured +2. Ensure you're passing the handler to all relevant components +3. Verify that all operations are properly ending/closing + +## How It Works + +The callback handler: +1. Creates a session span when initialized +2. Intercepts LangChain callbacks for various operations +3. Creates appropriate spans with meaningful attributes +4. Maintains proper parent-child relationships +5. Automatically cleans up and ends spans when operations complete \ No newline at end of file diff --git a/agentops/sdk/callbacks/langchain/__init__.py b/agentops/sdk/callbacks/langchain/__init__.py new file mode 100644 index 000000000..794594bfa --- /dev/null +++ b/agentops/sdk/callbacks/langchain/__init__.py @@ -0,0 +1,15 @@ +""" +LangChain integration for AgentOps. + +This module provides the AgentOps LangChain integration, including callbacks and utilities. +""" + +from agentops.sdk.callbacks.langchain.callback import ( + LangchainCallbackHandler, + AsyncLangchainCallbackHandler, +) + +__all__ = [ + "LangchainCallbackHandler", + "AsyncLangchainCallbackHandler", +] \ No newline at end of file diff --git a/agentops/sdk/callbacks/langchain/callback.py b/agentops/sdk/callbacks/langchain/callback.py new file mode 100644 index 000000000..8f1907426 --- /dev/null +++ b/agentops/sdk/callbacks/langchain/callback.py @@ -0,0 +1,896 @@ +""" +LangChain callback handler for AgentOps. + +This module provides the LangChain callback handler for AgentOps tracing and monitoring. +""" + +from typing import Any, Dict, List, Optional, Union + +from opentelemetry import trace +from opentelemetry.context import attach, detach, get_current +from opentelemetry.trace import SpanContext, set_span_in_context + +from agentops.helpers.serialization import safe_serialize +from agentops.logging import logger +from agentops.sdk.core import TracingCore +from agentops.semconv import SpanKind, SpanAttributes, LangChainAttributes, LangChainAttributeValues +from agentops.sdk.callbacks.langchain.utils import get_model_info + +from langchain_core.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler +from langchain_core.outputs import LLMResult +from langchain_core.agents import AgentAction, AgentFinish + +class LangchainCallbackHandler(BaseCallbackHandler): + """ + AgentOps sync callback handler for Langchain. + + This handler creates spans for LLM calls and other langchain operations, + maintaining proper parent-child relationships with session as root span. + + Args: + api_key (str, optional): AgentOps API key + tags (List[str], optional): Tags to add to the session + auto_session (bool, optional): Whether to automatically create a session span + """ + + def __init__( + self, + api_key: Optional[str] = None, + tags: Optional[List[str]] = None, + auto_session: bool = True, + ): + """Initialize the callback handler.""" + self.active_spans = {} + self.api_key = api_key + self.tags = tags or [] + self.session_span = None + self.session_token = None + self.context_tokens = {} # Store context tokens by run_id + self.token_counts = {} # Track token counts for streaming + + # Initialize AgentOps + if auto_session: + self._initialize_agentops() + + def _initialize_agentops(self): + """Initialize AgentOps""" + import agentops + + if not TracingCore.get_instance().initialized: + init_kwargs = { + "auto_start_session": False, + "instrument_llm_calls": True, + } + + if self.api_key: + init_kwargs["api_key"] = self.api_key + + agentops.init(**init_kwargs) + logger.debug("AgentOps initialized from LangChain callback handler") + + if not TracingCore.get_instance().initialized: + logger.warning("AgentOps not initialized, session span will not be created") + return + + tracer = TracingCore.get_instance().get_tracer() + + span_name = f"session.{SpanKind.SESSION}" + + attributes = { + SpanAttributes.AGENTOPS_SPAN_KIND: SpanKind.SESSION, + LangChainAttributes.SESSION_TAGS: self.tags, + "agentops.operation.name": "session", + "span.kind": SpanKind.SESSION, + } + + # Create a root session span + self.session_span = tracer.start_span(span_name, attributes=attributes) + + # Attach session span to the current context + self.session_token = attach(set_span_in_context(self.session_span)) + + logger.debug("Created session span as root span for LangChain") + + def _create_span( + self, + operation_name: str, + span_kind: str, + run_id: Any = None, + attributes: Optional[Dict[str, Any]] = None, + parent_run_id: Optional[Any] = None + ): + """ + Create a span for the operation. + + Args: + operation_name: Name of the operation + span_kind: Type of span + run_id: Unique identifier for the operation + attributes: Additional attributes for the span + parent_run_id: The run_id of the parent span if this is a child span + + Returns: + The created span + """ + if not TracingCore.get_instance().initialized: + logger.warning("AgentOps not initialized, spans will not be created") + return trace.NonRecordingSpan(SpanContext.INVALID) + + tracer = TracingCore.get_instance().get_tracer() + + span_name = f"{operation_name}.{span_kind}" + + if attributes is None: + attributes = {} + + attributes[SpanAttributes.AGENTOPS_SPAN_KIND] = span_kind + attributes["agentops.operation.name"] = operation_name + + if run_id is None: + run_id = id(attributes) + + # Get the current active context + current_context = get_current() + + parent_span = None + if parent_run_id is not None and parent_run_id in self.active_spans: + # Get parent span from active spans + parent_span = self.active_spans.get(parent_run_id) + # Create context with parent span + parent_ctx = set_span_in_context(parent_span) + # Start span with parent context + span = tracer.start_span(span_name, context=parent_ctx, attributes=attributes) + logger.debug(f"Started span: {span_name} with parent: {parent_run_id}") + else: + # If no parent_run_id or parent not found, use session as parent + parent_ctx = set_span_in_context(self.session_span) + # Start span with session as parent context + span = tracer.start_span(span_name, context=parent_ctx, attributes=attributes) + logger.debug(f"Started span: {span_name} with session as parent") + + # Store span in active_spans + self.active_spans[run_id] = span + + # Store token to detach later + token = attach(set_span_in_context(span)) + self.context_tokens[run_id] = token + + return span + + def _end_span(self, run_id: Any): + """ + End the span associated with the run_id. + + Args: + run_id: Unique identifier for the operation + """ + if run_id not in self.active_spans: + logger.warning(f"No span found for call {run_id}") + return + + span = self.active_spans.pop(run_id) + token = self.context_tokens.pop(run_id, None) + + if token is not None: + detach(token) + + try: + span.end() + logger.debug(f"Ended span: {span.name}") + except Exception as e: + logger.warning(f"Error ending span: {e}") + + # Clean up token counts if present + if run_id in self.token_counts: + del self.token_counts[run_id] + + def on_llm_start( + self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any + ) -> None: + """Run when LLM starts running.""" + try: + # Add null check for serialized + if serialized is None: + serialized = {} + + model_info = get_model_info(serialized) + # Ensure default values if model_info returns unknown + model_name = model_info.get("model_name", "unknown") + model_provider = model_info.get("provider", "unknown") + + attributes = { + SpanAttributes.LLM_REQUEST_MODEL: model_name, + SpanAttributes.LLM_PROMPTS: safe_serialize(prompts), + LangChainAttributes.LLM_MODEL: model_name, + LangChainAttributes.LLM_NAME: serialized.get("id", "unknown_llm"), + LangChainAttributes.LLM_PROVIDER: model_provider, + } + + if "kwargs" in serialized: + for key, value in serialized["kwargs"].items(): + if key == "temperature": + param_key = f"gen_ai.request.{key}" + attributes[param_key] = value + attributes[LangChainAttributes.LLM_TEMPERATURE] = value + elif key == "max_tokens": + param_key = f"gen_ai.request.{key}" + attributes[param_key] = value + attributes[LangChainAttributes.LLM_MAX_TOKENS] = value + elif key == "top_p": + param_key = f"gen_ai.request.{key}" + attributes[param_key] = value + attributes[LangChainAttributes.LLM_TOP_P] = value + + run_id = kwargs.get("run_id", id(serialized or {})) + parent_run_id = kwargs.get("parent_run_id", None) + + # Initialize token count for streaming if needed + self.token_counts[run_id] = 0 + + # Log parent relationship for debugging + if parent_run_id: + logger.debug(f"LLM span with run_id {run_id} has parent {parent_run_id}") + + self._create_span("llm", SpanKind.LLM, run_id, attributes, parent_run_id) + + logger.debug(f"Started LLM span for {model_name} ({model_provider})") + except Exception as e: + logger.warning(f"Error in on_llm_start: {e}") + + def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None: + """Run when LLM ends running.""" + try: + run_id = kwargs.get("run_id", id(response)) + + if run_id not in self.active_spans: + logger.warning(f"No span found for LLM call {run_id}") + return + + span = self.active_spans.get(run_id) + + if hasattr(response, "generations") and response.generations: + completions = [] + for gen_list in response.generations: + for gen in gen_list: + if hasattr(gen, "text"): + completions.append(gen.text) + + if completions: + try: + span.set_attribute( + SpanAttributes.LLM_COMPLETIONS, + safe_serialize(completions) + ) + except Exception as e: + logger.warning(f"Failed to set completions: {e}") + + if hasattr(response, "llm_output") and response.llm_output: + token_usage = response.llm_output.get("token_usage", {}) + + if "completion_tokens" in token_usage: + try: + span.set_attribute( + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, + token_usage["completion_tokens"] + ) + except Exception as e: + logger.warning(f"Failed to set completion tokens: {e}") + + if "prompt_tokens" in token_usage: + try: + span.set_attribute( + SpanAttributes.LLM_USAGE_PROMPT_TOKENS, + token_usage["prompt_tokens"] + ) + except Exception as e: + logger.warning(f"Failed to set prompt tokens: {e}") + + if "total_tokens" in token_usage: + try: + span.set_attribute( + SpanAttributes.LLM_USAGE_TOTAL_TOKENS, + token_usage["total_tokens"] + ) + except Exception as e: + logger.warning(f"Failed to set total tokens: {e}") + + # For streaming, record the total tokens streamed + if run_id in self.token_counts and self.token_counts[run_id] > 0: + try: + span.set_attribute( + "llm.usage.streaming_tokens", + self.token_counts[run_id] + ) + except Exception as e: + logger.warning(f"Failed to set streaming tokens: {e}") + + # End the span after setting all attributes + self._end_span(run_id) + + except Exception as e: + logger.warning(f"Error in on_llm_end: {e}") + + def on_chain_start( + self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any + ) -> None: + """Run when chain starts running.""" + try: + # Add null check for serialized + if serialized is None: + serialized = {} + + chain_type = serialized.get("name", "unknown_chain") + + attributes = { + LangChainAttributes.CHAIN_TYPE: chain_type, + LangChainAttributes.CHAIN_NAME: serialized.get("id", "unknown_chain"), + LangChainAttributes.CHAIN_VERBOSE: serialized.get("verbose", False), + "chain.inputs": safe_serialize(inputs), + } + + # Add specific chain types + if "sequential" in chain_type.lower(): + attributes[LangChainAttributes.CHAIN_KIND] = LangChainAttributeValues.CHAIN_KIND_SEQUENTIAL + elif "llm" in chain_type.lower(): + attributes[LangChainAttributes.CHAIN_KIND] = LangChainAttributeValues.CHAIN_KIND_LLM + elif "router" in chain_type.lower(): + attributes[LangChainAttributes.CHAIN_KIND] = LangChainAttributeValues.CHAIN_KIND_ROUTER + + run_id = kwargs.get("run_id", id(serialized or {})) + parent_run_id = kwargs.get("parent_run_id", None) + + # Log parent relationship for debugging + if parent_run_id: + logger.debug(f"Chain span with run_id {run_id} has parent {parent_run_id}") + + self._create_span("chain", SpanKind.CHAIN, run_id, attributes, parent_run_id) + + logger.debug(f"Started Chain span for {chain_type}") + except Exception as e: + logger.warning(f"Error in on_chain_start: {e}") + + def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None: + """Run when chain ends running.""" + try: + run_id = kwargs.get("run_id", id(outputs)) + + if run_id not in self.active_spans: + logger.warning(f"No span found for chain call {run_id}") + return + + span = self.active_spans.get(run_id) + + try: + span.set_attribute( + "chain.outputs", + safe_serialize(outputs) + ) + except Exception as e: + logger.warning(f"Failed to set chain outputs: {e}") + + # End the span after setting all attributes + self._end_span(run_id) + + except Exception as e: + logger.warning(f"Error in on_chain_end: {e}") + + def on_tool_start( + self, serialized: Dict[str, Any], input_str: str, **kwargs: Any + ) -> None: + """Run when tool starts running.""" + try: + # Add null check for serialized + if serialized is None: + serialized = {} + + tool_name = serialized.get("name", "unknown_tool") + + attributes = { + LangChainAttributes.TOOL_NAME: tool_name, + LangChainAttributes.TOOL_DESCRIPTION: serialized.get("description", ""), + LangChainAttributes.TOOL_INPUT: input_str, + } + + # Add more tool-specific attributes + if "return_direct" in serialized: + attributes[LangChainAttributes.TOOL_RETURN_DIRECT] = serialized["return_direct"] + + if "args_schema" in serialized: + schema = serialized.get("args_schema") + if schema: + schema_str = str(schema) + if len(schema_str) < 1000: # Avoid extremely large attributes + attributes[LangChainAttributes.TOOL_ARGS_SCHEMA] = schema_str + + run_id = kwargs.get("run_id", id(serialized or {})) + parent_run_id = kwargs.get("parent_run_id", None) + + self._create_span("tool", SpanKind.TOOL, run_id, attributes, parent_run_id) + + logger.debug(f"Started Tool span for {tool_name}") + except Exception as e: + logger.warning(f"Error in on_tool_start: {e}") + + def on_tool_end(self, output: str, **kwargs: Any) -> None: + """Run when tool ends running.""" + try: + run_id = kwargs.get("run_id", id(output)) + + if run_id not in self.active_spans: + logger.warning(f"No span found for tool call {run_id}") + return + + span = self.active_spans.get(run_id) + + try: + span.set_attribute( + LangChainAttributes.TOOL_OUTPUT, + output if isinstance(output, str) else safe_serialize(output) + ) + except Exception as e: + logger.warning(f"Failed to set tool output: {e}") + + # End the span after setting all attributes + self._end_span(run_id) + + except Exception as e: + logger.warning(f"Error in on_tool_end: {e}") + + def on_agent_action(self, action: AgentAction, **kwargs: Any) -> None: + """Run on agent action.""" + try: + tool = action.tool + tool_input = action.tool_input + log = action.log + + attributes = { + LangChainAttributes.AGENT_ACTION_TOOL: tool, + LangChainAttributes.AGENT_ACTION_INPUT: safe_serialize(tool_input), + LangChainAttributes.AGENT_ACTION_LOG: log, + } + + run_id = kwargs.get("run_id", id(action)) + parent_run_id = kwargs.get("parent_run_id", None) + + self._create_span("agent_action", SpanKind.AGENT_ACTION, run_id, attributes, parent_run_id) + + logger.debug(f"Started Agent Action span for {tool}") + except Exception as e: + logger.warning(f"Error in on_agent_action: {e}") + + def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> None: + """Run on agent end.""" + try: + run_id = kwargs.get("run_id", id(finish)) + + if run_id not in self.active_spans: + logger.warning(f"No span found for agent finish {run_id}") + return + + span = self.active_spans.get(run_id) + + try: + span.set_attribute( + LangChainAttributes.AGENT_FINISH_RETURN_VALUES, + safe_serialize(finish.return_values) + ) + except Exception as e: + logger.warning(f"Failed to set agent return values: {e}") + + try: + span.set_attribute( + LangChainAttributes.AGENT_FINISH_LOG, + finish.log + ) + except Exception as e: + logger.warning(f"Failed to set agent log: {e}") + + # End the span after setting all attributes + self._end_span(run_id) + + except Exception as e: + logger.warning(f"Error in on_agent_finish: {e}") + + def __del__(self): + """Clean up resources when the handler is deleted.""" + try: + # End any remaining spans + for run_id in list(self.active_spans.keys()): + try: + self._end_span(run_id) + except Exception as e: + logger.warning(f"Error ending span during cleanup: {e}") + + # End session span and detach session token + if self.session_span: + try: + # Detach session token if exists + if hasattr(self, 'session_token') and self.session_token: + detach(self.session_token) + + self.session_span.end() + logger.debug("Ended session span") + except Exception as e: + logger.warning(f"Error ending session span: {e}") + + except Exception as e: + logger.warning(f"Error in __del__: {e}") + + def on_llm_new_token(self, token: str, **kwargs: Any) -> None: + """Run on new token from LLM.""" + try: + run_id = kwargs.get("run_id") + + if not run_id: + logger.warning("No run_id provided for on_llm_new_token") + return + + if run_id not in self.active_spans: + logger.warning(f"No span found for token in run {run_id}") + return + + # Count tokens for later attribution + if run_id in self.token_counts: + self.token_counts[run_id] += 1 + else: + self.token_counts[run_id] = 1 + + # We don't set attributes on each token because it's inefficient + # and can lead to "setting attribute on ended span" errors + # Instead, we count tokens and set the total at the end + + except Exception as e: + logger.warning(f"Error in on_llm_new_token: {e}") + + def on_chat_model_start( + self, serialized: Dict[str, Any], messages: List[Any], **kwargs: Any + ) -> None: + """Run when a chat model starts generating.""" + try: + # Add null check for serialized + if serialized is None: + serialized = {} + + model_info = get_model_info(serialized) + # Ensure default values if model_info returns unknown + model_name = model_info.get("model_name", "unknown") + model_provider = model_info.get("provider", "unknown") + + # Extract message contents and roles + formatted_messages = [] + roles = [] + + for message in messages: + if hasattr(message, "content") and hasattr(message, "type"): + formatted_messages.append({ + "content": message.content, + "role": message.type + }) + roles.append(message.type) + + attributes = { + SpanAttributes.LLM_REQUEST_MODEL: model_name, + SpanAttributes.LLM_PROMPTS: safe_serialize(formatted_messages), + LangChainAttributes.LLM_MODEL: model_name, + LangChainAttributes.LLM_NAME: serialized.get("id", "unknown_chat_model"), + LangChainAttributes.LLM_PROVIDER: model_provider, + LangChainAttributes.CHAT_MESSAGE_ROLES: safe_serialize(roles), + LangChainAttributes.CHAT_MODEL_TYPE: "chat", + } + + # Add generation parameters + if "kwargs" in serialized: + for key, value in serialized["kwargs"].items(): + if key == "temperature": + param_key = f"gen_ai.request.{key}" + attributes[param_key] = value + attributes[LangChainAttributes.LLM_TEMPERATURE] = value + elif key == "max_tokens": + param_key = f"gen_ai.request.{key}" + attributes[param_key] = value + attributes[LangChainAttributes.LLM_MAX_TOKENS] = value + elif key == "top_p": + param_key = f"gen_ai.request.{key}" + attributes[param_key] = value + attributes[LangChainAttributes.LLM_TOP_P] = value + + run_id = kwargs.get("run_id", id(serialized or {})) + parent_run_id = kwargs.get("parent_run_id", None) + + # Initialize token count for streaming if needed + self.token_counts[run_id] = 0 + + self._create_span("chat_model", SpanKind.LLM, run_id, attributes, parent_run_id) + + logger.debug(f"Started Chat Model span for {model_name} ({model_provider})") + except Exception as e: + logger.warning(f"Error in on_chat_model_start: {e}") + + def on_llm_error( + self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any + ) -> None: + """Run when LLM errors.""" + try: + run_id = kwargs.get("run_id") + + if not run_id or run_id not in self.active_spans: + logger.warning(f"No span found for LLM error {run_id}") + return + + span = self.active_spans.get(run_id) + + # Record error attributes + try: + span.set_attribute( + "error", True + ) + span.set_attribute( + "error.type", error.__class__.__name__ + ) + span.set_attribute( + "error.message", str(error) + ) + span.set_attribute( + LangChainAttributes.LLM_ERROR, str(error) + ) + except Exception as e: + logger.warning(f"Failed to set error attributes: {e}") + + # End span with error + self._end_span(run_id) + + except Exception as e: + logger.warning(f"Error in on_llm_error: {e}") + + def on_chain_error( + self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any + ) -> None: + """Run when chain errors.""" + try: + run_id = kwargs.get("run_id") + + if not run_id or run_id not in self.active_spans: + logger.warning(f"No span found for chain error {run_id}") + return + + span = self.active_spans.get(run_id) + + # Record error attributes + try: + span.set_attribute( + "error", True + ) + span.set_attribute( + "error.type", error.__class__.__name__ + ) + span.set_attribute( + "error.message", str(error) + ) + span.set_attribute( + LangChainAttributes.CHAIN_ERROR, str(error) + ) + except Exception as e: + logger.warning(f"Failed to set error attributes: {e}") + + # End span with error + self._end_span(run_id) + + except Exception as e: + logger.warning(f"Error in on_chain_error: {e}") + + def on_tool_error( + self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any + ) -> None: + """Run when tool errors.""" + try: + run_id = kwargs.get("run_id") + + if not run_id or run_id not in self.active_spans: + logger.warning(f"No span found for tool error {run_id}") + return + + span = self.active_spans.get(run_id) + + # Record error attributes + try: + span.set_attribute( + "error", True + ) + span.set_attribute( + "error.type", error.__class__.__name__ + ) + span.set_attribute( + "error.message", str(error) + ) + span.set_attribute( + LangChainAttributes.TOOL_ERROR, str(error) + ) + except Exception as e: + logger.warning(f"Failed to set error attributes: {e}") + + # End span with error + self._end_span(run_id) + + except Exception as e: + logger.warning(f"Error in on_tool_error: {e}") + + def on_text(self, text: str, **kwargs: Any) -> None: + """ + Run on arbitrary text. + + This can be used for logging or recording intermediate steps. + """ + try: + run_id = kwargs.get("run_id") + + if run_id is None: + # Create a new span for this text + run_id = id(text) + parent_run_id = kwargs.get("parent_run_id") + + attributes = { + LangChainAttributes.TEXT_CONTENT: text, + } + + self._create_span("text", SpanKind.TEXT, run_id, attributes, parent_run_id) + + # Immediately end the span as text events are one-off + self._end_span(run_id) + else: + # Try to find a parent span to add the text to + parent_run_id = kwargs.get("parent_run_id") + + if parent_run_id and parent_run_id in self.active_spans: + # Add text to parent span + try: + parent_span = self.active_spans[parent_run_id] + # Use get_attribute to check if text already exists + existing_text = "" + try: + existing_text = parent_span.get_attribute(LangChainAttributes.TEXT_CONTENT) or "" + except Exception: + # If get_attribute isn't available or fails, just set the text + pass + + if existing_text: + parent_span.set_attribute( + LangChainAttributes.TEXT_CONTENT, + f"{existing_text}\n{text}" + ) + else: + parent_span.set_attribute( + LangChainAttributes.TEXT_CONTENT, + text + ) + except Exception as e: + logger.warning(f"Failed to update parent span with text: {e}") + except Exception as e: + logger.warning(f"Error in on_text: {e}") + +class AsyncLangchainCallbackHandler(AsyncCallbackHandler): + """ + AgentOps async callback handler for Langchain. + + This handler creates spans for LLM calls and other langchain operations, + maintaining proper parent-child relationships with session as root span. + This is the async version of the handler. + + Args: + api_key (str, optional): AgentOps API key + tags (List[str], optional): Tags to add to the session + auto_session (bool, optional): Whether to automatically create a session span + """ + + def __init__( + self, + api_key: Optional[str] = None, + tags: Optional[List[str]] = None, + auto_session: bool = True, + ): + """Initialize the callback handler.""" + # Create an internal sync handler to delegate to + self._sync_handler = LangchainCallbackHandler( + api_key=api_key, + tags=tags, + auto_session=auto_session + ) + + @property + def active_spans(self): + """Access to the active spans dictionary from sync handler.""" + return self._sync_handler.active_spans + + @property + def session_span(self): + """Access to the session span from sync handler.""" + return self._sync_handler.session_span + + async def on_llm_start( + self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any + ) -> None: + """Run when LLM starts running.""" + # Delegate to sync handler + self._sync_handler.on_llm_start(serialized, prompts, **kwargs) + + async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None: + """Run when LLM ends running.""" + # Delegate to sync handler + self._sync_handler.on_llm_end(response, **kwargs) + + async def on_chain_start( + self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any + ) -> None: + """Run when chain starts running.""" + # Delegate to sync handler + self._sync_handler.on_chain_start(serialized, inputs, **kwargs) + + async def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None: + """Run when chain ends running.""" + # Delegate to sync handler + self._sync_handler.on_chain_end(outputs, **kwargs) + + async def on_tool_start( + self, serialized: Dict[str, Any], input_str: str, **kwargs: Any + ) -> None: + """Run when tool starts running.""" + # Delegate to sync handler + self._sync_handler.on_tool_start(serialized, input_str, **kwargs) + + async def on_tool_end(self, output: str, **kwargs: Any) -> None: + """Run when tool ends running.""" + # Delegate to sync handler + self._sync_handler.on_tool_end(output, **kwargs) + + async def on_agent_action(self, action: AgentAction, **kwargs: Any) -> None: + """Run on agent action.""" + # Delegate to sync handler + self._sync_handler.on_agent_action(action, **kwargs) + + async def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> None: + """Run on agent end.""" + # Delegate to sync handler + self._sync_handler.on_agent_finish(finish, **kwargs) + + async def on_llm_new_token(self, token: str, **kwargs: Any) -> None: + """Run on new token from LLM.""" + # Delegate to sync handler + self._sync_handler.on_llm_new_token(token, **kwargs) + + async def on_chat_model_start( + self, serialized: Dict[str, Any], messages: List[Any], **kwargs: Any + ) -> None: + """Run when a chat model starts generating.""" + # Delegate to sync handler + self._sync_handler.on_chat_model_start(serialized, messages, **kwargs) + + async def on_llm_error( + self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any + ) -> None: + """Run when LLM errors.""" + # Delegate to sync handler + self._sync_handler.on_llm_error(error, **kwargs) + + async def on_chain_error( + self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any + ) -> None: + """Run when chain errors.""" + # Delegate to sync handler + self._sync_handler.on_chain_error(error, **kwargs) + + async def on_tool_error( + self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any + ) -> None: + """Run when tool errors.""" + # Delegate to sync handler + self._sync_handler.on_tool_error(error, **kwargs) + + async def on_text(self, text: str, **kwargs: Any) -> None: + """Run on arbitrary text.""" + # Delegate to sync handler + self._sync_handler.on_text(text, **kwargs) + + def __del__(self): + """Clean up resources when the handler is deleted.""" + # The sync handler's __del__ will handle cleanup + if hasattr(self, '_sync_handler'): + del self._sync_handler \ No newline at end of file diff --git a/agentops/sdk/callbacks/langchain/utils.py b/agentops/sdk/callbacks/langchain/utils.py new file mode 100644 index 000000000..bc103baf7 --- /dev/null +++ b/agentops/sdk/callbacks/langchain/utils.py @@ -0,0 +1,84 @@ +""" +Utility functions for LangChain integration. +""" + +from typing import Any, Dict, Optional + +from agentops.helpers.serialization import safe_serialize +from agentops.logging import logger + + +def get_model_info(serialized: Optional[Dict[str, Any]]) -> Dict[str, str]: + """ + Extract model information from serialized LangChain data. + + This function attempts to extract model name and provider information + from the serialized data of a LangChain model. + + Args: + serialized: Serialized data from LangChain + + Returns: + Dictionary with provider and model_name keys + """ + if serialized is None: + return {"provider": "unknown", "model_name": "unknown"} + + model_info = {"provider": "unknown", "model_name": "unknown"} + + try: + if isinstance(serialized.get("id"), list) and len(serialized["id"]) > 0: + id_list = serialized["id"] + + for item in id_list: + if isinstance(item, str): + item_lower = item.lower() + if any(provider in item_lower for provider in ["openai", "anthropic", "google", "azure", "huggingface", "replicate", "cohere", "llama"]): + model_info["provider"] = item + break + + if model_info["model_name"] == "unknown" and len(id_list) > 0: + model_info["model_name"] = id_list[-1] + + if isinstance(serialized.get("model_name"), str): + model_info["model_name"] = serialized["model_name"] + + elif serialized.get("id") and isinstance(serialized.get("id"), str): + model_id = serialized.get("id", "") + if "/" in model_id: + provider, model_name = model_id.split("/", 1) + model_info["provider"] = provider + model_info["model_name"] = model_name + else: + model_info["model_name"] = model_id + + if serialized.get("kwargs") and isinstance(serialized["kwargs"], dict): + if serialized["kwargs"].get("model_name"): + model_info["model_name"] = serialized["kwargs"]["model_name"] + elif serialized["kwargs"].get("model"): + model_info["model_name"] = serialized["kwargs"]["model"] + + if serialized.get("_type") and model_info["provider"] == "unknown": + model_info["provider"] = str(serialized["_type"]) + + if model_info["provider"] == "unknown" and model_info["model_name"] != "unknown": + model_name_lower = model_info["model_name"].lower() + if "gpt" in model_name_lower: + model_info["provider"] = "openai" + elif "claude" in model_name_lower: + model_info["provider"] = "anthropic" + elif "palm" in model_name_lower or "gemini" in model_name_lower: + model_info["provider"] = "google" + elif "llama" in model_name_lower: + model_info["provider"] = "meta" + + if serialized.get("name") and model_info["provider"] == "unknown": + name_lower = str(serialized["name"]).lower() + if "openai" in name_lower: + model_info["provider"] = "openai" + elif "anthropic" in name_lower: + model_info["provider"] = "anthropic" + except Exception as e: + logger.warning(f"Error extracting model info: {e}") + + return model_info \ No newline at end of file diff --git a/agentops/semconv/__init__.py b/agentops/semconv/__init__.py index ea26eed4b..ec06895f2 100644 --- a/agentops/semconv/__init__.py +++ b/agentops/semconv/__init__.py @@ -12,6 +12,7 @@ from .meters import Meters from .span_kinds import AgentOpsSpanKindValues from .resource import ResourceAttributes +from .langchain import LangChainAttributes, LangChainAttributeValues SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY = "suppress_language_model_instrumentation" __all__ = [ @@ -26,5 +27,8 @@ "LLMRequestTypeValues", "SpanAttributes", "Meters", - "AgentOpsSpanKindValuesResourceAttributes", + "AgentOpsSpanKindValues", + "ResourceAttributes", + "LangChainAttributes", + "LangChainAttributeValues", ] diff --git a/agentops/semconv/langchain.py b/agentops/semconv/langchain.py new file mode 100644 index 000000000..677b19e14 --- /dev/null +++ b/agentops/semconv/langchain.py @@ -0,0 +1,54 @@ +"""Semantic conventions for LangChain instrumentation.""" + +from enum import Enum + + +class LangChainAttributeValues: + """Standard values for LangChain attributes.""" + + CHAIN_KIND_SEQUENTIAL = "sequential" + CHAIN_KIND_LLM = "llm" + CHAIN_KIND_ROUTER = "router" + + +class LangChainAttributes: + """Attributes for LangChain instrumentation.""" + + SESSION_TAGS = "langchain.session.tags" + + CHAIN_NAME = "langchain.chain.name" + CHAIN_TYPE = "langchain.chain.type" + CHAIN_ERROR = "langchain.chain.error" + CHAIN_KIND = "langchain.chain.kind" + CHAIN_VERBOSE = "langchain.chain.verbose" + + LLM_NAME = "langchain.llm.name" + LLM_MODEL = "langchain.llm.model" + LLM_PROVIDER = "langchain.llm.provider" + LLM_TEMPERATURE = "langchain.llm.temperature" + LLM_MAX_TOKENS = "langchain.llm.max_tokens" + LLM_TOP_P = "langchain.llm.top_p" + LLM_ERROR = "langchain.llm.error" + + AGENT_ACTION_LOG = "langchain.agent.action_log" + AGENT_FINISH_RETURN_VALUES = "langchain.agent.finish.return_values" + AGENT_FINISH_LOG = "langchain.agent.finish.log" + AGENT_ACTION_LOG = "langchain.agent.action.log" + AGENT_ACTION_INPUT = "langchain.agent.action.input" + AGENT_FINISH_RETURN_VALUES = "langchain.agent.finish.return_values" + AGENT_ACTION_TOOL = "langchain.agent.action.tool" + + TOOL_NAME = "langchain.tool.name" + TOOL_INPUT = "langchain.tool.input" + TOOL_OUTPUT = "langchain.tool.output" + TOOL_DESCRIPTION = "langchain.tool.description" + TOOL_ERROR = "langchain.tool.error" + TOOL_ARGS_SCHEMA = "langchain.tool.args_schema" + TOOL_RETURN_DIRECT = "langchain.tool.return_direct" + + MESSAGE_ROLE = "langchain.message.role" + + CHAT_MESSAGE_ROLES = "langchain.chat_message.roles" + CHAT_MODEL_TYPE = "langchain.chat_model.type" + + TEXT_CONTENT = "langchain.text.content" \ No newline at end of file diff --git a/agentops/semconv/langchain_attributes.py b/agentops/semconv/langchain_attributes.py deleted file mode 100644 index 4682218e8..000000000 --- a/agentops/semconv/langchain_attributes.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Semantic conventions for Langchain integration. - -This module defines the semantic conventions used for Langchain spans and attributes -in OpenTelemetry traces. These conventions ensure consistent attribute naming -across the Langchain integration. -""" - -class LangchainAttributes: - """Semantic conventions for Langchain spans and attributes.""" - - # Run identifiers - RUN_ID = "langchain.run.id" - PARENT_RUN_ID = "langchain.parent_run.id" - TAGS = "langchain.tags" - METADATA = "langchain.metadata" - - # Response attributes - RESPONSE = "langchain.response" - PROMPT_TOKENS = "langchain.prompt_tokens" - COMPLETION_TOKENS = "langchain.completion_tokens" - TOTAL_TOKENS = "langchain.total_tokens" - - # Chain attributes - CHAIN_OUTPUTS = "langchain.chain.outputs" - - # Tool attributes - TOOL_OUTPUT = "langchain.tool.output" - TOOL_LOG = "langchain.tool.log" - OUTPUTS = "langchain.outputs" - - # Retriever attributes - RETRIEVER_DOCUMENTS = "langchain.retriever.documents" - - # Error attributes - ERROR_TYPE = "langchain.error.type" - ERROR_MESSAGE = "langchain.error.message" - ERROR_DETAILS = "langchain.error.details" \ No newline at end of file diff --git a/agentops/semconv/span_kinds.py b/agentops/semconv/span_kinds.py index 190afacbe..71c3fff79 100644 --- a/agentops/semconv/span_kinds.py +++ b/agentops/semconv/span_kinds.py @@ -25,7 +25,8 @@ class SpanKind: LLM = "llm" TEAM = "team" UNKNOWN = "unknown" - + CHAIN = "chain" + TEXT = "text" class AgentOpsSpanKindValues(Enum): WORKFLOW = "workflow" diff --git a/examples/openai_examples/openai_assistants_example.ipynb b/examples/openai_examples/openai_assistants_example.ipynb index f1c2595d3..5fbde966b 100644 --- a/examples/openai_examples/openai_assistants_example.ipynb +++ b/examples/openai_examples/openai_assistants_example.ipynb @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 62, "metadata": {}, "outputs": [], "source": [ @@ -173,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 63, "metadata": {}, "outputs": [], "source": [ @@ -189,11 +189,24 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agentops.init(api_key=AGENTOPS_API_KEY, default_tags=[\"openai\", \"beta-assistants\"])\n", + "execution_count": 64, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(DEBUG) 🖇 AgentOps: Instrumentors have already been populated.\n", + "(DEBUG) 🖇 AgentOps: [DEBUG] BEFORE _make_span session.session - Current context: {'span_id': '79636eb80922dac8', 'trace_id': '15191e1a9ca916a408f7537d83c210c0', 'name': 'session.session', 'is_recording': }\n", + "(DEBUG) 🖇 AgentOps: Started span: session.session (kind: session)\n", + "🖇 AgentOps: \u001b[92m\u001b[34mSession started: https://app.agentops.ai/drilldown?session_id=15191e1a-9ca9-16a4-08f7-537d83c210c0\u001b[0m\u001b[0m\n", + "(DEBUG) 🖇 AgentOps: [DEBUG] CREATED _make_span session.session - span_id: 4c582481db9c04be, parent: 79636eb80922dac8\n", + "🖇 AgentOps: \u001b[34m\u001b[34mSession Replay: https://app.agentops.ai/drilldown?session_id=15191e1a-9ca9-16a4-08f7-537d83c210c0\u001b[0m\u001b[0m\n" + ] + } + ], + "source": [ + "agentops.init(api_key=AGENTOPS_API_KEY, default_tags=[\"openai\", \"beta-assistants\"],auto_start_session=True)\n", "client = OpenAI(api_key=OPENAI_API_KEY)" ] }, @@ -206,9 +219,32 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 65, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", + " 'created_at': 1742586617,\n", + " 'description': None,\n", + " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", + " 'metadata': {},\n", + " 'model': 'gpt-4o-mini',\n", + " 'name': 'Math Tutor',\n", + " 'object': 'assistant',\n", + " 'tools': [],\n", + " 'response_format': 'auto',\n", + " 'temperature': 1.0,\n", + " 'tool_resources': {'code_interpreter': None, 'file_search': None},\n", + " 'top_p': 1.0,\n", + " 'reasoning_effort': None}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "assistant = client.beta.assistants.create(\n", " name=\"Math Tutor\",\n", @@ -248,9 +284,23 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp',\n", + " 'created_at': 1742586618,\n", + " 'metadata': {},\n", + " 'object': 'thread',\n", + " 'tool_resources': {'code_interpreter': None, 'file_search': None}}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "thread = client.beta.threads.create()\n", "show_json(thread)" @@ -265,9 +315,34 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'msg_H3Hcs3msbCoNxPUsb9xX8WnP',\n", + " 'assistant_id': None,\n", + " 'attachments': [],\n", + " 'completed_at': None,\n", + " 'content': [{'text': {'annotations': [],\n", + " 'value': 'I need to solve the equation `3x + 11 = 14`. Can you help me?'},\n", + " 'type': 'text'}],\n", + " 'created_at': 1742586618,\n", + " 'incomplete_at': None,\n", + " 'incomplete_details': None,\n", + " 'metadata': {},\n", + " 'object': 'thread.message',\n", + " 'role': 'user',\n", + " 'run_id': None,\n", + " 'status': None,\n", + " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp'}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "message = client.beta.threads.messages.create(\n", " thread_id=thread.id,\n", @@ -303,9 +378,47 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'run_4iI0RxJyKGWYflcgcwQEqTbX',\n", + " 'assistant_id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", + " 'cancelled_at': None,\n", + " 'completed_at': None,\n", + " 'created_at': 1742586619,\n", + " 'expires_at': 1742587219,\n", + " 'failed_at': None,\n", + " 'incomplete_details': None,\n", + " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", + " 'last_error': None,\n", + " 'max_completion_tokens': None,\n", + " 'max_prompt_tokens': None,\n", + " 'metadata': {},\n", + " 'model': 'gpt-4o-mini',\n", + " 'object': 'thread.run',\n", + " 'parallel_tool_calls': True,\n", + " 'required_action': None,\n", + " 'response_format': 'auto',\n", + " 'started_at': None,\n", + " 'status': 'queued',\n", + " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp',\n", + " 'tool_choice': 'auto',\n", + " 'tools': [],\n", + " 'truncation_strategy': {'type': 'auto', 'last_messages': None},\n", + " 'usage': None,\n", + " 'temperature': 1.0,\n", + " 'top_p': 1.0,\n", + " 'tool_resources': {},\n", + " 'reasoning_effort': None}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "run = client.beta.threads.runs.create(\n", " thread_id=thread.id,\n", @@ -325,7 +438,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -344,9 +457,51 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'run_4iI0RxJyKGWYflcgcwQEqTbX',\n", + " 'assistant_id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", + " 'cancelled_at': None,\n", + " 'completed_at': 1742586621,\n", + " 'created_at': 1742586619,\n", + " 'expires_at': None,\n", + " 'failed_at': None,\n", + " 'incomplete_details': None,\n", + " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", + " 'last_error': None,\n", + " 'max_completion_tokens': None,\n", + " 'max_prompt_tokens': None,\n", + " 'metadata': {},\n", + " 'model': 'gpt-4o-mini',\n", + " 'object': 'thread.run',\n", + " 'parallel_tool_calls': True,\n", + " 'required_action': None,\n", + " 'response_format': 'auto',\n", + " 'started_at': 1742586620,\n", + " 'status': 'completed',\n", + " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp',\n", + " 'tool_choice': 'auto',\n", + " 'tools': [],\n", + " 'truncation_strategy': {'type': 'auto', 'last_messages': None},\n", + " 'usage': {'completion_tokens': 36,\n", + " 'prompt_tokens': 66,\n", + " 'total_tokens': 102,\n", + " 'prompt_token_details': {'cached_tokens': 0},\n", + " 'completion_tokens_details': {'reasoning_tokens': 0}},\n", + " 'temperature': 1.0,\n", + " 'top_p': 1.0,\n", + " 'tool_resources': {},\n", + " 'reasoning_effort': None}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "run = wait_on_run(run, thread)\n", "show_json(run)" @@ -368,9 +523,63 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 71, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" + ] + }, + { + "data": { + "text/plain": [ + "{'data': [{'id': 'msg_WAMEjtq4CGdR28jNx8wa8Vhy',\n", + " 'assistant_id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", + " 'attachments': [],\n", + " 'completed_at': None,\n", + " 'content': [{'text': {'annotations': [],\n", + " 'value': 'Sure! Subtract 11 from both sides to get \\\\( 3x = 3 \\\\), then divide by 3 to find \\\\( x = 1 \\\\).'},\n", + " 'type': 'text'}],\n", + " 'created_at': 1742586620,\n", + " 'incomplete_at': None,\n", + " 'incomplete_details': None,\n", + " 'metadata': {},\n", + " 'object': 'thread.message',\n", + " 'role': 'assistant',\n", + " 'run_id': 'run_4iI0RxJyKGWYflcgcwQEqTbX',\n", + " 'status': None,\n", + " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp'},\n", + " {'id': 'msg_H3Hcs3msbCoNxPUsb9xX8WnP',\n", + " 'assistant_id': None,\n", + " 'attachments': [],\n", + " 'completed_at': None,\n", + " 'content': [{'text': {'annotations': [],\n", + " 'value': 'I need to solve the equation `3x + 11 = 14`. Can you help me?'},\n", + " 'type': 'text'}],\n", + " 'created_at': 1742586618,\n", + " 'incomplete_at': None,\n", + " 'incomplete_details': None,\n", + " 'metadata': {},\n", + " 'object': 'thread.message',\n", + " 'role': 'user',\n", + " 'run_id': None,\n", + " 'status': None,\n", + " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp'}],\n", + " 'has_more': False,\n", + " 'object': 'list',\n", + " 'first_id': 'msg_WAMEjtq4CGdR28jNx8wa8Vhy',\n", + " 'last_id': 'msg_H3Hcs3msbCoNxPUsb9xX8WnP'}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "messages = client.beta.threads.messages.list(thread_id=thread.id)\n", "show_json(messages)" @@ -392,9 +601,47 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 72, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" + ] + }, + { + "data": { + "text/plain": [ + "{'data': [{'id': 'msg_qS8ER2CuEovcezcm4ksQ4vOS',\n", + " 'assistant_id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", + " 'attachments': [],\n", + " 'completed_at': None,\n", + " 'content': [{'text': {'annotations': [],\n", + " 'value': 'Of course! To solve the equation \\\\( 3x + 11 = 14 \\\\), follow these steps:\\n\\n1. **Isolate the term with \\\\( x \\\\)**: Subtract 11 from both sides to eliminate 11 on the left. This gives you:\\n \\\\[\\n 3x + 11 - 11 = 14 - 11\\n \\\\]\\n Simplifying this, we get:\\n \\\\[\\n 3x = 3\\n \\\\]\\n\\n2. **Solve for \\\\( x \\\\)**: Now, divide both sides by 3 to isolate \\\\( x \\\\):\\n \\\\[\\n \\\\frac{3x}{3} = \\\\frac{3}{3}\\n \\\\]\\n This simplifies to:\\n \\\\[\\n x = 1\\n \\\\]\\n\\nSo, the solution is \\\\( x = 1 \\\\).'},\n", + " 'type': 'text'}],\n", + " 'created_at': 1742586625,\n", + " 'incomplete_at': None,\n", + " 'incomplete_details': None,\n", + " 'metadata': {},\n", + " 'object': 'thread.message',\n", + " 'role': 'assistant',\n", + " 'run_id': 'run_5hso6XQCmns3Q5dSdcSM9QDS',\n", + " 'status': None,\n", + " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp'}],\n", + " 'has_more': False,\n", + " 'object': 'list',\n", + " 'first_id': 'msg_qS8ER2CuEovcezcm4ksQ4vOS',\n", + " 'last_id': 'msg_qS8ER2CuEovcezcm4ksQ4vOS'}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Create a message to append to our thread\n", "message = client.beta.threads.messages.create(thread_id=thread.id, role=\"user\", content=\"Could you explain this to me?\")\n", @@ -441,7 +688,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 73, "metadata": {}, "outputs": [], "source": [ @@ -475,7 +722,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -502,9 +749,88 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 75, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Messages\n", + "user: I need to solve the equation `3x + 11 = 14`. Can you help me?\n", + "assistant: Sure! To solve for \\( x \\), subtract 11 from both sides: \\( 3x = 3 \\). Then, divide by 3: \\( x = 1 \\).\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Messages\n", + "user: Could you explain linear algebra to me?\n", + "assistant: Linear algebra is the branch of mathematics that deals with vectors, vector spaces, linear transformations, and systems of linear equations.\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Messages\n", + "user: I don't like math. What can I do?\n", + "assistant: Try to relate math to your interests or find practical applications to make it more engaging.\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Messages\n", + "user: I don't like math. What can I do?\n", + "assistant: Try to relate math to your interests or find practical applications to make it more engaging.\n", + "user: Thank you!\n", + "assistant: You're welcome! If you have any more questions, feel free to ask!\n", + "\n" + ] + } + ], "source": [ "# Pretty printing helper\n", "def pretty_print(messages):\n", @@ -581,9 +907,32 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 76, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", + " 'created_at': 1742586617,\n", + " 'description': None,\n", + " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", + " 'metadata': {},\n", + " 'model': 'gpt-4o-mini',\n", + " 'name': 'Math Tutor',\n", + " 'object': 'assistant',\n", + " 'tools': [{'type': 'code_interpreter'}],\n", + " 'response_format': 'auto',\n", + " 'temperature': 1.0,\n", + " 'tool_resources': {'code_interpreter': {'file_ids': []}, 'file_search': None},\n", + " 'top_p': 1.0,\n", + " 'reasoning_effort': None}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "assistant = client.beta.assistants.update(\n", " MATH_ASSISTANT_ID,\n", @@ -601,9 +950,30 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 77, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Messages\n", + "user: Generate the first 20 fibbonaci numbers with code.\n", + "assistant: The first 20 Fibonacci numbers are: \n", + "\\[ 0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181 \\]\n", + "\n" + ] + } + ], "source": [ "thread, run = create_thread_and_run(\"Generate the first 20 fibbonaci numbers with code.\")\n", "run = wait_on_run(run, thread)\n", @@ -630,7 +1000,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -646,9 +1016,47 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 79, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tool_calls': [{'id': 'call_0tCXMy7C6207VwnBgSBw95tb',\n", + " 'code_interpreter': {'input': 'def fibonacci(n):\\r\\n fib_sequence = [0, 1]\\r\\n for i in range(2, n):\\r\\n next_fib = fib_sequence[-1] + fib_sequence[-2]\\r\\n fib_sequence.append(next_fib)\\r\\n return fib_sequence\\r\\n\\r\\n# Generate the first 20 Fibonacci numbers\\r\\nfirst_20_fib = fibonacci(20)\\r\\nfirst_20_fib',\n", + " 'outputs': []},\n", + " 'type': 'code_interpreter'}],\n", + " 'type': 'tool_calls'}" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "null\n" + ] + }, + { + "data": { + "text/plain": [ + "{'message_creation': {'message_id': 'msg_HHSmc6qRqBNy61nSCRWcwIvX'},\n", + " 'type': 'message_creation'}" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "null\n" + ] + } + ], "source": [ "for step in run_steps.data:\n", " step_details = step.step_details\n", @@ -690,9 +1098,33 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", + " 'created_at': 1742586617,\n", + " 'description': None,\n", + " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", + " 'metadata': {},\n", + " 'model': 'gpt-4o-mini',\n", + " 'name': 'Math Tutor',\n", + " 'object': 'assistant',\n", + " 'tools': [{'type': 'code_interpreter'}],\n", + " 'response_format': 'auto',\n", + " 'temperature': 1.0,\n", + " 'tool_resources': {'code_interpreter': {'file_ids': ['file-6jj1Aik1ZvocsraUDL4zZD']},\n", + " 'file_search': None},\n", + " 'top_p': 1.0,\n", + " 'reasoning_effort': None}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Upload the file\n", "file = client.files.create(\n", @@ -755,7 +1187,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -801,9 +1233,27 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Quiz: Sample Quiz\n", + "\n", + "What is your name?\n", + "\n", + "What is your favorite color?\n", + "0. Red\n", + "1. Blue\n", + "2. Green\n", + "3. Yellow\n", + "\n", + "Responses: [\"I don't know.\", 'a']\n" + ] + } + ], "source": [ "responses = display_quiz(\n", " \"Sample Quiz\",\n", @@ -828,7 +1278,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ @@ -885,9 +1335,48 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'asst_RPYHIeGQKlcD6nsIVfGGABG9',\n", + " 'created_at': 1742586485,\n", + " 'description': None,\n", + " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", + " 'metadata': {},\n", + " 'model': 'gpt-4o-mini',\n", + " 'name': 'Math Tutor',\n", + " 'object': 'assistant',\n", + " 'tools': [{'type': 'code_interpreter'},\n", + " {'function': {'name': 'display_quiz',\n", + " 'description': \"Displays a quiz to the student, and returns the student's response. A single quiz can have multiple questions.\",\n", + " 'parameters': {'type': 'object',\n", + " 'properties': {'title': {'type': 'string'},\n", + " 'questions': {'type': 'array',\n", + " 'description': 'An array of questions, each with a title and potentially options (if multiple choice).',\n", + " 'items': {'type': 'object',\n", + " 'properties': {'question_text': {'type': 'string'},\n", + " 'question_type': {'type': 'string',\n", + " 'enum': ['MULTIPLE_CHOICE', 'FREE_RESPONSE']},\n", + " 'choices': {'type': 'array', 'items': {'type': 'string'}}},\n", + " 'required': ['question_text']}}},\n", + " 'required': ['title', 'questions']},\n", + " 'strict': False},\n", + " 'type': 'function'}],\n", + " 'response_format': 'auto',\n", + " 'temperature': 1.0,\n", + " 'tool_resources': {'code_interpreter': {'file_ids': ['file-Mj5PeMjX8z59R58wJNQZGM']},\n", + " 'file_search': None},\n", + " 'top_p': 1.0,\n", + " 'reasoning_effort': None}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "assistant = client.beta.assistants.update(\n", " MATH_ASSISTANT_ID,\n", @@ -908,9 +1397,20 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'requires_action'" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "thread, run = create_thread_and_run(\n", " \"Make a quiz with 2 questions: One open ended, one multiple choice. Then, give me feedback for the responses.\"\n", @@ -928,9 +1428,70 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 56, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'run_NTIvvHANWWpcdS1tWRE172mT',\n", + " 'assistant_id': 'asst_RPYHIeGQKlcD6nsIVfGGABG9',\n", + " 'cancelled_at': None,\n", + " 'completed_at': None,\n", + " 'created_at': 1742586538,\n", + " 'expires_at': 1742587138,\n", + " 'failed_at': None,\n", + " 'incomplete_details': None,\n", + " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", + " 'last_error': None,\n", + " 'max_completion_tokens': None,\n", + " 'max_prompt_tokens': None,\n", + " 'metadata': {},\n", + " 'model': 'gpt-4o-mini',\n", + " 'object': 'thread.run',\n", + " 'parallel_tool_calls': True,\n", + " 'required_action': {'submit_tool_outputs': {'tool_calls': [{'id': 'call_LhxjmdyANSlkhxgCwrzz6KZJ',\n", + " 'function': {'arguments': '{\"title\": \"Math Quiz\", \"questions\": [{\"question_text\": \"Explain what the Pythagorean theorem states and when it is used.\", \"question_type\": \"FREE_RESPONSE\"}, {\"question_text\": \"What is the value of 8^2 + 6^2? \\\\nA) 100\\\\nB) 64\\\\nC) 36\\\\nD) 80\", \"question_type\": \"MULTIPLE_CHOICE\", \"choices\": [\"A) 100\", \"B) 64\", \"C) 36\", \"D) 80\"]}]}',\n", + " 'name': 'display_quiz'},\n", + " 'type': 'function'},\n", + " {'id': 'call_C60BxNOu2wrtKRKH353w8Gk2',\n", + " 'function': {'arguments': '{\"title\": \"Math Quiz\", \"questions\": [{\"question_text\": \"Explain what the Pythagorean theorem states and when it is used.\", \"question_type\": \"FREE RESPONSE\"}, {\"question_text\": \"What is the value of 8^2 + 6^2? \\\\nA) 100\\\\nB) 64\\\\nC) 36\\\\nD) 80\", \"question_type\": \"MULTIPLE_CHOICE\", \"choices\": [\"A) 100\", \"B) 64\", \"C) 36\", \"D) 80\"]}]}',\n", + " 'name': 'display_quiz'},\n", + " 'type': 'function'}]},\n", + " 'type': 'submit_tool_outputs'},\n", + " 'response_format': 'auto',\n", + " 'started_at': 1742586539,\n", + " 'status': 'requires_action',\n", + " 'thread_id': 'thread_NplU8C6YUpGBO87xgcBlzV7S',\n", + " 'tool_choice': 'auto',\n", + " 'tools': [{'type': 'code_interpreter'},\n", + " {'function': {'name': 'display_quiz',\n", + " 'description': \"Displays a quiz to the student, and returns the student's response. A single quiz can have multiple questions.\",\n", + " 'parameters': {'type': 'object',\n", + " 'properties': {'title': {'type': 'string'},\n", + " 'questions': {'type': 'array',\n", + " 'description': 'An array of questions, each with a title and potentially options (if multiple choice).',\n", + " 'items': {'type': 'object',\n", + " 'properties': {'question_text': {'type': 'string'},\n", + " 'question_type': {'type': 'string',\n", + " 'enum': ['MULTIPLE_CHOICE', 'FREE_RESPONSE']},\n", + " 'choices': {'type': 'array', 'items': {'type': 'string'}}},\n", + " 'required': ['question_text']}}},\n", + " 'required': ['title', 'questions']},\n", + " 'strict': False},\n", + " 'type': 'function'}],\n", + " 'truncation_strategy': {'type': 'auto', 'last_messages': None},\n", + " 'usage': None,\n", + " 'temperature': 1.0,\n", + " 'top_p': 1.0,\n", + " 'tool_resources': {},\n", + " 'reasoning_effort': None}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "show_json(run)" ] @@ -947,9 +1508,33 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 57, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Function Name: display_quiz\n", + "Function Arguments:\n" + ] + }, + { + "data": { + "text/plain": [ + "{'title': 'Math Quiz',\n", + " 'questions': [{'question_text': 'Explain what the Pythagorean theorem states and when it is used.',\n", + " 'question_type': 'FREE_RESPONSE'},\n", + " {'question_text': 'What is the value of 8^2 + 6^2? \\nA) 100\\nB) 64\\nC) 36\\nD) 80',\n", + " 'question_type': 'MULTIPLE_CHOICE',\n", + " 'choices': ['A) 100', 'B) 64', 'C) 36', 'D) 80']}]}" + ] + }, + "execution_count": 57, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Extract single tool call\n", "tool_call = run.required_action.submit_tool_outputs.tool_calls[0]\n", @@ -970,9 +1555,31 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Quiz: Math Quiz\n", + "\n", + "Explain what the Pythagorean theorem states and when it is used.\n", + "\n", + "What is the value of 8^2 + 6^2? \n", + "A) 100\n", + "B) 64\n", + "C) 36\n", + "D) 80\n", + "0. A) 100\n", + "1. B) 64\n", + "2. C) 36\n", + "3. D) 80\n", + "\n", + "Responses: [\"I don't know.\", 'a']\n" + ] + } + ], "source": [ "responses = display_quiz(arguments[\"title\"], arguments[\"questions\"])\n", "print(\"Responses:\", responses)" @@ -989,9 +1596,44 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 59, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Quiz: Math Quiz\n", + "\n", + "Explain what the Pythagorean theorem states and when it is used.\n", + "\n", + "What is the value of 8^2 + 6^2? \n", + "A) 100\n", + "B) 64\n", + "C) 36\n", + "D) 80\n", + "0. A) 100\n", + "1. B) 64\n", + "2. C) 36\n", + "3. D) 80\n", + "\n", + "Quiz: Math Quiz\n", + "\n", + "Explain what the Pythagorean theorem states and when it is used.\n", + "\n", + "What is the value of 8^2 + 6^2? \n", + "A) 100\n", + "B) 64\n", + "C) 36\n", + "D) 80\n", + "0. A) 100\n", + "1. B) 64\n", + "2. C) 36\n", + "3. D) 80\n", + "\n" + ] + } + ], "source": [ "tool_outputs = []\n", "tool_calls = run.required_action.submit_tool_outputs.tool_calls\n", @@ -1009,9 +1651,62 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 60, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'run_NTIvvHANWWpcdS1tWRE172mT',\n", + " 'assistant_id': 'asst_RPYHIeGQKlcD6nsIVfGGABG9',\n", + " 'cancelled_at': None,\n", + " 'completed_at': None,\n", + " 'created_at': 1742586538,\n", + " 'expires_at': 1742587138,\n", + " 'failed_at': None,\n", + " 'incomplete_details': None,\n", + " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", + " 'last_error': None,\n", + " 'max_completion_tokens': None,\n", + " 'max_prompt_tokens': None,\n", + " 'metadata': {},\n", + " 'model': 'gpt-4o-mini',\n", + " 'object': 'thread.run',\n", + " 'parallel_tool_calls': True,\n", + " 'required_action': None,\n", + " 'response_format': 'auto',\n", + " 'started_at': 1742586539,\n", + " 'status': 'queued',\n", + " 'thread_id': 'thread_NplU8C6YUpGBO87xgcBlzV7S',\n", + " 'tool_choice': 'auto',\n", + " 'tools': [{'type': 'code_interpreter'},\n", + " {'function': {'name': 'display_quiz',\n", + " 'description': \"Displays a quiz to the student, and returns the student's response. A single quiz can have multiple questions.\",\n", + " 'parameters': {'type': 'object',\n", + " 'properties': {'title': {'type': 'string'},\n", + " 'questions': {'type': 'array',\n", + " 'description': 'An array of questions, each with a title and potentially options (if multiple choice).',\n", + " 'items': {'type': 'object',\n", + " 'properties': {'question_text': {'type': 'string'},\n", + " 'question_type': {'type': 'string',\n", + " 'enum': ['MULTIPLE_CHOICE', 'FREE_RESPONSE']},\n", + " 'choices': {'type': 'array', 'items': {'type': 'string'}}},\n", + " 'required': ['question_text']}}},\n", + " 'required': ['title', 'questions']},\n", + " 'strict': False},\n", + " 'type': 'function'}],\n", + " 'truncation_strategy': {'type': 'auto', 'last_messages': None},\n", + " 'usage': None,\n", + " 'temperature': 1.0,\n", + " 'top_p': 1.0,\n", + " 'tool_resources': {},\n", + " 'reasoning_effort': None}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "run = client.beta.threads.runs.submit_tool_outputs(thread_id=thread.id, run_id=run.id, tool_outputs=tool_outputs)\n", "show_json(run)" @@ -1026,9 +1721,42 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 61, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", + "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Messages\n", + "user: Make a quiz with 2 questions: One open ended, one multiple choice. Then, give me feedback for the responses.\n", + "assistant: Here are the quiz questions:\n", + "\n", + "1. Explain what the Pythagorean theorem states and when it is used.\n", + "2. What is the value of \\(8^2 + 6^2\\)? \n", + " - A) 100\n", + " - B) 64\n", + " - C) 36\n", + " - D) 80\n", + "\n", + "### Feedback:\n", + "1. **Open-ended response:** It's okay to not remember; the Pythagorean theorem relates the lengths of the sides of a right triangle, stating that \\(a^2 + b^2 = c^2\\), where \\(c\\) is the hypotenuse.\n", + "2. **Multiple choice response:** The answer is A) 100, as \\(8^2 + 6^2 = 64 + 36 = 100\\). \n", + "\n", + "Feel free to ask if you have any questions or need more clarification!\n", + "\n" + ] + } + ], "source": [ "run = wait_on_run(run, thread)\n", "pretty_print(get_response(thread))" diff --git a/tests/handlers/conftest.py b/tests/handlers/conftest.py deleted file mode 100644 index a4635c339..000000000 --- a/tests/handlers/conftest.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Test fixtures for integrations.""" - -import os -import re -import uuid -from collections import defaultdict -from unittest import mock - -import pytest -import requests_mock -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider, ReadableSpan -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.trace import Status, StatusCode, SpanKind - -import agentops -from agentops.config import Config -from tests.fixtures.client import * # noqa -from tests.unit.sdk.instrumentation_tester import InstrumentationTester - - -@pytest.fixture -def api_key() -> str: - """Standard API key for testing""" - return "test-api-key" - - -@pytest.fixture -def endpoint() -> str: - """Base API URL""" - return Config().endpoint - - -@pytest.fixture(autouse=True) -def mock_req(endpoint, api_key): - """ - Mocks AgentOps backend API requests. - """ - with requests_mock.Mocker(real_http=False) as m: - # Map session IDs to their JWTs - m.post(endpoint + "/v3/auth/token", json={"token": str(uuid.uuid4()), - "project_id": "test-project-id", "api_key": api_key}) - yield m - - -@pytest.fixture -def noinstrument(): - # Tells the client to not instrument LLM calls - yield - - -@pytest.fixture -def mock_config(mocker): - """Mock the Client.configure method""" - return mocker.patch("agentops.client.Client.configure") - - -@pytest.fixture -def instrumentation(): - """Fixture for the instrumentation tester.""" - tester = InstrumentationTester() - yield tester - tester.reset() - - -@pytest.fixture -def tracer_provider(): - """Create a tracer provider with memory exporter for testing.""" - provider = TracerProvider() - exporter = InMemorySpanExporter() - processor = SimpleSpanProcessor(exporter) - provider.add_span_processor(processor) - trace.set_tracer_provider(provider) - return provider, exporter - - -@pytest.fixture -def mock_span(): - """Create a mock span for testing.""" - span = mock.Mock(spec=ReadableSpan) - span.name = "test_span" - span.kind = SpanKind.INTERNAL - span.attributes = {} - span.status = Status(StatusCode.OK) - span.parent = None - span.context = mock.Mock() - span.context.trace_id = 0x1234567890abcdef1234567890abcdef - span.context.span_id = 0x1234567890abcdef - return span - - -@pytest.fixture -def test_run_ids(): - """Create test run IDs for callback testing.""" - return { - "run_id": uuid.uuid4(), - "parent_run_id": uuid.uuid4(), - } - - -@pytest.fixture -def test_llm_inputs(): - """Create test LLM inputs for callback testing.""" - return { - "serialized": {"name": "test-model"}, - "prompts": ["test prompt"], - "metadata": {"test": "metadata"}, - } - - -@pytest.fixture -def test_chain_inputs(): - """Create test chain inputs for callback testing.""" - return { - "serialized": {"name": "test-chain"}, - "inputs": {"test": "input"}, - } - - -@pytest.fixture -def test_tool_inputs(): - """Create test tool inputs for callback testing.""" - return { - "serialized": {"name": "test-tool"}, - "input_str": "test input", - } - - -@pytest.fixture -def test_agent_inputs(): - """Create test agent inputs for callback testing.""" - return { - "action": mock.Mock( - tool="test-tool", - tool_input="test input", - log="test log", - ), - "finish": mock.Mock( - return_values={"output": "test output"}, - log="test log", - ), - } - - -@pytest.fixture -def test_retry_state(): - """Create test retry state for callback testing.""" - return type("RetryState", (), { - "attempt_number": 2, - "outcome": type("Outcome", (), { - "exception": lambda: Exception("test retry error"), - })(), - })() \ No newline at end of file diff --git a/tests/handlers/test_langchain_callback.py b/tests/handlers/test_langchain_callback.py deleted file mode 100644 index eab8918d0..000000000 --- a/tests/handlers/test_langchain_callback.py +++ /dev/null @@ -1,923 +0,0 @@ -"""Test suite for Langchain callback handlers. - -This test suite verifies the functionality of both synchronous and asynchronous -Langchain callback handlers. It tests the following aspects: - -1. Basic Functionality: - - Handler initialization and configuration - - Span creation and management - - Attribute recording - - Error handling - -2. LLM Operations: - - LLM start/end events - - Token streaming - - Error handling - - Response processing - -3. Chat Model Operations: - - Chat model start/end events - - Message handling - - Response processing - -4. Chain Operations: - - Chain start/end events - - Input/output handling - - Error propagation - -5. Tool Operations: - - Tool start/end events - - Input/output recording - - Error handling - -6. Retriever Operations: - - Retriever start/end events - - Query handling - - Document processing - -7. Agent Operations: - - Agent action events - - Tool usage tracking - - Finish event handling - -8. Error Scenarios: - - Exception handling - - Error propagation - - Span error status - -9. Async Functionality: - - Async handler initialization - - Async event handling - - Async error handling - -10. Edge Cases: - - Missing run IDs - - Invalid inputs - - Stream handling - - Retry scenarios - -The tests use mock objects to simulate Langchain operations and verify that -the handlers correctly create and manage OpenTelemetry spans with appropriate -attributes and error handling. -""" - -import asyncio -from typing import Dict, Any, List -from uuid import UUID, uuid4 -import pytest -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider, ReadableSpan -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.trace import Status, StatusCode, SpanKind -from contextlib import contextmanager -from unittest.mock import MagicMock, patch - -from langchain_core.agents import AgentFinish, AgentAction -from langchain_core.documents import Document -from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResult, Generation -from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, AIMessageChunk -from tenacity import RetryCallState - -from agentops import init -from agentops.sdk.core import TracingCore -from agentops.integrations.langchain.callback_handler import ( - LangchainCallbackHandler, - AsyncLangchainCallbackHandler, -) -from agentops.semconv import SpanKind -from agentops.semconv.span_attributes import SpanAttributes -from agentops.semconv.langchain_attributes import LangchainAttributes - - -pytestmark = pytest.mark.asyncio - - -def get_model_from_kwargs(kwargs: dict) -> str: - """Extract model name from kwargs.""" - if "model" in kwargs.get("invocation_params", {}): - return kwargs["invocation_params"]["model"] - elif "_type" in kwargs.get("invocation_params", {}): - return kwargs["invocation_params"]["_type"] - return "unknown_model" - - -@contextmanager -def _create_as_current_span( - name: str, - kind: SpanKind, - attributes: Dict[str, Any] = None, -): - """Create a span and set it as the current span.""" - tracer = trace.get_tracer(__name__) - with tracer.start_as_current_span( - name, - kind=kind, - attributes=attributes or {}, - ) as span: - yield span - - -@pytest.fixture(autouse=True) -def setup_agentops(): - """Initialize AgentOps client for testing.""" - init(api_key="test-api-key") - yield - # Cleanup will be handled by the test framework - - -@pytest.fixture -def tracer_provider(): - """Create a tracer provider with an in-memory exporter for testing.""" - provider = TracerProvider() - exporter = InMemorySpanExporter() - processor = SimpleSpanProcessor(exporter) - provider.add_span_processor(processor) - return provider, exporter - - -@pytest.fixture -def tracing_core(): - """Initialize TracingCore for testing.""" - core = TracingCore.get_instance() - core.initialize( - service_name="test_service", - ) - yield core - core.shutdown() - - -@pytest.fixture -def mock_client(): - """Create a mock AgentOps client.""" - with patch("agentops.Client") as mock: - client_instance = MagicMock() - mock.return_value = client_instance - client_instance.configure.return_value = None - client_instance.init.return_value = None - client_instance.current_session_ids = ["test-session-id"] - yield client_instance - - -@pytest.fixture -def callback_handler(mock_client): - """Create a callback handler with mocked client.""" - return LangchainCallbackHandler() - - -@pytest.fixture -def async_callback_handler(mock_client): - """Create an async callback handler with mocked client.""" - return AsyncLangchainCallbackHandler() - - -@pytest.fixture -def test_run_ids(): - """Generate test run IDs.""" - return { - "run_id": UUID("12345678-1234-5678-1234-567812345678"), - "parent_run_id": UUID("87654321-4321-8765-4321-876543210987"), - } - - -@pytest.fixture -def test_llm_inputs(): - """Generate test LLM inputs.""" - return { - "serialized": {"name": "test-llm"}, - "prompts": ["test prompt"], - "invocation_params": {"model": "test-model"}, - } - - -@pytest.fixture -def test_chain_inputs(): - """Generate test chain inputs.""" - return { - "serialized": {"name": "test-chain"}, - "inputs": {"input": "test input"}, - } - - -@pytest.fixture -def test_tool_inputs(): - """Generate test tool inputs.""" - return { - "serialized": {"name": "test-tool"}, - "input_str": "test input", - "inputs": {"input": "test input"}, - } - - -@pytest.fixture -def test_agent_inputs(): - """Generate test agent inputs.""" - return { - "action": AgentAction( - tool="test-tool", - tool_input="test input", - log="test log", - ), - "finish": AgentFinish( - return_values={"output": "test output"}, - log="test log", - ), - } - - -@pytest.fixture -def test_retry_state(): - """Generate test retry state.""" - state = MagicMock(spec=RetryCallState) - state.attempt_number = 1 - state.outcome = MagicMock() - state.outcome.exception.return_value = Exception("test error") - return state - - -def test_llm_events(callback_handler, test_run_ids, test_llm_inputs): - """Test LLM events.""" - # Test LLM start - callback_handler.on_llm_start( - **test_llm_inputs, - **test_run_ids, - ) - - # Test LLM end - response = LLMResult( - generations=[[Generation(text="test response")]], - llm_output={"token_usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}}, - ) - callback_handler.on_llm_end( - response=response, - **test_run_ids, - ) - - -def test_chain_events(callback_handler, test_run_ids, test_chain_inputs): - """Test chain events.""" - # Test chain start - callback_handler.on_chain_start( - **test_chain_inputs, - **test_run_ids, - ) - - # Test chain end - callback_handler.on_chain_end( - outputs={"output": "test output"}, - **test_run_ids, - ) - - -def test_tool_events(callback_handler, test_run_ids, test_tool_inputs): - """Test tool events.""" - # Test tool start - callback_handler.on_tool_start( - **test_tool_inputs, - **test_run_ids, - ) - - # Test tool end - callback_handler.on_tool_end( - output="test output", - **test_run_ids, - ) - - -def test_agent_events(callback_handler, test_run_ids, test_agent_inputs): - """Test agent events.""" - # Test agent action - callback_handler.on_agent_action( - action=test_agent_inputs["action"], - **test_run_ids, - ) - - # Test agent finish - callback_handler.on_agent_finish( - finish=test_agent_inputs["finish"], - **test_run_ids, - ) - - -def test_retry_events(callback_handler, test_run_ids, test_retry_state): - """Test retry events.""" - callback_handler.on_retry( - retry_state=test_retry_state, - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_llm_events(async_callback_handler, test_run_ids, test_llm_inputs): - """Test async LLM events.""" - # Test LLM start - await async_callback_handler.on_llm_start( - **test_llm_inputs, - **test_run_ids, - ) - - # Test LLM end - response = LLMResult( - generations=[[Generation(text="test response")]], - llm_output={"token_usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}}, - ) - await async_callback_handler.on_llm_end( - response=response, - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_chain_events(async_callback_handler, test_run_ids, test_chain_inputs): - """Test async chain events.""" - # Test chain start - await async_callback_handler.on_chain_start( - **test_chain_inputs, - **test_run_ids, - ) - - # Test chain end - await async_callback_handler.on_chain_end( - outputs={"output": "test output"}, - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_tool_events(async_callback_handler, test_run_ids, test_tool_inputs): - """Test async tool events.""" - # Test tool start - await async_callback_handler.on_tool_start( - **test_tool_inputs, - **test_run_ids, - ) - - # Test tool end - await async_callback_handler.on_tool_end( - output="test output", - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_agent_events(async_callback_handler, test_run_ids, test_agent_inputs): - """Test async agent events.""" - # Test agent action - await async_callback_handler.on_agent_action( - action=test_agent_inputs["action"], - **test_run_ids, - ) - - # Test agent finish - await async_callback_handler.on_agent_finish( - finish=test_agent_inputs["finish"], - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_retry_events(async_callback_handler, test_run_ids, test_retry_state): - """Test async retry events.""" - await async_callback_handler.on_retry( - retry_state=test_retry_state, - **test_run_ids, - ) - - -@pytest.fixture -def test_llm_responses(): - """Generate test LLM responses.""" - return { - "text_response": LLMResult( - generations=[[Generation(text="test response")]], - llm_output={"token_usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}}, - ), - "message_response": LLMResult( - generations=[[ChatGenerationChunk(message=AIMessageChunk(content="test message"))]], - llm_output={"token_usage": {"prompt_tokens": 5, "completion_tokens": 15, "total_tokens": 20}}, - ), - "empty_response": LLMResult( - generations=[], - llm_output=None, - ), - "error_response": LLMResult( - generations=[[Generation(text="error response")]], - llm_output={"error": "test error"}, - ), - } - - -@pytest.fixture -def test_chain_outputs(): - """Generate test chain outputs.""" - return { - "simple_output": {"output": "test output"}, - "complex_output": {"output": {"nested": "value", "list": [1, 2, 3]}}, - "error_output": {"error": "test error"}, - } - - -@pytest.fixture -def test_tool_outputs(): - """Generate test tool outputs.""" - return { - "success_output": "test output", - "exception_output": "_Exception", - "error_output": "error: test error", - } - - -@pytest.fixture -def test_agent_sequences(): - """Generate test agent action sequences.""" - return { - "single_action": [ - AgentAction(tool="tool1", tool_input="input1", log="log1"), - AgentFinish(return_values={"output": "output1"}, log="finish1"), - ], - "multiple_actions": [ - AgentAction(tool="tool1", tool_input="input1", log="log1"), - AgentAction(tool="tool2", tool_input="input2", log="log2"), - AgentFinish(return_values={"output": "output2"}, log="finish2"), - ], - "error_action": [ - AgentAction(tool="error_tool", tool_input="error_input", log="error_log"), - AgentFinish(return_values={"error": "test error"}, log="error_finish"), - ], - } - - -def test_llm_events_with_different_response_types(callback_handler, test_run_ids, test_llm_inputs, test_llm_responses): - """Test LLM event handling with various response types and scenarios. - - This test verifies that the handler correctly processes: - - LLM start events with different input configurations - - LLM end events with different response types: - * Text-based responses - * Message-based responses (using AIMessageChunk) - * Empty responses - * Error responses - - Streaming token updates - - Error handling scenarios - """ - # Test LLM start - callback_handler.on_llm_start( - **test_llm_inputs, - **test_run_ids, - ) - - # Test different response types - callback_handler.on_llm_end( - response=test_llm_responses["text_response"], - **test_run_ids, - ) - - # Test message-based response - callback_handler.on_llm_end( - response=test_llm_responses["message_response"], - **test_run_ids, - ) - - # Test empty response - callback_handler.on_llm_end( - response=test_llm_responses["empty_response"], - **test_run_ids, - ) - - # Test error response - callback_handler.on_llm_end( - response=test_llm_responses["error_response"], - **test_run_ids, - ) - - # Test streaming tokens - callback_handler.on_llm_new_token( - token="test", - **test_run_ids, - ) - callback_handler.on_llm_new_token( - token=" token", - **test_run_ids, - ) - - # Test LLM error - callback_handler.on_llm_error( - error=Exception("test error"), - **test_run_ids, - ) - - -def test_chain_events_with_metadata_and_outputs(callback_handler, test_run_ids, test_chain_inputs, test_chain_outputs): - """Test chain event handling with metadata and various output formats. - - This test verifies that the handler correctly processes: - - Chain start events with metadata - - Chain end events with different output formats: - * Simple key-value outputs - * Complex nested outputs - * Error outputs - - Chain error handling - """ - # Test chain start with metadata - callback_handler.on_chain_start( - **test_chain_inputs, - metadata={"test": "metadata"}, - **test_run_ids, - ) - - # Test different output types - callback_handler.on_chain_end( - outputs=test_chain_outputs["simple_output"], - **test_run_ids, - ) - - callback_handler.on_chain_end( - outputs=test_chain_outputs["complex_output"], - **test_run_ids, - ) - - # Test chain error - callback_handler.on_chain_error( - error=Exception("test error"), - **test_run_ids, - ) - - -def test_tool_events_with_exceptions_and_errors(callback_handler, test_run_ids, test_tool_inputs, test_tool_outputs): - """Test tool event handling with various input/output types and error scenarios. - - This test verifies that the handler correctly processes: - - Tool start events with different input configurations - - Tool end events with different output types: - * Successful outputs - * Exception outputs - * Error outputs - - Tool error handling - """ - # Test tool start with different inputs - callback_handler.on_tool_start( - **{k: v for k, v in test_tool_inputs.items() if k != 'inputs'}, - **test_run_ids, - ) - - # Test different output types - callback_handler.on_tool_end( - output=test_tool_outputs["success_output"], - **test_run_ids, - ) - - # Test exception tool - callback_handler.on_tool_end( - output=test_tool_outputs["exception_output"], - name="_Exception", - **test_run_ids, - ) - - # Test tool error - callback_handler.on_tool_error( - error=Exception("test error"), - **test_run_ids, - ) - - -def test_agent_events_with_action_sequences(callback_handler, test_run_ids, test_agent_sequences): - """Test agent event handling with different action sequences and scenarios. - - This test verifies that the handler correctly processes: - - Single action sequences (action + finish) - - Multiple action sequences (multiple actions + finish) - - Error action sequences - - Different types of agent actions and finishes - """ - # Test single action sequence - for action in test_agent_sequences["single_action"]: - if isinstance(action, AgentAction): - callback_handler.on_agent_action( - action=action, - **test_run_ids, - ) - else: - callback_handler.on_agent_finish( - finish=action, - **test_run_ids, - ) - - # Test multiple actions sequence - for action in test_agent_sequences["multiple_actions"]: - if isinstance(action, AgentAction): - callback_handler.on_agent_action( - action=action, - **test_run_ids, - ) - else: - callback_handler.on_agent_finish( - finish=action, - **test_run_ids, - ) - - # Test error sequence - for action in test_agent_sequences["error_action"]: - if isinstance(action, AgentAction): - callback_handler.on_agent_action( - action=action, - **test_run_ids, - ) - else: - callback_handler.on_agent_finish( - finish=action, - **test_run_ids, - ) - - -def test_retriever_events_with_documents(callback_handler, test_run_ids): - """Test retriever event handling with document processing. - - This test verifies that the handler correctly processes: - - Retriever start events with query information - - Retriever end events with document results - - Retriever error handling - """ - # Test retriever start - callback_handler.on_retriever_start( - serialized={"name": "test-retriever"}, - query="test query", - **test_run_ids, - ) - - # Test retriever end - callback_handler.on_retriever_end( - documents=[Document(page_content="test content")], - **test_run_ids, - ) - - # Test retriever error - callback_handler.on_retriever_error( - error=Exception("test error"), - **test_run_ids, - ) - - -def test_retry_events_with_different_states(callback_handler, test_run_ids, test_retry_state): - """Test retry event handling with different retry states and error types. - - This test verifies that the handler correctly processes: - - Retry events with standard retry states - - Retry events with different error types - - Retry state information tracking - """ - # Test retry with different states - callback_handler.on_retry( - retry_state=test_retry_state, - **test_run_ids, - ) - - # Test retry with different error types - error_state = MagicMock(spec=RetryCallState) - error_state.attempt_number = 2 - error_state.outcome = MagicMock() - error_state.outcome.exception.return_value = ValueError("test error") - callback_handler.on_retry( - retry_state=error_state, - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_llm_events_with_different_response_types(async_callback_handler, test_run_ids, test_llm_inputs, test_llm_responses): - """Test async LLM event handling with various response types and scenarios. - - This test verifies that the async handler correctly processes: - - Async LLM start events with different input configurations - - Async LLM end events with different response types: - * Text-based responses - * Message-based responses (using AIMessageChunk) - * Empty responses - * Error responses - - Async streaming token updates - - Async error handling scenarios - """ - # Test LLM start - await async_callback_handler.on_llm_start( - **test_llm_inputs, - **test_run_ids, - ) - - # Test different response types - await async_callback_handler.on_llm_end( - response=test_llm_responses["text_response"], - **test_run_ids, - ) - - # Test message-based response - await async_callback_handler.on_llm_end( - response=test_llm_responses["message_response"], - **test_run_ids, - ) - - # Test empty response - await async_callback_handler.on_llm_end( - response=test_llm_responses["empty_response"], - **test_run_ids, - ) - - # Test error response - await async_callback_handler.on_llm_end( - response=test_llm_responses["error_response"], - **test_run_ids, - ) - - # Test streaming tokens - await async_callback_handler.on_llm_new_token( - token="test", - **test_run_ids, - ) - await async_callback_handler.on_llm_new_token( - token=" token", - **test_run_ids, - ) - - # Test LLM error - await async_callback_handler.on_llm_error( - error=Exception("test error"), - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_chain_events_with_metadata_and_outputs(async_callback_handler, test_run_ids, test_chain_inputs, test_chain_outputs): - """Test async chain event handling with metadata and various output formats. - - This test verifies that the async handler correctly processes: - - Async chain start events with metadata - - Async chain end events with different output formats: - * Simple key-value outputs - * Complex nested outputs - * Error outputs - - Async chain error handling - """ - # Test chain start with metadata - await async_callback_handler.on_chain_start( - **test_chain_inputs, - metadata={"test": "metadata"}, - **test_run_ids, - ) - - # Test different output types - await async_callback_handler.on_chain_end( - outputs=test_chain_outputs["simple_output"], - **test_run_ids, - ) - - await async_callback_handler.on_chain_end( - outputs=test_chain_outputs["complex_output"], - **test_run_ids, - ) - - # Test chain error - await async_callback_handler.on_chain_error( - error=Exception("test error"), - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_tool_events_with_exceptions_and_errors(async_callback_handler, test_run_ids, test_tool_inputs, test_tool_outputs): - """Test async tool event handling with various input/output types and error scenarios. - - This test verifies that the async handler correctly processes: - - Async tool start events with different input configurations - - Async tool end events with different output types: - * Successful outputs - * Exception outputs - * Error outputs - - Async tool error handling - """ - # Test tool start with different inputs - await async_callback_handler.on_tool_start( - **{k: v for k, v in test_tool_inputs.items() if k != 'inputs'}, - **test_run_ids, - ) - - # Test different output types - await async_callback_handler.on_tool_end( - output=test_tool_outputs["success_output"], - **test_run_ids, - ) - - # Test exception tool - await async_callback_handler.on_tool_end( - output=test_tool_outputs["exception_output"], - name="_Exception", - **test_run_ids, - ) - - # Test tool error - await async_callback_handler.on_tool_error( - error=Exception("test error"), - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_agent_events_with_action_sequences(async_callback_handler, test_run_ids, test_agent_sequences): - """Test async agent event handling with different action sequences and scenarios. - - This test verifies that the async handler correctly processes: - - Async single action sequences (action + finish) - - Async multiple action sequences (multiple actions + finish) - - Async error action sequences - - Async different types of agent actions and finishes - """ - # Test single action sequence - for action in test_agent_sequences["single_action"]: - if isinstance(action, AgentAction): - await async_callback_handler.on_agent_action( - action=action, - **test_run_ids, - ) - else: - await async_callback_handler.on_agent_finish( - finish=action, - **test_run_ids, - ) - - # Test multiple actions sequence - for action in test_agent_sequences["multiple_actions"]: - if isinstance(action, AgentAction): - await async_callback_handler.on_agent_action( - action=action, - **test_run_ids, - ) - else: - await async_callback_handler.on_agent_finish( - finish=action, - **test_run_ids, - ) - - # Test error sequence - for action in test_agent_sequences["error_action"]: - if isinstance(action, AgentAction): - await async_callback_handler.on_agent_action( - action=action, - **test_run_ids, - ) - else: - await async_callback_handler.on_agent_finish( - finish=action, - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_retriever_events_with_documents(async_callback_handler, test_run_ids): - """Test async retriever event handling with document processing. - - This test verifies that the async handler correctly processes: - - Async retriever start events with query information - - Async retriever end events with document results - - Async retriever error handling - """ - # Test retriever start - await async_callback_handler.on_retriever_start( - serialized={"name": "test-retriever"}, - query="test query", - **test_run_ids, - ) - - # Test retriever end - await async_callback_handler.on_retriever_end( - documents=[Document(page_content="test content")], - **test_run_ids, - ) - - # Test retriever error - await async_callback_handler.on_retriever_error( - error=Exception("test error"), - **test_run_ids, - ) - - -@pytest.mark.asyncio -async def test_async_retry_events_with_different_states(async_callback_handler, test_run_ids, test_retry_state): - """Test async retry event handling with different retry states and error types. - - This test verifies that the async handler correctly processes: - - Async retry events with standard retry states - - Async retry events with different error types - - Async retry state information tracking - """ - # Test retry with different states - await async_callback_handler.on_retry( - retry_state=test_retry_state, - **test_run_ids, - ) - - # Test retry with different error types - error_state = MagicMock(spec=RetryCallState) - error_state.attempt_number = 2 - error_state.outcome = MagicMock() - error_state.outcome.exception.return_value = ValueError("test error") - await async_callback_handler.on_retry( - retry_state=error_state, - **test_run_ids, - ) \ No newline at end of file From 6b060c1c765876d96d7e1b3df0deae5ec42bf8b7 Mon Sep 17 00:00:00 2001 From: Dwij Patel Date: Sat, 22 Mar 2025 05:46:51 +0530 Subject: [PATCH 3/5] Removed outputs from notebook --- .../openai_assistants_example.ipynb | 860 ++---------------- 1 file changed, 66 insertions(+), 794 deletions(-) diff --git a/examples/openai_examples/openai_assistants_example.ipynb b/examples/openai_examples/openai_assistants_example.ipynb index 5fbde966b..b85f447b8 100644 --- a/examples/openai_examples/openai_assistants_example.ipynb +++ b/examples/openai_examples/openai_assistants_example.ipynb @@ -189,22 +189,9 @@ }, { "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(DEBUG) 🖇 AgentOps: Instrumentors have already been populated.\n", - "(DEBUG) 🖇 AgentOps: [DEBUG] BEFORE _make_span session.session - Current context: {'span_id': '79636eb80922dac8', 'trace_id': '15191e1a9ca916a408f7537d83c210c0', 'name': 'session.session', 'is_recording': }\n", - "(DEBUG) 🖇 AgentOps: Started span: session.session (kind: session)\n", - "🖇 AgentOps: \u001b[92m\u001b[34mSession started: https://app.agentops.ai/drilldown?session_id=15191e1a-9ca9-16a4-08f7-537d83c210c0\u001b[0m\u001b[0m\n", - "(DEBUG) 🖇 AgentOps: [DEBUG] CREATED _make_span session.session - span_id: 4c582481db9c04be, parent: 79636eb80922dac8\n", - "🖇 AgentOps: \u001b[34m\u001b[34mSession Replay: https://app.agentops.ai/drilldown?session_id=15191e1a-9ca9-16a4-08f7-537d83c210c0\u001b[0m\u001b[0m\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "agentops.init(api_key=AGENTOPS_API_KEY, default_tags=[\"openai\", \"beta-assistants\"],auto_start_session=True)\n", "client = OpenAI(api_key=OPENAI_API_KEY)" @@ -219,32 +206,9 @@ }, { "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", - " 'created_at': 1742586617,\n", - " 'description': None,\n", - " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", - " 'metadata': {},\n", - " 'model': 'gpt-4o-mini',\n", - " 'name': 'Math Tutor',\n", - " 'object': 'assistant',\n", - " 'tools': [],\n", - " 'response_format': 'auto',\n", - " 'temperature': 1.0,\n", - " 'tool_resources': {'code_interpreter': None, 'file_search': None},\n", - " 'top_p': 1.0,\n", - " 'reasoning_effort': None}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "assistant = client.beta.assistants.create(\n", " name=\"Math Tutor\",\n", @@ -284,23 +248,9 @@ }, { "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp',\n", - " 'created_at': 1742586618,\n", - " 'metadata': {},\n", - " 'object': 'thread',\n", - " 'tool_resources': {'code_interpreter': None, 'file_search': None}}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "thread = client.beta.threads.create()\n", "show_json(thread)" @@ -315,34 +265,9 @@ }, { "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'msg_H3Hcs3msbCoNxPUsb9xX8WnP',\n", - " 'assistant_id': None,\n", - " 'attachments': [],\n", - " 'completed_at': None,\n", - " 'content': [{'text': {'annotations': [],\n", - " 'value': 'I need to solve the equation `3x + 11 = 14`. Can you help me?'},\n", - " 'type': 'text'}],\n", - " 'created_at': 1742586618,\n", - " 'incomplete_at': None,\n", - " 'incomplete_details': None,\n", - " 'metadata': {},\n", - " 'object': 'thread.message',\n", - " 'role': 'user',\n", - " 'run_id': None,\n", - " 'status': None,\n", - " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp'}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "message = client.beta.threads.messages.create(\n", " thread_id=thread.id,\n", @@ -378,47 +303,9 @@ }, { "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'run_4iI0RxJyKGWYflcgcwQEqTbX',\n", - " 'assistant_id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", - " 'cancelled_at': None,\n", - " 'completed_at': None,\n", - " 'created_at': 1742586619,\n", - " 'expires_at': 1742587219,\n", - " 'failed_at': None,\n", - " 'incomplete_details': None,\n", - " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", - " 'last_error': None,\n", - " 'max_completion_tokens': None,\n", - " 'max_prompt_tokens': None,\n", - " 'metadata': {},\n", - " 'model': 'gpt-4o-mini',\n", - " 'object': 'thread.run',\n", - " 'parallel_tool_calls': True,\n", - " 'required_action': None,\n", - " 'response_format': 'auto',\n", - " 'started_at': None,\n", - " 'status': 'queued',\n", - " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp',\n", - " 'tool_choice': 'auto',\n", - " 'tools': [],\n", - " 'truncation_strategy': {'type': 'auto', 'last_messages': None},\n", - " 'usage': None,\n", - " 'temperature': 1.0,\n", - " 'top_p': 1.0,\n", - " 'tool_resources': {},\n", - " 'reasoning_effort': None}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "run = client.beta.threads.runs.create(\n", " thread_id=thread.id,\n", @@ -457,51 +344,9 @@ }, { "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'run_4iI0RxJyKGWYflcgcwQEqTbX',\n", - " 'assistant_id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", - " 'cancelled_at': None,\n", - " 'completed_at': 1742586621,\n", - " 'created_at': 1742586619,\n", - " 'expires_at': None,\n", - " 'failed_at': None,\n", - " 'incomplete_details': None,\n", - " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", - " 'last_error': None,\n", - " 'max_completion_tokens': None,\n", - " 'max_prompt_tokens': None,\n", - " 'metadata': {},\n", - " 'model': 'gpt-4o-mini',\n", - " 'object': 'thread.run',\n", - " 'parallel_tool_calls': True,\n", - " 'required_action': None,\n", - " 'response_format': 'auto',\n", - " 'started_at': 1742586620,\n", - " 'status': 'completed',\n", - " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp',\n", - " 'tool_choice': 'auto',\n", - " 'tools': [],\n", - " 'truncation_strategy': {'type': 'auto', 'last_messages': None},\n", - " 'usage': {'completion_tokens': 36,\n", - " 'prompt_tokens': 66,\n", - " 'total_tokens': 102,\n", - " 'prompt_token_details': {'cached_tokens': 0},\n", - " 'completion_tokens_details': {'reasoning_tokens': 0}},\n", - " 'temperature': 1.0,\n", - " 'top_p': 1.0,\n", - " 'tool_resources': {},\n", - " 'reasoning_effort': None}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "run = wait_on_run(run, thread)\n", "show_json(run)" @@ -523,63 +368,9 @@ }, { "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" - ] - }, - { - "data": { - "text/plain": [ - "{'data': [{'id': 'msg_WAMEjtq4CGdR28jNx8wa8Vhy',\n", - " 'assistant_id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", - " 'attachments': [],\n", - " 'completed_at': None,\n", - " 'content': [{'text': {'annotations': [],\n", - " 'value': 'Sure! Subtract 11 from both sides to get \\\\( 3x = 3 \\\\), then divide by 3 to find \\\\( x = 1 \\\\).'},\n", - " 'type': 'text'}],\n", - " 'created_at': 1742586620,\n", - " 'incomplete_at': None,\n", - " 'incomplete_details': None,\n", - " 'metadata': {},\n", - " 'object': 'thread.message',\n", - " 'role': 'assistant',\n", - " 'run_id': 'run_4iI0RxJyKGWYflcgcwQEqTbX',\n", - " 'status': None,\n", - " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp'},\n", - " {'id': 'msg_H3Hcs3msbCoNxPUsb9xX8WnP',\n", - " 'assistant_id': None,\n", - " 'attachments': [],\n", - " 'completed_at': None,\n", - " 'content': [{'text': {'annotations': [],\n", - " 'value': 'I need to solve the equation `3x + 11 = 14`. Can you help me?'},\n", - " 'type': 'text'}],\n", - " 'created_at': 1742586618,\n", - " 'incomplete_at': None,\n", - " 'incomplete_details': None,\n", - " 'metadata': {},\n", - " 'object': 'thread.message',\n", - " 'role': 'user',\n", - " 'run_id': None,\n", - " 'status': None,\n", - " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp'}],\n", - " 'has_more': False,\n", - " 'object': 'list',\n", - " 'first_id': 'msg_WAMEjtq4CGdR28jNx8wa8Vhy',\n", - " 'last_id': 'msg_H3Hcs3msbCoNxPUsb9xX8WnP'}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "messages = client.beta.threads.messages.list(thread_id=thread.id)\n", "show_json(messages)" @@ -601,47 +392,9 @@ }, { "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" - ] - }, - { - "data": { - "text/plain": [ - "{'data': [{'id': 'msg_qS8ER2CuEovcezcm4ksQ4vOS',\n", - " 'assistant_id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", - " 'attachments': [],\n", - " 'completed_at': None,\n", - " 'content': [{'text': {'annotations': [],\n", - " 'value': 'Of course! To solve the equation \\\\( 3x + 11 = 14 \\\\), follow these steps:\\n\\n1. **Isolate the term with \\\\( x \\\\)**: Subtract 11 from both sides to eliminate 11 on the left. This gives you:\\n \\\\[\\n 3x + 11 - 11 = 14 - 11\\n \\\\]\\n Simplifying this, we get:\\n \\\\[\\n 3x = 3\\n \\\\]\\n\\n2. **Solve for \\\\( x \\\\)**: Now, divide both sides by 3 to isolate \\\\( x \\\\):\\n \\\\[\\n \\\\frac{3x}{3} = \\\\frac{3}{3}\\n \\\\]\\n This simplifies to:\\n \\\\[\\n x = 1\\n \\\\]\\n\\nSo, the solution is \\\\( x = 1 \\\\).'},\n", - " 'type': 'text'}],\n", - " 'created_at': 1742586625,\n", - " 'incomplete_at': None,\n", - " 'incomplete_details': None,\n", - " 'metadata': {},\n", - " 'object': 'thread.message',\n", - " 'role': 'assistant',\n", - " 'run_id': 'run_5hso6XQCmns3Q5dSdcSM9QDS',\n", - " 'status': None,\n", - " 'thread_id': 'thread_uw17QRPJhQkAMtOTCGwKrBMp'}],\n", - " 'has_more': False,\n", - " 'object': 'list',\n", - " 'first_id': 'msg_qS8ER2CuEovcezcm4ksQ4vOS',\n", - " 'last_id': 'msg_qS8ER2CuEovcezcm4ksQ4vOS'}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Create a message to append to our thread\n", "message = client.beta.threads.messages.create(thread_id=thread.id, role=\"user\", content=\"Could you explain this to me?\")\n", @@ -749,88 +502,9 @@ }, { "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# Messages\n", - "user: I need to solve the equation `3x + 11 = 14`. Can you help me?\n", - "assistant: Sure! To solve for \\( x \\), subtract 11 from both sides: \\( 3x = 3 \\). Then, divide by 3: \\( x = 1 \\).\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# Messages\n", - "user: Could you explain linear algebra to me?\n", - "assistant: Linear algebra is the branch of mathematics that deals with vectors, vector spaces, linear transformations, and systems of linear equations.\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# Messages\n", - "user: I don't like math. What can I do?\n", - "assistant: Try to relate math to your interests or find practical applications to make it more engaging.\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# Messages\n", - "user: I don't like math. What can I do?\n", - "assistant: Try to relate math to your interests or find practical applications to make it more engaging.\n", - "user: Thank you!\n", - "assistant: You're welcome! If you have any more questions, feel free to ask!\n", - "\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Pretty printing helper\n", "def pretty_print(messages):\n", @@ -907,32 +581,9 @@ }, { "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", - " 'created_at': 1742586617,\n", - " 'description': None,\n", - " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", - " 'metadata': {},\n", - " 'model': 'gpt-4o-mini',\n", - " 'name': 'Math Tutor',\n", - " 'object': 'assistant',\n", - " 'tools': [{'type': 'code_interpreter'}],\n", - " 'response_format': 'auto',\n", - " 'temperature': 1.0,\n", - " 'tool_resources': {'code_interpreter': {'file_ids': []}, 'file_search': None},\n", - " 'top_p': 1.0,\n", - " 'reasoning_effort': None}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "assistant = client.beta.assistants.update(\n", " MATH_ASSISTANT_ID,\n", @@ -950,30 +601,9 @@ }, { "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# Messages\n", - "user: Generate the first 20 fibbonaci numbers with code.\n", - "assistant: The first 20 Fibonacci numbers are: \n", - "\\[ 0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181 \\]\n", - "\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "thread, run = create_thread_and_run(\"Generate the first 20 fibbonaci numbers with code.\")\n", "run = wait_on_run(run, thread)\n", @@ -1016,47 +646,9 @@ }, { "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'tool_calls': [{'id': 'call_0tCXMy7C6207VwnBgSBw95tb',\n", - " 'code_interpreter': {'input': 'def fibonacci(n):\\r\\n fib_sequence = [0, 1]\\r\\n for i in range(2, n):\\r\\n next_fib = fib_sequence[-1] + fib_sequence[-2]\\r\\n fib_sequence.append(next_fib)\\r\\n return fib_sequence\\r\\n\\r\\n# Generate the first 20 Fibonacci numbers\\r\\nfirst_20_fib = fibonacci(20)\\r\\nfirst_20_fib',\n", - " 'outputs': []},\n", - " 'type': 'code_interpreter'}],\n", - " 'type': 'tool_calls'}" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "null\n" - ] - }, - { - "data": { - "text/plain": [ - "{'message_creation': {'message_id': 'msg_HHSmc6qRqBNy61nSCRWcwIvX'},\n", - " 'type': 'message_creation'}" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "null\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "for step in run_steps.data:\n", " step_details = step.step_details\n", @@ -1098,33 +690,9 @@ }, { "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'asst_ElpoBZrn06hMdUp1LO5p2CFD',\n", - " 'created_at': 1742586617,\n", - " 'description': None,\n", - " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", - " 'metadata': {},\n", - " 'model': 'gpt-4o-mini',\n", - " 'name': 'Math Tutor',\n", - " 'object': 'assistant',\n", - " 'tools': [{'type': 'code_interpreter'}],\n", - " 'response_format': 'auto',\n", - " 'temperature': 1.0,\n", - " 'tool_resources': {'code_interpreter': {'file_ids': ['file-6jj1Aik1ZvocsraUDL4zZD']},\n", - " 'file_search': None},\n", - " 'top_p': 1.0,\n", - " 'reasoning_effort': None}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Upload the file\n", "file = client.files.create(\n", @@ -1233,27 +801,9 @@ }, { "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Quiz: Sample Quiz\n", - "\n", - "What is your name?\n", - "\n", - "What is your favorite color?\n", - "0. Red\n", - "1. Blue\n", - "2. Green\n", - "3. Yellow\n", - "\n", - "Responses: [\"I don't know.\", 'a']\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "responses = display_quiz(\n", " \"Sample Quiz\",\n", @@ -1335,48 +885,9 @@ }, { "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'asst_RPYHIeGQKlcD6nsIVfGGABG9',\n", - " 'created_at': 1742586485,\n", - " 'description': None,\n", - " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", - " 'metadata': {},\n", - " 'model': 'gpt-4o-mini',\n", - " 'name': 'Math Tutor',\n", - " 'object': 'assistant',\n", - " 'tools': [{'type': 'code_interpreter'},\n", - " {'function': {'name': 'display_quiz',\n", - " 'description': \"Displays a quiz to the student, and returns the student's response. A single quiz can have multiple questions.\",\n", - " 'parameters': {'type': 'object',\n", - " 'properties': {'title': {'type': 'string'},\n", - " 'questions': {'type': 'array',\n", - " 'description': 'An array of questions, each with a title and potentially options (if multiple choice).',\n", - " 'items': {'type': 'object',\n", - " 'properties': {'question_text': {'type': 'string'},\n", - " 'question_type': {'type': 'string',\n", - " 'enum': ['MULTIPLE_CHOICE', 'FREE_RESPONSE']},\n", - " 'choices': {'type': 'array', 'items': {'type': 'string'}}},\n", - " 'required': ['question_text']}}},\n", - " 'required': ['title', 'questions']},\n", - " 'strict': False},\n", - " 'type': 'function'}],\n", - " 'response_format': 'auto',\n", - " 'temperature': 1.0,\n", - " 'tool_resources': {'code_interpreter': {'file_ids': ['file-Mj5PeMjX8z59R58wJNQZGM']},\n", - " 'file_search': None},\n", - " 'top_p': 1.0,\n", - " 'reasoning_effort': None}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "assistant = client.beta.assistants.update(\n", " MATH_ASSISTANT_ID,\n", @@ -1397,20 +908,9 @@ }, { "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'requires_action'" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "thread, run = create_thread_and_run(\n", " \"Make a quiz with 2 questions: One open ended, one multiple choice. Then, give me feedback for the responses.\"\n", @@ -1428,70 +928,9 @@ }, { "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'run_NTIvvHANWWpcdS1tWRE172mT',\n", - " 'assistant_id': 'asst_RPYHIeGQKlcD6nsIVfGGABG9',\n", - " 'cancelled_at': None,\n", - " 'completed_at': None,\n", - " 'created_at': 1742586538,\n", - " 'expires_at': 1742587138,\n", - " 'failed_at': None,\n", - " 'incomplete_details': None,\n", - " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", - " 'last_error': None,\n", - " 'max_completion_tokens': None,\n", - " 'max_prompt_tokens': None,\n", - " 'metadata': {},\n", - " 'model': 'gpt-4o-mini',\n", - " 'object': 'thread.run',\n", - " 'parallel_tool_calls': True,\n", - " 'required_action': {'submit_tool_outputs': {'tool_calls': [{'id': 'call_LhxjmdyANSlkhxgCwrzz6KZJ',\n", - " 'function': {'arguments': '{\"title\": \"Math Quiz\", \"questions\": [{\"question_text\": \"Explain what the Pythagorean theorem states and when it is used.\", \"question_type\": \"FREE_RESPONSE\"}, {\"question_text\": \"What is the value of 8^2 + 6^2? \\\\nA) 100\\\\nB) 64\\\\nC) 36\\\\nD) 80\", \"question_type\": \"MULTIPLE_CHOICE\", \"choices\": [\"A) 100\", \"B) 64\", \"C) 36\", \"D) 80\"]}]}',\n", - " 'name': 'display_quiz'},\n", - " 'type': 'function'},\n", - " {'id': 'call_C60BxNOu2wrtKRKH353w8Gk2',\n", - " 'function': {'arguments': '{\"title\": \"Math Quiz\", \"questions\": [{\"question_text\": \"Explain what the Pythagorean theorem states and when it is used.\", \"question_type\": \"FREE RESPONSE\"}, {\"question_text\": \"What is the value of 8^2 + 6^2? \\\\nA) 100\\\\nB) 64\\\\nC) 36\\\\nD) 80\", \"question_type\": \"MULTIPLE_CHOICE\", \"choices\": [\"A) 100\", \"B) 64\", \"C) 36\", \"D) 80\"]}]}',\n", - " 'name': 'display_quiz'},\n", - " 'type': 'function'}]},\n", - " 'type': 'submit_tool_outputs'},\n", - " 'response_format': 'auto',\n", - " 'started_at': 1742586539,\n", - " 'status': 'requires_action',\n", - " 'thread_id': 'thread_NplU8C6YUpGBO87xgcBlzV7S',\n", - " 'tool_choice': 'auto',\n", - " 'tools': [{'type': 'code_interpreter'},\n", - " {'function': {'name': 'display_quiz',\n", - " 'description': \"Displays a quiz to the student, and returns the student's response. A single quiz can have multiple questions.\",\n", - " 'parameters': {'type': 'object',\n", - " 'properties': {'title': {'type': 'string'},\n", - " 'questions': {'type': 'array',\n", - " 'description': 'An array of questions, each with a title and potentially options (if multiple choice).',\n", - " 'items': {'type': 'object',\n", - " 'properties': {'question_text': {'type': 'string'},\n", - " 'question_type': {'type': 'string',\n", - " 'enum': ['MULTIPLE_CHOICE', 'FREE_RESPONSE']},\n", - " 'choices': {'type': 'array', 'items': {'type': 'string'}}},\n", - " 'required': ['question_text']}}},\n", - " 'required': ['title', 'questions']},\n", - " 'strict': False},\n", - " 'type': 'function'}],\n", - " 'truncation_strategy': {'type': 'auto', 'last_messages': None},\n", - " 'usage': None,\n", - " 'temperature': 1.0,\n", - " 'top_p': 1.0,\n", - " 'tool_resources': {},\n", - " 'reasoning_effort': None}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "show_json(run)" ] @@ -1508,33 +947,9 @@ }, { "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Function Name: display_quiz\n", - "Function Arguments:\n" - ] - }, - { - "data": { - "text/plain": [ - "{'title': 'Math Quiz',\n", - " 'questions': [{'question_text': 'Explain what the Pythagorean theorem states and when it is used.',\n", - " 'question_type': 'FREE_RESPONSE'},\n", - " {'question_text': 'What is the value of 8^2 + 6^2? \\nA) 100\\nB) 64\\nC) 36\\nD) 80',\n", - " 'question_type': 'MULTIPLE_CHOICE',\n", - " 'choices': ['A) 100', 'B) 64', 'C) 36', 'D) 80']}]}" - ] - }, - "execution_count": 57, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Extract single tool call\n", "tool_call = run.required_action.submit_tool_outputs.tool_calls[0]\n", @@ -1555,31 +970,9 @@ }, { "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Quiz: Math Quiz\n", - "\n", - "Explain what the Pythagorean theorem states and when it is used.\n", - "\n", - "What is the value of 8^2 + 6^2? \n", - "A) 100\n", - "B) 64\n", - "C) 36\n", - "D) 80\n", - "0. A) 100\n", - "1. B) 64\n", - "2. C) 36\n", - "3. D) 80\n", - "\n", - "Responses: [\"I don't know.\", 'a']\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "responses = display_quiz(arguments[\"title\"], arguments[\"questions\"])\n", "print(\"Responses:\", responses)" @@ -1596,44 +989,9 @@ }, { "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Quiz: Math Quiz\n", - "\n", - "Explain what the Pythagorean theorem states and when it is used.\n", - "\n", - "What is the value of 8^2 + 6^2? \n", - "A) 100\n", - "B) 64\n", - "C) 36\n", - "D) 80\n", - "0. A) 100\n", - "1. B) 64\n", - "2. C) 36\n", - "3. D) 80\n", - "\n", - "Quiz: Math Quiz\n", - "\n", - "Explain what the Pythagorean theorem states and when it is used.\n", - "\n", - "What is the value of 8^2 + 6^2? \n", - "A) 100\n", - "B) 64\n", - "C) 36\n", - "D) 80\n", - "0. A) 100\n", - "1. B) 64\n", - "2. C) 36\n", - "3. D) 80\n", - "\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "tool_outputs = []\n", "tool_calls = run.required_action.submit_tool_outputs.tool_calls\n", @@ -1651,62 +1009,9 @@ }, { "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'run_NTIvvHANWWpcdS1tWRE172mT',\n", - " 'assistant_id': 'asst_RPYHIeGQKlcD6nsIVfGGABG9',\n", - " 'cancelled_at': None,\n", - " 'completed_at': None,\n", - " 'created_at': 1742586538,\n", - " 'expires_at': 1742587138,\n", - " 'failed_at': None,\n", - " 'incomplete_details': None,\n", - " 'instructions': 'You are a personal math tutor. Answer questions briefly, in a sentence or less.',\n", - " 'last_error': None,\n", - " 'max_completion_tokens': None,\n", - " 'max_prompt_tokens': None,\n", - " 'metadata': {},\n", - " 'model': 'gpt-4o-mini',\n", - " 'object': 'thread.run',\n", - " 'parallel_tool_calls': True,\n", - " 'required_action': None,\n", - " 'response_format': 'auto',\n", - " 'started_at': 1742586539,\n", - " 'status': 'queued',\n", - " 'thread_id': 'thread_NplU8C6YUpGBO87xgcBlzV7S',\n", - " 'tool_choice': 'auto',\n", - " 'tools': [{'type': 'code_interpreter'},\n", - " {'function': {'name': 'display_quiz',\n", - " 'description': \"Displays a quiz to the student, and returns the student's response. A single quiz can have multiple questions.\",\n", - " 'parameters': {'type': 'object',\n", - " 'properties': {'title': {'type': 'string'},\n", - " 'questions': {'type': 'array',\n", - " 'description': 'An array of questions, each with a title and potentially options (if multiple choice).',\n", - " 'items': {'type': 'object',\n", - " 'properties': {'question_text': {'type': 'string'},\n", - " 'question_type': {'type': 'string',\n", - " 'enum': ['MULTIPLE_CHOICE', 'FREE_RESPONSE']},\n", - " 'choices': {'type': 'array', 'items': {'type': 'string'}}},\n", - " 'required': ['question_text']}}},\n", - " 'required': ['title', 'questions']},\n", - " 'strict': False},\n", - " 'type': 'function'}],\n", - " 'truncation_strategy': {'type': 'auto', 'last_messages': None},\n", - " 'usage': None,\n", - " 'temperature': 1.0,\n", - " 'top_p': 1.0,\n", - " 'tool_resources': {},\n", - " 'reasoning_effort': None}" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "run = client.beta.threads.runs.submit_tool_outputs(thread_id=thread.id, run_id=run.id, tool_outputs=tool_outputs)\n", "show_json(run)" @@ -1721,42 +1026,9 @@ }, { "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(DEBUG) 🖇 AgentOps: Started span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n", - "(DEBUG) 🖇 AgentOps: Ended span: openai.assistant.run (kind: unknown)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# Messages\n", - "user: Make a quiz with 2 questions: One open ended, one multiple choice. Then, give me feedback for the responses.\n", - "assistant: Here are the quiz questions:\n", - "\n", - "1. Explain what the Pythagorean theorem states and when it is used.\n", - "2. What is the value of \\(8^2 + 6^2\\)? \n", - " - A) 100\n", - " - B) 64\n", - " - C) 36\n", - " - D) 80\n", - "\n", - "### Feedback:\n", - "1. **Open-ended response:** It's okay to not remember; the Pythagorean theorem relates the lengths of the sides of a right triangle, stating that \\(a^2 + b^2 = c^2\\), where \\(c\\) is the hypotenuse.\n", - "2. **Multiple choice response:** The answer is A) 100, as \\(8^2 + 6^2 = 64 + 36 = 100\\). \n", - "\n", - "Feel free to ask if you have any questions or need more clarification!\n", - "\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "run = wait_on_run(run, thread)\n", "pretty_print(get_response(thread))" From 5fdee1ef3ebf9efa7e70e2fbb27eeb51f6abfb3e Mon Sep 17 00:00:00 2001 From: Dwij Patel Date: Sat, 22 Mar 2025 05:50:07 +0530 Subject: [PATCH 4/5] Revert changes in notebook --- examples/openai_examples/openai_assistants_example.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/openai_examples/openai_assistants_example.ipynb b/examples/openai_examples/openai_assistants_example.ipynb index b85f447b8..b0d6c3b83 100644 --- a/examples/openai_examples/openai_assistants_example.ipynb +++ b/examples/openai_examples/openai_assistants_example.ipynb @@ -193,7 +193,7 @@ "metadata": {}, "outputs": [], "source": [ - "agentops.init(api_key=AGENTOPS_API_KEY, default_tags=[\"openai\", \"beta-assistants\"],auto_start_session=True)\n", + "agentops.init(api_key=AGENTOPS_API_KEY, default_tags=[\"openai\", \"beta-assistants\"])\n", "client = OpenAI(api_key=OPENAI_API_KEY)" ] }, From 4ae4bf22a86c3cd45ac2c6067ee8b8378bdf0d45 Mon Sep 17 00:00:00 2001 From: Dwij Patel Date: Tue, 25 Mar 2025 01:49:19 +0530 Subject: [PATCH 5/5] Remove LangChain callback handler and related files, including utility functions and documentation, to streamline the AgentOps SDK. Updated semconv, excluded model provider info --- .../callbacks/langchain/README.md | 0 .../callbacks/langchain/__init__.py | 2 +- .../callbacks/langchain/callback.py | 58 +++++-------- .../integration/callbacks/langchain/utils.py | 55 ++++++++++++ agentops/sdk/callbacks/langchain/utils.py | 84 ------------------- agentops/semconv/langchain.py | 44 +++++----- agentops/semconv/span_attributes.py | 1 + 7 files changed, 103 insertions(+), 141 deletions(-) rename agentops/{sdk => integration}/callbacks/langchain/README.md (100%) rename agentops/{sdk => integration}/callbacks/langchain/__init__.py (81%) rename agentops/{sdk => integration}/callbacks/langchain/callback.py (94%) create mode 100644 agentops/integration/callbacks/langchain/utils.py delete mode 100644 agentops/sdk/callbacks/langchain/utils.py diff --git a/agentops/sdk/callbacks/langchain/README.md b/agentops/integration/callbacks/langchain/README.md similarity index 100% rename from agentops/sdk/callbacks/langchain/README.md rename to agentops/integration/callbacks/langchain/README.md diff --git a/agentops/sdk/callbacks/langchain/__init__.py b/agentops/integration/callbacks/langchain/__init__.py similarity index 81% rename from agentops/sdk/callbacks/langchain/__init__.py rename to agentops/integration/callbacks/langchain/__init__.py index 794594bfa..6bdcf8e6a 100644 --- a/agentops/sdk/callbacks/langchain/__init__.py +++ b/agentops/integration/callbacks/langchain/__init__.py @@ -4,7 +4,7 @@ This module provides the AgentOps LangChain integration, including callbacks and utilities. """ -from agentops.sdk.callbacks.langchain.callback import ( +from agentops.integration.callbacks.langchain.callback import ( LangchainCallbackHandler, AsyncLangchainCallbackHandler, ) diff --git a/agentops/sdk/callbacks/langchain/callback.py b/agentops/integration/callbacks/langchain/callback.py similarity index 94% rename from agentops/sdk/callbacks/langchain/callback.py rename to agentops/integration/callbacks/langchain/callback.py index 8f1907426..4ebb70ef7 100644 --- a/agentops/sdk/callbacks/langchain/callback.py +++ b/agentops/integration/callbacks/langchain/callback.py @@ -13,8 +13,8 @@ from agentops.helpers.serialization import safe_serialize from agentops.logging import logger from agentops.sdk.core import TracingCore -from agentops.semconv import SpanKind, SpanAttributes, LangChainAttributes, LangChainAttributeValues -from agentops.sdk.callbacks.langchain.utils import get_model_info +from agentops.semconv import SpanKind, SpanAttributes, LangChainAttributes, LangChainAttributeValues, CoreAttributes +from agentops.integration.callbacks.langchain.utils import get_model_info from langchain_core.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler from langchain_core.outputs import LLMResult @@ -78,7 +78,7 @@ def _initialize_agentops(self): attributes = { SpanAttributes.AGENTOPS_SPAN_KIND: SpanKind.SESSION, - LangChainAttributes.SESSION_TAGS: self.tags, + "session.tags": self.tags, "agentops.operation.name": "session", "span.kind": SpanKind.SESSION, } @@ -196,30 +196,23 @@ def on_llm_start( model_info = get_model_info(serialized) # Ensure default values if model_info returns unknown model_name = model_info.get("model_name", "unknown") - model_provider = model_info.get("provider", "unknown") attributes = { + # Use both standard and LangChain-specific attributes SpanAttributes.LLM_REQUEST_MODEL: model_name, - SpanAttributes.LLM_PROMPTS: safe_serialize(prompts), LangChainAttributes.LLM_MODEL: model_name, + SpanAttributes.LLM_PROMPTS: safe_serialize(prompts), LangChainAttributes.LLM_NAME: serialized.get("id", "unknown_llm"), - LangChainAttributes.LLM_PROVIDER: model_provider, } if "kwargs" in serialized: for key, value in serialized["kwargs"].items(): if key == "temperature": - param_key = f"gen_ai.request.{key}" - attributes[param_key] = value - attributes[LangChainAttributes.LLM_TEMPERATURE] = value + attributes[SpanAttributes.LLM_REQUEST_TEMPERATURE] = value elif key == "max_tokens": - param_key = f"gen_ai.request.{key}" - attributes[param_key] = value - attributes[LangChainAttributes.LLM_MAX_TOKENS] = value + attributes[SpanAttributes.LLM_REQUEST_MAX_TOKENS] = value elif key == "top_p": - param_key = f"gen_ai.request.{key}" - attributes[param_key] = value - attributes[LangChainAttributes.LLM_TOP_P] = value + attributes[SpanAttributes.LLM_REQUEST_TOP_P] = value run_id = kwargs.get("run_id", id(serialized or {})) parent_run_id = kwargs.get("parent_run_id", None) @@ -233,7 +226,7 @@ def on_llm_start( self._create_span("llm", SpanKind.LLM, run_id, attributes, parent_run_id) - logger.debug(f"Started LLM span for {model_name} ({model_provider})") + logger.debug(f"Started LLM span for {model_name}") except Exception as e: logger.warning(f"Error in on_llm_start: {e}") @@ -298,7 +291,7 @@ def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None: if run_id in self.token_counts and self.token_counts[run_id] > 0: try: span.set_attribute( - "llm.usage.streaming_tokens", + SpanAttributes.LLM_USAGE_STREAMING_TOKENS, self.token_counts[run_id] ) except Exception as e: @@ -554,7 +547,6 @@ def on_chat_model_start( model_info = get_model_info(serialized) # Ensure default values if model_info returns unknown model_name = model_info.get("model_name", "unknown") - model_provider = model_info.get("provider", "unknown") # Extract message contents and roles formatted_messages = [] @@ -569,11 +561,11 @@ def on_chat_model_start( roles.append(message.type) attributes = { + # Use both standard and LangChain-specific attributes SpanAttributes.LLM_REQUEST_MODEL: model_name, - SpanAttributes.LLM_PROMPTS: safe_serialize(formatted_messages), LangChainAttributes.LLM_MODEL: model_name, + SpanAttributes.LLM_PROMPTS: safe_serialize(formatted_messages), LangChainAttributes.LLM_NAME: serialized.get("id", "unknown_chat_model"), - LangChainAttributes.LLM_PROVIDER: model_provider, LangChainAttributes.CHAT_MESSAGE_ROLES: safe_serialize(roles), LangChainAttributes.CHAT_MODEL_TYPE: "chat", } @@ -582,17 +574,11 @@ def on_chat_model_start( if "kwargs" in serialized: for key, value in serialized["kwargs"].items(): if key == "temperature": - param_key = f"gen_ai.request.{key}" - attributes[param_key] = value - attributes[LangChainAttributes.LLM_TEMPERATURE] = value + attributes[SpanAttributes.LLM_REQUEST_TEMPERATURE] = value elif key == "max_tokens": - param_key = f"gen_ai.request.{key}" - attributes[param_key] = value - attributes[LangChainAttributes.LLM_MAX_TOKENS] = value + attributes[SpanAttributes.LLM_REQUEST_MAX_TOKENS] = value elif key == "top_p": - param_key = f"gen_ai.request.{key}" - attributes[param_key] = value - attributes[LangChainAttributes.LLM_TOP_P] = value + attributes[SpanAttributes.LLM_REQUEST_TOP_P] = value run_id = kwargs.get("run_id", id(serialized or {})) parent_run_id = kwargs.get("parent_run_id", None) @@ -602,7 +588,7 @@ def on_chat_model_start( self._create_span("chat_model", SpanKind.LLM, run_id, attributes, parent_run_id) - logger.debug(f"Started Chat Model span for {model_name} ({model_provider})") + logger.debug(f"Started Chat Model span for {model_name}") except Exception as e: logger.warning(f"Error in on_chat_model_start: {e}") @@ -625,10 +611,10 @@ def on_llm_error( "error", True ) span.set_attribute( - "error.type", error.__class__.__name__ + CoreAttributes.ERROR_TYPE, error.__class__.__name__ ) span.set_attribute( - "error.message", str(error) + CoreAttributes.ERROR_MESSAGE, str(error) ) span.set_attribute( LangChainAttributes.LLM_ERROR, str(error) @@ -661,10 +647,10 @@ def on_chain_error( "error", True ) span.set_attribute( - "error.type", error.__class__.__name__ + CoreAttributes.ERROR_TYPE, error.__class__.__name__ ) span.set_attribute( - "error.message", str(error) + CoreAttributes.ERROR_MESSAGE, str(error) ) span.set_attribute( LangChainAttributes.CHAIN_ERROR, str(error) @@ -697,10 +683,10 @@ def on_tool_error( "error", True ) span.set_attribute( - "error.type", error.__class__.__name__ + CoreAttributes.ERROR_TYPE, error.__class__.__name__ ) span.set_attribute( - "error.message", str(error) + CoreAttributes.ERROR_MESSAGE, str(error) ) span.set_attribute( LangChainAttributes.TOOL_ERROR, str(error) diff --git a/agentops/integration/callbacks/langchain/utils.py b/agentops/integration/callbacks/langchain/utils.py new file mode 100644 index 000000000..913984372 --- /dev/null +++ b/agentops/integration/callbacks/langchain/utils.py @@ -0,0 +1,55 @@ +""" +Utility functions for LangChain integration. +""" + +from typing import Any, Dict, Optional + +from agentops.helpers.serialization import safe_serialize +from agentops.logging import logger + + +def get_model_info(serialized: Optional[Dict[str, Any]]) -> Dict[str, str]: + """ + Extract model information from serialized LangChain data. + + This function attempts to extract model name information + from the serialized data of a LangChain model. + + Args: + serialized: Serialized data from LangChain + + Returns: + Dictionary with model_name key + """ + if serialized is None: + return {"model_name": "unknown"} + + model_info = {"model_name": "unknown"} + + try: + if isinstance(serialized.get("id"), list) and len(serialized["id"]) > 0: + id_list = serialized["id"] + if len(id_list) > 0: + model_info["model_name"] = id_list[-1] + + if isinstance(serialized.get("model_name"), str): + model_info["model_name"] = serialized["model_name"] + + elif serialized.get("id") and isinstance(serialized.get("id"), str): + model_id = serialized.get("id", "") + if "/" in model_id: + _, model_name = model_id.split("/", 1) + model_info["model_name"] = model_name + else: + model_info["model_name"] = model_id + + if serialized.get("kwargs") and isinstance(serialized["kwargs"], dict): + if serialized["kwargs"].get("model_name"): + model_info["model_name"] = serialized["kwargs"]["model_name"] + elif serialized["kwargs"].get("model"): + model_info["model_name"] = serialized["kwargs"]["model"] + + except Exception as e: + logger.warning(f"Error extracting model info: {e}") + + return model_info \ No newline at end of file diff --git a/agentops/sdk/callbacks/langchain/utils.py b/agentops/sdk/callbacks/langchain/utils.py deleted file mode 100644 index bc103baf7..000000000 --- a/agentops/sdk/callbacks/langchain/utils.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Utility functions for LangChain integration. -""" - -from typing import Any, Dict, Optional - -from agentops.helpers.serialization import safe_serialize -from agentops.logging import logger - - -def get_model_info(serialized: Optional[Dict[str, Any]]) -> Dict[str, str]: - """ - Extract model information from serialized LangChain data. - - This function attempts to extract model name and provider information - from the serialized data of a LangChain model. - - Args: - serialized: Serialized data from LangChain - - Returns: - Dictionary with provider and model_name keys - """ - if serialized is None: - return {"provider": "unknown", "model_name": "unknown"} - - model_info = {"provider": "unknown", "model_name": "unknown"} - - try: - if isinstance(serialized.get("id"), list) and len(serialized["id"]) > 0: - id_list = serialized["id"] - - for item in id_list: - if isinstance(item, str): - item_lower = item.lower() - if any(provider in item_lower for provider in ["openai", "anthropic", "google", "azure", "huggingface", "replicate", "cohere", "llama"]): - model_info["provider"] = item - break - - if model_info["model_name"] == "unknown" and len(id_list) > 0: - model_info["model_name"] = id_list[-1] - - if isinstance(serialized.get("model_name"), str): - model_info["model_name"] = serialized["model_name"] - - elif serialized.get("id") and isinstance(serialized.get("id"), str): - model_id = serialized.get("id", "") - if "/" in model_id: - provider, model_name = model_id.split("/", 1) - model_info["provider"] = provider - model_info["model_name"] = model_name - else: - model_info["model_name"] = model_id - - if serialized.get("kwargs") and isinstance(serialized["kwargs"], dict): - if serialized["kwargs"].get("model_name"): - model_info["model_name"] = serialized["kwargs"]["model_name"] - elif serialized["kwargs"].get("model"): - model_info["model_name"] = serialized["kwargs"]["model"] - - if serialized.get("_type") and model_info["provider"] == "unknown": - model_info["provider"] = str(serialized["_type"]) - - if model_info["provider"] == "unknown" and model_info["model_name"] != "unknown": - model_name_lower = model_info["model_name"].lower() - if "gpt" in model_name_lower: - model_info["provider"] = "openai" - elif "claude" in model_name_lower: - model_info["provider"] = "anthropic" - elif "palm" in model_name_lower or "gemini" in model_name_lower: - model_info["provider"] = "google" - elif "llama" in model_name_lower: - model_info["provider"] = "meta" - - if serialized.get("name") and model_info["provider"] == "unknown": - name_lower = str(serialized["name"]).lower() - if "openai" in name_lower: - model_info["provider"] = "openai" - elif "anthropic" in name_lower: - model_info["provider"] = "anthropic" - except Exception as e: - logger.warning(f"Error extracting model info: {e}") - - return model_info \ No newline at end of file diff --git a/agentops/semconv/langchain.py b/agentops/semconv/langchain.py index 677b19e14..c462c695c 100644 --- a/agentops/semconv/langchain.py +++ b/agentops/semconv/langchain.py @@ -1,43 +1,45 @@ """Semantic conventions for LangChain instrumentation.""" - -from enum import Enum - - class LangChainAttributeValues: """Standard values for LangChain attributes.""" CHAIN_KIND_SEQUENTIAL = "sequential" CHAIN_KIND_LLM = "llm" CHAIN_KIND_ROUTER = "router" + + # Chat message roles + ROLE_SYSTEM = "system" + ROLE_USER = "user" + ROLE_ASSISTANT = "assistant" + ROLE_FUNCTION = "function" + ROLE_TOOL = "tool" class LangChainAttributes: - """Attributes for LangChain instrumentation.""" + """ + Attributes for LangChain instrumentation. + Note: LLM-specific attributes are derived from SpanAttributes to maintain + consistency across instrumentations. + """ + + # Session attributes SESSION_TAGS = "langchain.session.tags" + # Chain attributes - specific to LangChain CHAIN_NAME = "langchain.chain.name" CHAIN_TYPE = "langchain.chain.type" CHAIN_ERROR = "langchain.chain.error" CHAIN_KIND = "langchain.chain.kind" CHAIN_VERBOSE = "langchain.chain.verbose" - LLM_NAME = "langchain.llm.name" - LLM_MODEL = "langchain.llm.model" - LLM_PROVIDER = "langchain.llm.provider" - LLM_TEMPERATURE = "langchain.llm.temperature" - LLM_MAX_TOKENS = "langchain.llm.max_tokens" - LLM_TOP_P = "langchain.llm.top_p" - LLM_ERROR = "langchain.llm.error" - - AGENT_ACTION_LOG = "langchain.agent.action_log" - AGENT_FINISH_RETURN_VALUES = "langchain.agent.finish.return_values" - AGENT_FINISH_LOG = "langchain.agent.finish.log" + # Agent attributes - specific to LangChain agents AGENT_ACTION_LOG = "langchain.agent.action.log" AGENT_ACTION_INPUT = "langchain.agent.action.input" - AGENT_FINISH_RETURN_VALUES = "langchain.agent.finish.return_values" AGENT_ACTION_TOOL = "langchain.agent.action.tool" + AGENT_FINISH_RETURN_VALUES = "langchain.agent.finish.return_values" + AGENT_FINISH_LOG = "langchain.agent.finish.log" + # Tool attributes - specific to LangChain tools TOOL_NAME = "langchain.tool.name" TOOL_INPUT = "langchain.tool.input" TOOL_OUTPUT = "langchain.tool.output" @@ -46,9 +48,11 @@ class LangChainAttributes: TOOL_ARGS_SCHEMA = "langchain.tool.args_schema" TOOL_RETURN_DIRECT = "langchain.tool.return_direct" - MESSAGE_ROLE = "langchain.message.role" - + # Chat attributes - specific to LangChain chat models CHAT_MESSAGE_ROLES = "langchain.chat_message.roles" CHAT_MODEL_TYPE = "langchain.chat_model.type" - TEXT_CONTENT = "langchain.text.content" \ No newline at end of file + # Text callback attributes + TEXT_CONTENT = "langchain.text.content" + + LLM_ERROR = "langchain.llm.error" \ No newline at end of file diff --git a/agentops/semconv/span_attributes.py b/agentops/semconv/span_attributes.py index 324c7b443..27476801a 100644 --- a/agentops/semconv/span_attributes.py +++ b/agentops/semconv/span_attributes.py @@ -37,6 +37,7 @@ class SpanAttributes: LLM_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens" LLM_USAGE_CACHE_CREATION_INPUT_TOKENS = "gen_ai.usage.cache_creation_input_tokens" LLM_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read_input_tokens" + LLM_USAGE_STREAMING_TOKENS = "gen_ai.usage.streaming_tokens" # Token type LLM_TOKEN_TYPE = "gen_ai.token.type"