From 7f8c4c8f2d7f3d457ffb5281353f516f9f07b8bf Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 15:56:14 -0700 Subject: [PATCH 01/25] examples with validators + new validationcheck logic --- agentops/__init__.py | 36 ++- agentops/validation.py | 289 ++++++++++++++++++ examples/ag2/async_human_input.py | 12 + examples/ag2/tools_wikipedia_search.py | 12 + .../agentops-anthropic-understanding-tools.py | 12 + examples/anthropic/anthropic-example-async.py | 12 + examples/anthropic/anthropic-example-sync.py | 12 + examples/crewai/job_posting.py | 12 + examples/crewai/markdown_validator.py | 11 + examples/langgraph/langgraph_example.py | 11 + examples/litellm/litellm_example.py | 12 + examples/mem0/mem0_memory_example.py | 37 ++- examples/openai/multi_tool_orchestration.py | 12 + examples/openai/openai_example_async.py | 12 +- examples/openai/openai_example_sync.py | 10 + examples/openai/web_search.py | 12 + examples/openai_agents/agent_guardrails.py | 13 + examples/openai_agents/agent_patterns.py | 12 + examples/openai_agents/agents_tools.py | 11 + .../openai_agents/customer_service_agent.py | 12 + .../smolagents/multi_smolagents_system.py | 14 +- examples/smolagents/text_to_sql.py | 13 + examples/xai/grok_examples.py | 12 + examples/xai/grok_vision_examples.py | 12 + 24 files changed, 588 insertions(+), 25 deletions(-) create mode 100644 agentops/validation.py diff --git a/agentops/__init__.py b/agentops/__init__.py index fc4ceb273..d7ad52930 100755 --- a/agentops/__init__.py +++ b/agentops/__init__.py @@ -34,6 +34,13 @@ from agentops.helpers.deprecation import deprecated, warn_deprecated_param import threading +# Import validation functions +from agentops.validation import ( + validate_trace_spans, + print_validation_summary, + ValidationError +) + # Thread-safe client management _client_lock = threading.Lock() _client = None @@ -442,37 +449,40 @@ def extract_key_from_attr(attr_value: str) -> str: __all__ = [ - "init", - "configure", - "get_client", - "record", - "start_trace", - "end_trace", - "update_trace_metadata", + # Legacy exports "start_session", "end_session", "track_agent", "track_tool", "end_all_sessions", + "Session", "ToolEvent", "ErrorEvent", "ActionEvent", "LLMEvent", - "Session", + # Modern exports + "init", + "start_trace", + "end_trace", + "update_trace_metadata", + "Client", + "get_client", + # Decorators "trace", "session", "agent", "task", "workflow", "operation", - "guardrail", - "tracer", "tool", - # Trace state enums + "guardrail", + # Enums "TraceState", "SUCCESS", "ERROR", "UNSET", - # OpenTelemetry status codes (for advanced users) - "StatusCode", + # Validation + "validate_trace_spans", + "print_validation_summary", + "ValidationError", ] diff --git a/agentops/validation.py b/agentops/validation.py new file mode 100644 index 000000000..ada86bb3a --- /dev/null +++ b/agentops/validation.py @@ -0,0 +1,289 @@ +""" +AgentOps Validation Module + +This module provides functions to validate that spans have been sent to AgentOps +using the public API. This is useful for testing and verification purposes. +""" + +import time +import requests +from typing import Optional, Dict, List, Any, Tuple + +from agentops.logging import logger +from agentops.exceptions import ApiServerException + + +class ValidationError(Exception): + """Raised when span validation fails.""" + pass + + +def get_jwt_token(api_key: Optional[str] = None) -> str: + """ + Exchange API key for JWT token. + + Args: + api_key: Optional API key. If not provided, uses AGENTOPS_API_KEY env var. + + Returns: + JWT bearer token + + Raises: + ApiServerException: If token exchange fails + """ + if api_key is None: + from agentops import get_client + client = get_client() + if client and client.config.api_key: + api_key = client.config.api_key + else: + import os + api_key = os.getenv("AGENTOPS_API_KEY") + if not api_key: + raise ValueError("No API key provided and AGENTOPS_API_KEY environment variable not set") + + try: + response = requests.post( + "https://api.agentops.ai/public/v1/auth/access_token", + json={"api_key": api_key}, + timeout=10 + ) + response.raise_for_status() + return response.json()["bearer"] + except requests.exceptions.RequestException as e: + raise ApiServerException(f"Failed to get JWT token: {e}") + + +def get_trace_details(trace_id: str, jwt_token: str) -> Dict[str, Any]: + """ + Get trace details from AgentOps API. + + Args: + trace_id: The trace ID to query + jwt_token: JWT authentication token + + Returns: + Trace details including spans + + Raises: + ApiServerException: If API request fails + """ + try: + response = requests.get( + f"https://api.agentops.ai/public/v1/traces/{trace_id}", + headers={"Authorization": f"Bearer {jwt_token}"}, + timeout=10 + ) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + raise ApiServerException(f"Failed to get trace details: {e}") + + +def get_trace_metrics(trace_id: str, jwt_token: str) -> Dict[str, Any]: + """ + Get trace metrics from AgentOps API. + + Args: + trace_id: The trace ID to query + jwt_token: JWT authentication token + + Returns: + Trace metrics including token counts and costs + + Raises: + ApiServerException: If API request fails + """ + try: + response = requests.get( + f"https://api.agentops.ai/public/v1/traces/{trace_id}/metrics", + headers={"Authorization": f"Bearer {jwt_token}"}, + timeout=10 + ) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + raise ApiServerException(f"Failed to get trace metrics: {e}") + + +def check_llm_spans(spans: List[Dict[str, Any]]) -> Tuple[bool, List[str]]: + """ + Check if any LLM spans are present in the trace. + + Args: + spans: List of span dictionaries + + Returns: + Tuple of (has_llm_spans, llm_span_names) + """ + llm_spans = [] + + for span in spans: + # Check span name for common LLM patterns + span_name = span.get("span_name", "").lower() + if any(pattern in span_name for pattern in [ + "openai", "gpt", "claude", "anthropic", "llm", "chat", + "completion", "gemini", "mistral", "cohere", "ai21" + ]): + llm_spans.append(span["span_name"]) + + return len(llm_spans) > 0, llm_spans + + +def validate_trace_spans( + trace_id: Optional[str] = None, + trace_context: Optional[Any] = None, + max_retries: int = 10, + retry_delay: float = 1.0, + check_llm: bool = True, + min_spans: int = 1, + api_key: Optional[str] = None +) -> Dict[str, Any]: + """ + Validate that spans have been sent to AgentOps. + + Args: + trace_id: Direct trace ID to validate + trace_context: TraceContext object from start_trace (alternative to trace_id) + max_retries: Maximum number of retries to wait for spans to appear + retry_delay: Delay between retries in seconds + check_llm: Whether to specifically check for LLM spans + min_spans: Minimum number of spans expected + api_key: Optional API key (uses environment variable if not provided) + + Returns: + Dictionary containing validation results and metrics + + Raises: + ValidationError: If validation fails + ValueError: If neither trace_id nor trace_context is provided + """ + # Extract trace ID + if trace_id is None and trace_context is None: + # Try to get from current span + try: + from opentelemetry.trace import get_current_span + current_span = get_current_span() + if current_span and hasattr(current_span, 'get_span_context'): + span_context = current_span.get_span_context() + if hasattr(span_context, 'trace_id') and span_context.trace_id: + if isinstance(span_context.trace_id, int): + trace_id = format(span_context.trace_id, "032x") + else: + trace_id = str(span_context.trace_id) + except ImportError: + pass + + elif trace_context is not None and trace_id is None: + # Extract from TraceContext + if hasattr(trace_context, 'span') and trace_context.span: + span_context = trace_context.span.get_span_context() + if hasattr(span_context, 'trace_id'): + if isinstance(span_context.trace_id, int): + trace_id = format(span_context.trace_id, "032x") + else: + trace_id = str(span_context.trace_id) + + if trace_id is None: + raise ValueError("No trace ID found. Provide either trace_id or trace_context parameter.") + + # Get JWT token + jwt_token = get_jwt_token(api_key) + + logger.info(f"Validating spans for trace ID: {trace_id}") + + for attempt in range(max_retries): + try: + # Get trace details + trace_details = get_trace_details(trace_id, jwt_token) + spans = trace_details.get("spans", []) + + if len(spans) >= min_spans: + logger.info(f"Found {len(spans)} span(s) in trace") + + # Prepare result + result = { + "trace_id": trace_id, + "span_count": len(spans), + "spans": spans, + "has_llm_spans": False, + "llm_span_names": [], + "metrics": None + } + + # Check for LLM spans if requested + if check_llm: + has_llm_spans, llm_span_names = check_llm_spans(spans) + result["has_llm_spans"] = has_llm_spans + result["llm_span_names"] = llm_span_names + + if has_llm_spans: + logger.info(f"Found LLM spans: {', '.join(llm_span_names)}") + elif check_llm: + logger.warning("No LLM spans found in trace") + + # Get metrics + try: + metrics = get_trace_metrics(trace_id, jwt_token) + result["metrics"] = metrics + + if metrics: + logger.info(f"Trace metrics - Total tokens: {metrics.get('total_tokens', 0)}, " + f"Cost: ${metrics.get('total_cost', '0.0000')}") + except Exception as e: + logger.warning(f"Could not retrieve metrics: {e}") + + # Validate based on requirements + if check_llm and not has_llm_spans: + raise ValidationError( + f"No LLM spans found in trace {trace_id}. " + f"Found spans: {[s.get('span_name', 'unnamed') for s in spans]}" + ) + + return result + + else: + logger.debug(f"Only {len(spans)} spans found, expected at least {min_spans}. " + f"Retrying... ({attempt + 1}/{max_retries})") + + except ApiServerException as e: + logger.debug(f"API error during validation: {e}. Retrying... ({attempt + 1}/{max_retries})") + + if attempt < max_retries - 1: + time.sleep(retry_delay) + + raise ValidationError( + f"Validation failed for trace {trace_id} after {max_retries} attempts. " + f"Expected at least {min_spans} spans" + + (", including LLM spans" if check_llm else "") + + ". Please check that tracking is properly configured." + ) + + +def print_validation_summary(result: Dict[str, Any]) -> None: + """ + Print a user-friendly summary of validation results. + + Args: + result: Validation result dictionary from validate_trace_spans + """ + print("\n" + "="*50) + print("šŸ” AgentOps Span Validation Results") + print("="*50) + + print(f"āœ… Found {result['span_count']} span(s) in trace") + + if result.get('has_llm_spans'): + print(f"āœ… Found LLM spans: {', '.join(result['llm_span_names'])}") + elif result.get('llm_span_names') is not None: + print("āš ļø No LLM spans detected") + + if result.get('metrics'): + metrics = result['metrics'] + print(f"\nšŸ“Š Trace Metrics:") + print(f" - Total tokens: {metrics.get('total_tokens', 0)}") + print(f" - Prompt tokens: {metrics.get('prompt_tokens', 0)}") + print(f" - Completion tokens: {metrics.get('completion_tokens', 0)}") + print(f" - Total cost: ${metrics.get('total_cost', '0.0000')}") + + print("\nāœ… Validation successful!") diff --git a/examples/ag2/async_human_input.py b/examples/ag2/async_human_input.py index 6484773a4..4b6265d64 100644 --- a/examples/ag2/async_human_input.py +++ b/examples/ag2/async_human_input.py @@ -104,3 +104,15 @@ async def main(): # await main() agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + diff --git a/examples/ag2/tools_wikipedia_search.py b/examples/ag2/tools_wikipedia_search.py index 95aff7308..391de9f82 100644 --- a/examples/ag2/tools_wikipedia_search.py +++ b/examples/ag2/tools_wikipedia_search.py @@ -70,3 +70,15 @@ ) agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + diff --git a/examples/anthropic/agentops-anthropic-understanding-tools.py b/examples/anthropic/agentops-anthropic-understanding-tools.py index bbce678ff..b15a3d78c 100644 --- a/examples/anthropic/agentops-anthropic-understanding-tools.py +++ b/examples/anthropic/agentops-anthropic-understanding-tools.py @@ -371,3 +371,15 @@ def inventoryscan(): message = response.content[0].text print(message) + + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + diff --git a/examples/anthropic/anthropic-example-async.py b/examples/anthropic/anthropic-example-async.py index f1bff79a5..5d5b63d7d 100644 --- a/examples/anthropic/anthropic-example-async.py +++ b/examples/anthropic/anthropic-example-async.py @@ -108,3 +108,15 @@ async def main(): # Run the main function asyncio.run(main()) # We can observe the trace in the AgentOps dashboard by going to the trace URL provided above. + + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + diff --git a/examples/anthropic/anthropic-example-sync.py b/examples/anthropic/anthropic-example-sync.py index f97429964..06bd10d23 100644 --- a/examples/anthropic/anthropic-example-sync.py +++ b/examples/anthropic/anthropic-example-sync.py @@ -110,3 +110,15 @@ # # Now we will end the session with a success message. We can also end the session with a failure or intdeterminate status. By default, the session will be marked as indeterminate. agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + diff --git a/examples/crewai/job_posting.py b/examples/crewai/job_posting.py index 3f8d6021c..99853c494 100644 --- a/examples/crewai/job_posting.py +++ b/examples/crewai/job_posting.py @@ -172,3 +172,15 @@ def industry_analysis_task(self, agent, company_domain, company_description): print(result) agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + diff --git a/examples/crewai/markdown_validator.py b/examples/crewai/markdown_validator.py index ea9d36761..19184aa8b 100644 --- a/examples/crewai/markdown_validator.py +++ b/examples/crewai/markdown_validator.py @@ -104,3 +104,14 @@ def markdown_validation_tool(file_path: str) -> str: # Now lets run our task! syntax_review_task.execute_sync() + + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/langgraph/langgraph_example.py b/examples/langgraph/langgraph_example.py index 2a42b84f6..d63397034 100644 --- a/examples/langgraph/langgraph_example.py +++ b/examples/langgraph/langgraph_example.py @@ -116,3 +116,14 @@ def run_example(): if __name__ == "__main__": run_example() print("āœ… Check your AgentOps dashboard for the trace!") + + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/litellm/litellm_example.py b/examples/litellm/litellm_example.py index 8c9126c83..437f8b456 100644 --- a/examples/litellm/litellm_example.py +++ b/examples/litellm/litellm_example.py @@ -50,3 +50,15 @@ print(response.choices[0].message.content) agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + diff --git a/examples/mem0/mem0_memory_example.py b/examples/mem0/mem0_memory_example.py index 4a51e79d9..0573e06d6 100644 --- a/examples/mem0/mem0_memory_example.py +++ b/examples/mem0/mem0_memory_example.py @@ -26,8 +26,8 @@ os.environ["AGENTOPS_LOG_LEVEL"] = "DEBUG" # Set environment variables before importing -os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY") -os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY") +os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "") +os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "") # Now import mem0 - it will be instrumented by agentops from mem0 import Memory, AsyncMemory # noqa E402 @@ -57,7 +57,7 @@ def demonstrate_sync_memory(local_config, sample_messages, sample_preferences, u must complete before the next one begins. """ - agentops.start_trace("mem0_memory_example", tags=["mem0_memory_example"]) + tracer = agentops.start_trace("mem0_memory_example", tags=["mem0_memory_example"]) try: # Initialize sync Memory with local configuration memory = Memory.from_config(local_config) @@ -96,9 +96,10 @@ def demonstrate_sync_memory(local_config, sample_messages, sample_preferences, u delete_all_result = memory.delete_all(user_id=user_id) print(f"Delete all result: {delete_all_result}") - agentops.end_trace(end_state="success") - except Exception: - agentops.end_trace(end_state="error") + agentops.end_trace(tracer, end_state="Success") + except Exception as e: + agentops.end_trace(tracer, end_state="Error") + raise e async def demonstrate_async_memory(local_config, sample_messages, sample_preferences, user_id): @@ -122,7 +123,7 @@ async def demonstrate_async_memory(local_config, sample_messages, sample_prefere by running multiple memory operations in parallel. """ - agentops.start_trace("mem0_memory_async_example") + tracer = agentops.start_trace("mem0_memory_async_example") try: # Initialize async Memory with configuration async_memory = await AsyncMemory.from_config(local_config) @@ -177,12 +178,16 @@ async def search_memory(query): delete_all_result = await async_memory.delete_all(user_id=user_id) print(f"Delete all result: {delete_all_result}") - agentops.end_trace(end_state="success") + agentops.end_trace(tracer, end_state="Success") - except Exception: - agentops.end_trace(end_state="error") + except Exception as e: + agentops.end_trace(tracer, end_state="Error") + raise e +# Initialize AgentOps +agentops.init() + # Configuration for local memory (Memory) # This configuration specifies the LLM provider and model settings local_config = { @@ -222,3 +227,15 @@ async def search_memory(query): # Execute both sync and async demonstrations demonstrate_sync_memory(local_config, sample_messages, sample_preferences, user_id) asyncio.run(demonstrate_async_memory(local_config, sample_messages, sample_preferences, user_id)) + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + # Note: Using trace_id since we ran multiple traces + # In a real application, you would store each tracer and validate individually + result = agentops.validate_trace_spans(check_llm=False) # Don't check for LLM spans as this uses memory operations + agentops.print_validation_summary(result) +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/openai/multi_tool_orchestration.py b/examples/openai/multi_tool_orchestration.py index 552751b86..d0ded8974 100644 --- a/examples/openai/multi_tool_orchestration.py +++ b/examples/openai/multi_tool_orchestration.py @@ -342,4 +342,16 @@ def query_pinecone_index(client, index, model, query_text): print(response_2.output_text) agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # Here, we have seen how to utilize OpenAI's Responses API to implement a Retrieval-Augmented Generation (RAG) approach with multi-tool calling capabilities. It showcases an example where the model selects the appropriate tool based on the input query: general questions may be handled by built-in tools such as web-search, while specific medical inquiries related to internal knowledge are addressed by retrieving context from a vector database (such as Pinecone) via function calls. Additonally, we have showcased how multiple tool calls can be sequentially combined to generate a final response based on our instructions provided to responses API. Happy coding! + diff --git a/examples/openai/openai_example_async.py b/examples/openai/openai_example_async.py index 1e32f52b4..7e18ed1a3 100644 --- a/examples/openai/openai_example_async.py +++ b/examples/openai/openai_example_async.py @@ -14,7 +14,6 @@ import os import asyncio from dotenv import load_dotenv - # Next, we'll grab our API keys. You can use dotenv like below or however else you like to load environment variables load_dotenv() os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") @@ -78,5 +77,16 @@ async def main_stream(): asyncio.run(main_stream()) agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # Note that the response is a generator that yields chunks of the story. We can track this with AgentOps by navigating to the trace url and viewing the run. # All done! diff --git a/examples/openai/openai_example_sync.py b/examples/openai/openai_example_sync.py index 6b535251b..13501b59b 100644 --- a/examples/openai/openai_example_sync.py +++ b/examples/openai/openai_example_sync.py @@ -64,5 +64,15 @@ agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + result = agentops.validate_trace_spans(trace_context=tracer) + agentops.print_validation_summary(result) +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + # Note that the response is a generator that yields chunks of the story. We can track this with AgentOps by navigating to the trace url and viewing the run. # All done! diff --git a/examples/openai/web_search.py b/examples/openai/web_search.py index d632e0cc4..2aa051f1d 100644 --- a/examples/openai/web_search.py +++ b/examples/openai/web_search.py @@ -101,6 +101,17 @@ print(json.dumps(response_multimodal.__dict__, default=lambda o: o.__dict__, indent=4)) agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # In the above example, we were able to use the `web_search` tool to search the web for news related to the image in one API call instead of multiple round trips that would be required if we were using the Chat Completions API. # With the responses API # šŸ”„ a single API call can handle: @@ -120,3 +131,4 @@ # 3ļøāƒ£ Re-submit tool results for summarization ā†’ another request # # We are very excited for you to try out the Responses API and see how it can simplify your code and make it easier to build complex, multimodal, tool-augmented interactions! + diff --git a/examples/openai_agents/agent_guardrails.py b/examples/openai_agents/agent_guardrails.py index 2429dab4b..ace92488f 100644 --- a/examples/openai_agents/agent_guardrails.py +++ b/examples/openai_agents/agent_guardrails.py @@ -10,6 +10,7 @@ # Import dependencies from pydantic import BaseModel from agents import ( + Agent, GuardrailFunctionOutput, InputGuardrailTripwireTriggered, @@ -81,3 +82,15 @@ async def main(): if __name__ == "__main__": asyncio.run(main()) + + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + diff --git a/examples/openai_agents/agent_patterns.py b/examples/openai_agents/agent_patterns.py index 5482fc7b4..f2357649e 100644 --- a/examples/openai_agents/agent_patterns.py +++ b/examples/openai_agents/agent_patterns.py @@ -723,6 +723,17 @@ async def run_streaming_guardrails_demo(): # End the AgentOps trace session agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + # ## Conclusion # # This notebook has demonstrated 9 key agent patterns that are commonly used in production AI applications. Each pattern showcases how agents can be orchestrated to perform complex tasks, validate inputs and outputs, and improve overall application performance. @@ -737,3 +748,4 @@ async def run_streaming_guardrails_demo(): # - **Workflow optimization** - Identifying bottlenecks and improving agent coordination # # Visit [app.agentops.ai](https://app.agentops.ai) to explore your agent sessions and gain deeper insights into your AI application's behavior. + diff --git a/examples/openai_agents/agents_tools.py b/examples/openai_agents/agents_tools.py index 71c8644ea..b76bea472 100644 --- a/examples/openai_agents/agents_tools.py +++ b/examples/openai_agents/agents_tools.py @@ -248,6 +248,17 @@ async def run_web_search_demo(): # End the AgentOps trace session agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # ## Conclusion # # Each tool extends agent capabilities and enables sophisticated automation. **AgentOps makes tool observability effortless** - simply import the library and all your tool interactions are automatically tracked, visualized, and analyzed. This enables you to: diff --git a/examples/openai_agents/customer_service_agent.py b/examples/openai_agents/customer_service_agent.py index 92fa34bb1..67b19f56b 100644 --- a/examples/openai_agents/customer_service_agent.py +++ b/examples/openai_agents/customer_service_agent.py @@ -187,6 +187,17 @@ async def main(): # await main() agentops.end_trace(tracer, status="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # ## Conclusion # # **AgentOps makes observability effortless** - simply import the library and all your interactions are automatically tracked, visualized, and analyzed. This enables you to: @@ -197,3 +208,4 @@ async def main(): # - Scale your AI applications with confidence in tool reliability # # Visit [app.agentops.ai](https://app.agentops.ai) to explore your tool usage sessions and gain deeper insights into your AI application's tool interactions. + diff --git a/examples/smolagents/multi_smolagents_system.py b/examples/smolagents/multi_smolagents_system.py index c14740b30..e223c1291 100644 --- a/examples/smolagents/multi_smolagents_system.py +++ b/examples/smolagents/multi_smolagents_system.py @@ -82,7 +82,7 @@ def visit_webpage(url: str) -> str: except RequestException as e: return f"Error fetching the webpage: {str(e)}" - except Exception as e: + except agentops.ValidationError as e: return f"An unexpected error occurred: {str(e)}" @@ -111,4 +111,16 @@ def visit_webpage(url: str) -> str: print(answer) # Awesome! We've successfully run a multi-agent system. Let's end the agentops session with a "Success" state. You can also end the session with a "Failure" or "Indeterminate" state, which is set as default. agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + # You can view the session in the [AgentOps dashboard](https://app.agentops.ai/sessions) by clicking the link provided after ending the session. + diff --git a/examples/smolagents/text_to_sql.py b/examples/smolagents/text_to_sql.py index 10eeefa66..e532c0695 100644 --- a/examples/smolagents/text_to_sql.py +++ b/examples/smolagents/text_to_sql.py @@ -15,6 +15,7 @@ # %pip install agentops # ## Setting up the SQL Table from sqlalchemy import ( + create_engine, MetaData, Table, @@ -156,4 +157,16 @@ def sql_engine(query: str) -> str: agent.run("Which waiter got more total money from tips?") # All done! Now we can end the agentops session with a "Success" state. You can also end the session with a "Failure" or "Indeterminate" state, where the "Indeterminate" state is used by default. agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + # You can view the session in the [AgentOps dashboard](https://app.agentops.ai/sessions) by clicking the link provided after ending the session. + diff --git a/examples/xai/grok_examples.py b/examples/xai/grok_examples.py index 43718a2ca..420f42a51 100644 --- a/examples/xai/grok_examples.py +++ b/examples/xai/grok_examples.py @@ -77,4 +77,16 @@ # Awesome! We can now transliterate from English to any language! And all of this can be tracked with AgentOps by going to the session url above. agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # We end the session with a success state and a success reason. This is useful if you want to track the success or failure of the chatbot. In that case you can set the end state to failure and provide a reason. By default the session will have an indeterminate end state. + diff --git a/examples/xai/grok_vision_examples.py b/examples/xai/grok_vision_examples.py index 3f620d637..8ceb56efb 100644 --- a/examples/xai/grok_vision_examples.py +++ b/examples/xai/grok_vision_examples.py @@ -60,4 +60,16 @@ # Awesome! It returns a fascinating response explaining the image and also deciphering the text content. All of this can be tracked with AgentOps by going to the session url above. agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # We end the session with a success state and a success reason. This is useful if you want to track the success or failure of the chatbot. In that case you can set the end state to failure and provide a reason. By default the session will have an indeterminate end state. + From 55a2ab43b7c1d70fde814bf4aebd6258635803c2 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 16:02:37 -0700 Subject: [PATCH 02/25] pytest fix --- tests/unit/test_validation.py | 259 ++++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 tests/unit/test_validation.py diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py new file mode 100644 index 000000000..f9a97308d --- /dev/null +++ b/tests/unit/test_validation.py @@ -0,0 +1,259 @@ +""" +Unit tests for the AgentOps validation module. +""" + +import pytest +from unittest.mock import patch, Mock, MagicMock +import requests + +from agentops.validation import ( + get_jwt_token, + get_trace_details, + get_trace_metrics, + check_llm_spans, + validate_trace_spans, + ValidationError, + print_validation_summary, +) +from agentops.exceptions import ApiServerException + + +class TestGetJwtToken: + """Test JWT token exchange functionality.""" + + @patch('agentops.validation.requests.post') + def test_get_jwt_token_success(self, mock_post): + """Test successful JWT token retrieval.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"bearer": "test-token"} + mock_post.return_value = mock_response + + token = get_jwt_token("test-api-key") + assert token == "test-token" + + mock_post.assert_called_once_with( + "https://api.agentops.ai/public/v1/auth/access_token", + json={"api_key": "test-api-key"}, + timeout=10 + ) + + @patch('agentops.validation.requests.post') + def test_get_jwt_token_failure(self, mock_post): + """Test JWT token retrieval failure.""" + mock_response = Mock() + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("401 Unauthorized") + mock_post.return_value = mock_response + + with pytest.raises(ApiServerException, match="Failed to get JWT token"): + get_jwt_token("invalid-api-key") + + @patch('os.getenv') + @patch('agentops.get_client') + @patch('agentops.validation.requests.post') + def test_get_jwt_token_from_env(self, mock_post, mock_get_client, mock_getenv): + """Test JWT token retrieval using environment variable.""" + mock_get_client.return_value = None + mock_getenv.return_value = "env-api-key" + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"bearer": "env-token"} + mock_post.return_value = mock_response + + token = get_jwt_token() + assert token == "env-token" + + mock_getenv.assert_called_once_with("AGENTOPS_API_KEY") + + +class TestGetTraceDetails: + """Test trace details retrieval.""" + + @patch('agentops.validation.requests.get') + def test_get_trace_details_success(self, mock_get): + """Test successful trace details retrieval.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "trace_id": "test-trace", + "spans": [{"span_name": "test-span"}] + } + mock_get.return_value = mock_response + + details = get_trace_details("test-trace", "test-token") + assert details["trace_id"] == "test-trace" + assert len(details["spans"]) == 1 + + mock_get.assert_called_once_with( + "https://api.agentops.ai/public/v1/traces/test-trace", + headers={"Authorization": "Bearer test-token"}, + timeout=10 + ) + + @patch('agentops.validation.requests.get') + def test_get_trace_details_failure(self, mock_get): + """Test trace details retrieval failure.""" + mock_response = Mock() + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("404 Not Found") + mock_get.return_value = mock_response + + with pytest.raises(ApiServerException, match="Failed to get trace details"): + get_trace_details("invalid-trace", "test-token") + + +class TestCheckLlmSpans: + """Test LLM span checking.""" + + def test_check_llm_spans_found(self): + """Test when LLM spans are found.""" + spans = [ + {"span_name": "OpenAI Chat Completion"}, + {"span_name": "Some other span"}, + {"span_name": "anthropic.messages.create"} + ] + + has_llm, llm_names = check_llm_spans(spans) + assert has_llm is True + assert len(llm_names) == 2 + assert "OpenAI Chat Completion" in llm_names + assert "anthropic.messages.create" in llm_names + + def test_check_llm_spans_not_found(self): + """Test when no LLM spans are found.""" + spans = [ + {"span_name": "database.query"}, + {"span_name": "http.request"} + ] + + has_llm, llm_names = check_llm_spans(spans) + assert has_llm is False + assert len(llm_names) == 0 + + def test_check_llm_spans_empty(self): + """Test with empty spans list.""" + has_llm, llm_names = check_llm_spans([]) + assert has_llm is False + assert len(llm_names) == 0 + + +class TestValidateTraceSpans: + """Test the main validation function.""" + + @patch('agentops.validation.get_jwt_token') + @patch('agentops.validation.get_trace_details') + @patch('agentops.validation.get_trace_metrics') + def test_validate_trace_spans_success(self, mock_metrics, mock_details, mock_token): + """Test successful validation.""" + mock_token.return_value = "test-token" + mock_details.return_value = { + "spans": [ + {"span_name": "OpenAI Chat Completion"}, + {"span_name": "Other span"} + ] + } + mock_metrics.return_value = { + "total_tokens": 100, + "total_cost": "0.0025" + } + + result = validate_trace_spans(trace_id="test-trace") + + assert result["trace_id"] == "test-trace" + assert result["span_count"] == 2 + assert result["has_llm_spans"] is True + assert "OpenAI Chat Completion" in result["llm_span_names"] + assert result["metrics"]["total_tokens"] == 100 + + @patch('agentops.validation.get_jwt_token') + @patch('agentops.validation.get_trace_details') + def test_validate_trace_spans_no_llm(self, mock_details, mock_token): + """Test validation failure when no LLM spans found.""" + mock_token.return_value = "test-token" + mock_details.return_value = { + "spans": [{"span_name": "database.query"}] + } + + with pytest.raises(ValidationError, match="No LLM spans found"): + validate_trace_spans(trace_id="test-trace", check_llm=True) + + @patch('agentops.validation.get_jwt_token') + @patch('agentops.validation.get_trace_details') + def test_validate_trace_spans_retry(self, mock_details, mock_token): + """Test validation with retries.""" + mock_token.return_value = "test-token" + + # First two calls return empty, third returns spans + mock_details.side_effect = [ + {"spans": []}, + {"spans": []}, + {"spans": [{"span_name": "OpenAI Chat Completion"}]} + ] + + result = validate_trace_spans( + trace_id="test-trace", + max_retries=3, + retry_delay=0.01 + ) + + assert result["span_count"] == 1 + assert mock_details.call_count == 3 + + def test_validate_trace_spans_no_trace_id(self): + """Test validation without trace ID.""" + with pytest.raises(ValueError, match="No trace ID found"): + validate_trace_spans() + + @patch('opentelemetry.trace.get_current_span') + @patch('agentops.validation.get_jwt_token') + @patch('agentops.validation.get_trace_details') + @patch('agentops.validation.get_trace_metrics') + def test_validate_trace_spans_from_current_span(self, mock_metrics, mock_details, mock_token, mock_get_span): + """Test extracting trace ID from current span.""" + # Mock the current span + mock_span_context = Mock() + mock_span_context.trace_id = 12345678901234567890 + + mock_span = Mock() + mock_span.get_span_context.return_value = mock_span_context + + mock_get_span.return_value = mock_span + + mock_token.return_value = "test-token" + mock_details.return_value = { + "spans": [{"span_name": "OpenAI Chat Completion"}] # Use an LLM span name + } + mock_metrics.return_value = { + "total_tokens": 100, + "total_cost": "0.0025" + } + + result = validate_trace_spans() + assert result["trace_id"] == "0000000000000000ab54a98ceb1f0ad2" # hex format of trace ID + + +class TestPrintValidationSummary: + """Test validation summary printing.""" + + def test_print_validation_summary(self, capsys): + """Test printing validation summary.""" + result = { + "span_count": 3, + "has_llm_spans": True, + "llm_span_names": ["OpenAI Chat", "Claude Message"], + "metrics": { + "total_tokens": 150, + "prompt_tokens": 100, + "completion_tokens": 50, + "total_cost": "0.0030" + } + } + + print_validation_summary(result) + + captured = capsys.readouterr() + assert "Found 3 span(s)" in captured.out + assert "OpenAI Chat" in captured.out + assert "Total tokens: 150" in captured.out + assert "Total cost: $0.0030" in captured.out + assert "āœ… Validation successful!" in captured.out From 97f1537cd24c61ff34729300341f2f00c751d85c Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 16:03:36 -0700 Subject: [PATCH 03/25] readme --- examples/README_TESTING.md | 103 +++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 examples/README_TESTING.md diff --git a/examples/README_TESTING.md b/examples/README_TESTING.md new file mode 100644 index 000000000..4d9e8105a --- /dev/null +++ b/examples/README_TESTING.md @@ -0,0 +1,103 @@ +# AgentOps Examples Integration Testing + +This directory contains example scripts demonstrating how to use AgentOps with various LLM providers and frameworks. Each example includes automatic validation to ensure that LLM spans are properly tracked by AgentOps. + +## What's Being Tested + +Each example script now includes automated span validation that: + +1. **Runs the example** - Executes the normal example code +2. **Validates span tracking** - Uses the AgentOps integrated validation to verify that: + - Spans were successfully sent to AgentOps + - LLM calls were properly instrumented and tracked + - Token counts and costs were recorded + +## How It Works + +### 1. Integrated Validation + +AgentOps now includes built-in validation functionality (`agentops.validate_trace_spans`) that: +- Exchanges API keys for JWT tokens using the public API +- Queries the AgentOps API for trace and span data +- Validates that expected spans are present +- Retrieves metrics like token usage and costs + +### 2. Example Structure + +Each example follows this pattern: + +```python +import agentops + +# Initialize AgentOps +agentops.init() + +# Start a trace +tracer = agentops.start_trace("example-name") + +# ... perform operations with LLMs ... + +# End the trace +agentops.end_trace(tracer, end_state="Success") + +# Validate spans were tracked +try: + result = agentops.validate_trace_spans(trace_context=tracer) + agentops.print_validation_summary(result) +except agentops.ValidationError as e: + print(f"āŒ Error validating spans: {e}") + raise +``` + +### 3. CI/CD Integration + +The GitHub Actions workflow (`examples-integration-test.yml`) runs all examples automatically on: +- Push to main/develop branches +- Pull requests +- Manual workflow dispatch + +Each example is run in isolation with proper error handling and reporting. + +## Running Tests Locally + +To run a specific example with validation: + +```bash +cd examples/openai +python openai_example_sync.py +``` + +To run all examples: + +```bash +# From the examples directory +for script in $(find . -name "*.py" -type f | grep -v "__pycache__"); do + echo "Running $script..." + python "$script" +done +``` + +## Adding New Examples + +When adding a new example: + +1. Include the standard validation at the end: + ```python + # Validate spans were tracked + try: + result = agentops.validate_trace_spans(trace_context=tracer) + agentops.print_validation_summary(result) + except agentops.ValidationError as e: + print(f"āŒ Error validating spans: {e}") + raise + ``` + +2. Add the example to the GitHub Actions matrix in `.github/workflows/examples-integration-test.yml` + +3. Ensure the example has proper error handling + +## Requirements + +- Valid `AGENTOPS_API_KEY` environment variable +- API keys for the specific LLM provider being tested +- Python 3.12+ with required dependencies \ No newline at end of file From 9b24f4a74cfb4e08e3aa7c5376850cdad2500931 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 18:00:08 -0700 Subject: [PATCH 04/25] fix langgraph --- examples/langgraph/langgraph_example.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/langgraph/langgraph_example.py b/examples/langgraph/langgraph_example.py index d63397034..0ce06535e 100644 --- a/examples/langgraph/langgraph_example.py +++ b/examples/langgraph/langgraph_example.py @@ -120,10 +120,11 @@ def run_example(): # Let's check programmatically that spans were recorded in AgentOps print("\n" + "="*50) -print("Now let's verify that our LLM calls were tracked properly...") +print("Now let's verify that we have enough spans tracked properly...") try: - agentops.validate_trace_spans(trace_context=None) - print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + # LangGraph doesn't emit LLM spans in the same format, so we just check span count + result = agentops.validate_trace_spans(trace_context=None, check_llm=False, min_spans=5) + print(f"\nāœ… Success! {result['span_count']} spans were properly recorded in AgentOps.") except agentops.ValidationError as e: print(f"\nāŒ Error validating spans: {e}") raise From 55be51c8b468ad4004d23970a6cf7a7ab1e52d7e Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 19:55:46 -0700 Subject: [PATCH 05/25] validator fix --- agentops/validation.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/agentops/validation.py b/agentops/validation.py index ada86bb3a..c53e9908c 100644 --- a/agentops/validation.py +++ b/agentops/validation.py @@ -121,10 +121,25 @@ def check_llm_spans(spans: List[Dict[str, Any]]) -> Tuple[bool, List[str]]: for span in spans: # Check span name for common LLM patterns span_name = span.get("span_name", "").lower() - if any(pattern in span_name for pattern in [ - "openai", "gpt", "claude", "anthropic", "llm", "chat", - "completion", "gemini", "mistral", "cohere", "ai21" - ]): + + # Skip session spans - these are not LLM spans + if ".session" in span_name or span_name.endswith("session"): + continue + + # Check for actual LLM provider patterns - be more specific + # Look for patterns like "openai.chat.completion", "anthropic.messages", etc. + llm_provider_patterns = [ + "openai.chat", "openai.completion", "openai.embedding", + "anthropic.messages", "anthropic.completion", + "google.generativeai", "gemini.generate", + "mistral.chat", "mistral.completion", + "cohere.generate", "cohere.chat", + "ai21.complete", "ai21.generate", + "gpt-", "claude-", # Model names in span names + "litellm.completion", "litellm.acompletion" + ] + + if any(pattern in span_name for pattern in llm_provider_patterns): llm_spans.append(span["span_name"]) return len(llm_spans) > 0, llm_spans From 263ebcf5410a4e228847a892a15161813620383d Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 19:55:54 -0700 Subject: [PATCH 06/25] check for span kinds --- agentops/validation.py | 51 ++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/agentops/validation.py b/agentops/validation.py index c53e9908c..25acc9350 100644 --- a/agentops/validation.py +++ b/agentops/validation.py @@ -119,28 +119,35 @@ def check_llm_spans(spans: List[Dict[str, Any]]) -> Tuple[bool, List[str]]: llm_spans = [] for span in spans: - # Check span name for common LLM patterns - span_name = span.get("span_name", "").lower() - - # Skip session spans - these are not LLM spans - if ".session" in span_name or span_name.endswith("session"): - continue - - # Check for actual LLM provider patterns - be more specific - # Look for patterns like "openai.chat.completion", "anthropic.messages", etc. - llm_provider_patterns = [ - "openai.chat", "openai.completion", "openai.embedding", - "anthropic.messages", "anthropic.completion", - "google.generativeai", "gemini.generate", - "mistral.chat", "mistral.completion", - "cohere.generate", "cohere.chat", - "ai21.complete", "ai21.generate", - "gpt-", "claude-", # Model names in span names - "litellm.completion", "litellm.acompletion" - ] - - if any(pattern in span_name for pattern in llm_provider_patterns): - llm_spans.append(span["span_name"]) + span_name = span.get("span_name", "unnamed_span") + + # Check span attributes for LLM span kind + span_attributes = span.get("span_attributes", {}) + + # Try different possible structures for span kind + span_kind = None + + # Structure 1: span_attributes.agentops.span.kind + if isinstance(span_attributes, dict): + agentops_attrs = span_attributes.get("agentops", {}) + if isinstance(agentops_attrs, dict): + span_info = agentops_attrs.get("span", {}) + if isinstance(span_info, dict): + span_kind = span_info.get("kind", "") + + # Structure 2: Direct in span_attributes + if not span_kind and isinstance(span_attributes, dict): + # Try looking for agentops.span.kind as a flattened key + span_kind = span_attributes.get("agentops.span.kind", "") + + # Structure 3: Look for SpanAttributes.AGENTOPS_SPAN_KIND + if not span_kind and isinstance(span_attributes, dict): + from agentops.semconv import SpanAttributes + span_kind = span_attributes.get(SpanAttributes.AGENTOPS_SPAN_KIND, "") + + # Check if this is an LLM span + if span_kind == "llm": + llm_spans.append(span_name) return len(llm_spans) > 0, llm_spans From c22f3ce699fd2fed605dc836ee7d24b2202b6577 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 20:10:03 -0700 Subject: [PATCH 07/25] model update --- examples/litellm/litellm_example.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/litellm/litellm_example.py b/examples/litellm/litellm_example.py index 437f8b456..67bcde0c0 100644 --- a/examples/litellm/litellm_example.py +++ b/examples/litellm/litellm_example.py @@ -46,7 +46,7 @@ # litellm.completion() # ``` messages = [{"role": "user", "content": "Write a 12 word poem about secret agents."}] -response = litellm.completion(model="gpt-4", messages=messages) # or the model of your choosing +response = litellm.completion(model="gpt-4o-mini", messages=messages) # or the model of your choosing print(response.choices[0].message.content) agentops.end_trace(tracer, end_state="Success") @@ -60,5 +60,3 @@ except agentops.ValidationError as e: print(f"\nāŒ Error validating spans: {e}") raise - - From 31f4735025536042c2fdb04be144d7a31dc9532f Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 20:10:10 -0700 Subject: [PATCH 08/25] model update --- examples/autogen/AgentChat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/autogen/AgentChat.py b/examples/autogen/AgentChat.py index b0ef8d412..8d6ed955e 100644 --- a/examples/autogen/AgentChat.py +++ b/examples/autogen/AgentChat.py @@ -50,7 +50,7 @@ # * Errors # # Simple Chat Example # Define model and API key -model_name = "gpt-4-turbo" # Or "gpt-4o" / "gpt-4o-mini" as per migration guide examples +model_name = "gpt-4o-mini" # Or "gpt-4o" / "gpt-4o-mini" as per migration guide examples api_key = os.getenv("OPENAI_API_KEY") # Create the model client From fa5776f992c3c39ad0ab17f208f0b742da482f66 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 20:12:46 -0700 Subject: [PATCH 09/25] fix validator --- agentops/validation.py | 62 ++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/agentops/validation.py b/agentops/validation.py index 25acc9350..9eca5005e 100644 --- a/agentops/validation.py +++ b/agentops/validation.py @@ -123,30 +123,44 @@ def check_llm_spans(spans: List[Dict[str, Any]]) -> Tuple[bool, List[str]]: # Check span attributes for LLM span kind span_attributes = span.get("span_attributes", {}) - - # Try different possible structures for span kind - span_kind = None - - # Structure 1: span_attributes.agentops.span.kind - if isinstance(span_attributes, dict): - agentops_attrs = span_attributes.get("agentops", {}) - if isinstance(agentops_attrs, dict): - span_info = agentops_attrs.get("span", {}) - if isinstance(span_info, dict): - span_kind = span_info.get("kind", "") - - # Structure 2: Direct in span_attributes - if not span_kind and isinstance(span_attributes, dict): - # Try looking for agentops.span.kind as a flattened key - span_kind = span_attributes.get("agentops.span.kind", "") - - # Structure 3: Look for SpanAttributes.AGENTOPS_SPAN_KIND - if not span_kind and isinstance(span_attributes, dict): - from agentops.semconv import SpanAttributes - span_kind = span_attributes.get(SpanAttributes.AGENTOPS_SPAN_KIND, "") - - # Check if this is an LLM span - if span_kind == "llm": + is_llm_span = False + + # If we have span_attributes, check them + if span_attributes: + # Try different possible structures for span kind + span_kind = None + + # Structure 1: span_attributes.agentops.span.kind + if isinstance(span_attributes, dict): + agentops_attrs = span_attributes.get("agentops", {}) + if isinstance(agentops_attrs, dict): + span_info = agentops_attrs.get("span", {}) + if isinstance(span_info, dict): + span_kind = span_info.get("kind", "") + + # Structure 2: Direct in span_attributes + if not span_kind and isinstance(span_attributes, dict): + # Try looking for agentops.span.kind as a flattened key + span_kind = span_attributes.get("agentops.span.kind", "") + + # Structure 3: Look for SpanAttributes.AGENTOPS_SPAN_KIND + if not span_kind and isinstance(span_attributes, dict): + from agentops.semconv import SpanAttributes + span_kind = span_attributes.get(SpanAttributes.AGENTOPS_SPAN_KIND, "") + + # Check if this is an LLM span by span kind + is_llm_span = span_kind == "llm" + + # Alternative check: Look for gen_ai.prompt or gen_ai.completion attributes + # These are standard semantic conventions for LLM spans + if not is_llm_span and isinstance(span_attributes, dict): + gen_ai_attrs = span_attributes.get("gen_ai", {}) + if isinstance(gen_ai_attrs, dict): + # If we have prompt or completion data, it's an LLM span + if "prompt" in gen_ai_attrs or "completion" in gen_ai_attrs: + is_llm_span = True + + if is_llm_span: llm_spans.append(span_name) return len(llm_spans) > 0, llm_spans From 07ded576f3c0efb7fc7643b881d7a9f31ba4d024 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 20:42:39 -0700 Subject: [PATCH 10/25] update examples --- examples/agno/agno_async_operations.py | 10 ++++++++++ examples/agno/agno_basic_agents.py | 10 ++++++++++ examples/agno/agno_research_team.py | 10 ++++++++++ examples/agno/agno_tool_integrations.py | 10 ++++++++++ examples/agno/agno_workflow_setup.py | 10 ++++++++++ examples/autogen/AgentChat.py | 10 ++++++++++ examples/autogen/MathAgent.py | 10 ++++++++++ examples/context_manager/basic_usage.py | 10 ++++++++++ examples/context_manager/error_handling.py | 10 ++++++++++ examples/context_manager/production_patterns.py | 10 ++++++++++ examples/google_adk/human_approval.py | 10 ++++++++++ examples/google_genai/gemini_example.py | 10 ++++++++++ examples/langchain/langchain_examples.py | 13 +++++++++++++ examples/llamaindex/llamaindex_example.py | 11 +++++++++++ examples/mem0/mem0_memoryclient_example.py | 10 ++++++++++ examples/watsonx/watsonx-streaming.py | 10 ++++++++++ examples/watsonx/watsonx-text-chat.py | 10 ++++++++++ examples/watsonx/watsonx-tokeniation-model.py | 10 ++++++++++ 18 files changed, 184 insertions(+) diff --git a/examples/agno/agno_async_operations.py b/examples/agno/agno_async_operations.py index 5e994e2c1..c88e30481 100644 --- a/examples/agno/agno_async_operations.py +++ b/examples/agno/agno_async_operations.py @@ -75,6 +75,16 @@ async def task3(): print(f"An error occurred: {e}") agentops.end_trace(tracer, end_state="Error") + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "="*50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + if __name__ == "__main__": asyncio.run(demonstrate_async_operations()) diff --git a/examples/agno/agno_basic_agents.py b/examples/agno/agno_basic_agents.py index ef61a642f..eeb430114 100644 --- a/examples/agno/agno_basic_agents.py +++ b/examples/agno/agno_basic_agents.py @@ -88,6 +88,16 @@ def demonstrate_basic_agents(): print(f"An error occurred: {e}") agentops.end_trace(tracer, end_state="Error") + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "="*50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + if __name__ == "__main__": demonstrate_basic_agents() diff --git a/examples/agno/agno_research_team.py b/examples/agno/agno_research_team.py index c9ddc5e93..9b96d5442 100644 --- a/examples/agno/agno_research_team.py +++ b/examples/agno/agno_research_team.py @@ -204,5 +204,15 @@ def demonstrate_research_team(): except Exception: agentops.end_trace(tracer, end_state="Error") + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "="*50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + demonstrate_research_team() diff --git a/examples/agno/agno_tool_integrations.py b/examples/agno/agno_tool_integrations.py index 3a28f0e95..db5c29648 100644 --- a/examples/agno/agno_tool_integrations.py +++ b/examples/agno/agno_tool_integrations.py @@ -144,6 +144,16 @@ def demonstrate_tool_integration(): agentops.end_trace(tracer, end_state="Error") raise + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "="*50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + if __name__ == "__main__": demonstrate_tool_integration() diff --git a/examples/agno/agno_workflow_setup.py b/examples/agno/agno_workflow_setup.py index d3aa0c9ea..5b8c87560 100644 --- a/examples/agno/agno_workflow_setup.py +++ b/examples/agno/agno_workflow_setup.py @@ -113,5 +113,15 @@ def demonstrate_workflows(): except Exception: agentops.end_trace(tracer, end_state="Error") + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "="*50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + asyncio.run(demonstrate_workflows()) diff --git a/examples/autogen/AgentChat.py b/examples/autogen/AgentChat.py index 8d6ed955e..cbe6c4a10 100644 --- a/examples/autogen/AgentChat.py +++ b/examples/autogen/AgentChat.py @@ -92,6 +92,16 @@ async def main(): finally: await model_client.close() + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "="*50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + if __name__ == "__main__": try: diff --git a/examples/autogen/MathAgent.py b/examples/autogen/MathAgent.py index 64348b65d..8672bcc45 100644 --- a/examples/autogen/MathAgent.py +++ b/examples/autogen/MathAgent.py @@ -106,6 +106,16 @@ async def main(): finally: await model_client.close() + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "="*50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + if __name__ == "__main__": try: diff --git a/examples/context_manager/basic_usage.py b/examples/context_manager/basic_usage.py index 122991339..8a50c7677 100644 --- a/examples/context_manager/basic_usage.py +++ b/examples/context_manager/basic_usage.py @@ -138,3 +138,13 @@ def nested_traces_example(): nested_traces_example() print("\nAll examples completed!") + + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "="*50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/context_manager/error_handling.py b/examples/context_manager/error_handling.py index 9f53c3cb7..299a16ee3 100644 --- a/examples/context_manager/error_handling.py +++ b/examples/context_manager/error_handling.py @@ -315,3 +315,13 @@ def exception_chaining_example(): exception_chaining_example() print("\nAll error handling examples completed!") + + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "="*50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/context_manager/production_patterns.py b/examples/context_manager/production_patterns.py index 8dedc50ec..5da0d06e0 100644 --- a/examples/context_manager/production_patterns.py +++ b/examples/context_manager/production_patterns.py @@ -338,3 +338,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): monitoring_pattern() print("\nAll production pattern examples completed!") + + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "="*50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/google_adk/human_approval.py b/examples/google_adk/human_approval.py index 1cbe55e7f..c9be0f75f 100644 --- a/examples/google_adk/human_approval.py +++ b/examples/google_adk/human_approval.py @@ -209,3 +209,13 @@ async def main_notebook(): except Exception as e: print(f"Error: {e}") agentops.end_trace(end_state="Error") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/google_genai/gemini_example.py b/examples/google_genai/gemini_example.py index 6a8450809..401fb112c 100644 --- a/examples/google_genai/gemini_example.py +++ b/examples/google_genai/gemini_example.py @@ -46,3 +46,13 @@ model="gemini-1.5-flash", contents="This is a test sentence to count tokens." ) print(f"Token count: {token_response.total_tokens}") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/langchain/langchain_examples.py b/examples/langchain/langchain_examples.py index 33455bbd5..36de4d6b7 100644 --- a/examples/langchain/langchain_examples.py +++ b/examples/langchain/langchain_examples.py @@ -77,3 +77,16 @@ def find_movie(genre: str) -> str: # ## Check your session # Finally, check your run on [AgentOps](https://app.agentops.ai). You will see a session recorded with the LLM calls and tool usage. + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + import agentops + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except ImportError: + print("\nāŒ Error: agentops library not installed. Please install it to validate spans.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/llamaindex/llamaindex_example.py b/examples/llamaindex/llamaindex_example.py index ed7826a52..f1aa2c750 100644 --- a/examples/llamaindex/llamaindex_example.py +++ b/examples/llamaindex/llamaindex_example.py @@ -64,3 +64,14 @@ print("šŸŽ‰ Example completed successfully!") print("šŸ“Š Check your AgentOps dashboard to see the recorded session with LLM calls and operations.") print("šŸ”— The session link should be printed above by AgentOps.") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + import agentops + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/mem0/mem0_memoryclient_example.py b/examples/mem0/mem0_memoryclient_example.py index cfb060cb2..79f85d962 100644 --- a/examples/mem0/mem0_memoryclient_example.py +++ b/examples/mem0/mem0_memoryclient_example.py @@ -183,3 +183,13 @@ async def demonstrate_async_memory_client(sample_messages, sample_preferences, u # Note: The async version typically completes faster due to concurrent operations demonstrate_sync_memory_client(sample_messages, sample_preferences, user_id) asyncio.run(demonstrate_async_memory_client(sample_messages, sample_preferences, user_id)) + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/watsonx/watsonx-streaming.py b/examples/watsonx/watsonx-streaming.py index a78214f4f..10167d88b 100644 --- a/examples/watsonx/watsonx-streaming.py +++ b/examples/watsonx/watsonx-streaming.py @@ -117,3 +117,13 @@ # Close connections gen_model.close_persistent_connection() chat_model.close_persistent_connection() + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/watsonx/watsonx-text-chat.py b/examples/watsonx/watsonx-text-chat.py index c21391584..dbdd8ba60 100644 --- a/examples/watsonx/watsonx-text-chat.py +++ b/examples/watsonx/watsonx-text-chat.py @@ -76,3 +76,13 @@ # Close connections gen_model.close_persistent_connection() chat_model.close_persistent_connection() + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/watsonx/watsonx-tokeniation-model.py b/examples/watsonx/watsonx-tokeniation-model.py index 90d8210d9..4a8da8580 100644 --- a/examples/watsonx/watsonx-tokeniation-model.py +++ b/examples/watsonx/watsonx-tokeniation-model.py @@ -95,3 +95,13 @@ # Close connections model.close_persistent_connection() llama_model.close_persistent_connection() + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "="*50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise From dd874ffa0201e82e51e84260a9a758f78dd945b6 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 22:06:01 -0700 Subject: [PATCH 11/25] add secrets --- .../workflows/examples-integration-test.yml | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 .github/workflows/examples-integration-test.yml diff --git a/.github/workflows/examples-integration-test.yml b/.github/workflows/examples-integration-test.yml new file mode 100644 index 000000000..575a52c12 --- /dev/null +++ b/.github/workflows/examples-integration-test.yml @@ -0,0 +1,253 @@ +name: Examples Integration Test + +# This workflow runs all example scripts to ensure they work correctly +# and that LLM spans are properly tracked in AgentOps using the +# integrated validation functionality. + +on: + push: + branches: [ main, develop ] + paths: + - 'examples/**/*.py' + - 'agentops/**' + - '.github/workflows/examples-integration-test.yml' + pull_request: + branches: [ main, develop ] + paths: + - 'examples/**/*.py' + - 'agentops/**' + - '.github/workflows/examples-integration-test.yml' + workflow_dispatch: + +env: + PYTHON_VERSION: '3.11' + +jobs: + test-examples: + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + example: + # OpenAI examples + - { path: 'examples/openai/openai_example_sync.py', name: 'OpenAI Sync' } + - { path: 'examples/openai/openai_example_async.py', name: 'OpenAI Async' } + - { path: 'examples/openai/multi_tool_orchestration.py', name: 'OpenAI Multi-Tool' } + - { path: 'examples/openai/web_search.py', name: 'OpenAI Web Search' } + + # Anthropic examples + - { path: 'examples/anthropic/anthropic-example-sync.py', name: 'Anthropic Sync' } + - { path: 'examples/anthropic/anthropic-example-async.py', name: 'Anthropic Async' } + - { path: 'examples/anthropic/agentops-anthropic-understanding-tools.py', name: 'Anthropic Tools' } + + # LangChain examples + - { path: 'examples/langchain/langchain_examples.py', name: 'LangChain' } + + # LiteLLM examples + - { path: 'examples/litellm/litellm_example.py', name: 'LiteLLM' } + + # Google Generative AI examples + - { path: 'examples/google_genai/gemini_example.py', name: 'Google Gemini' } + + # xAI examples + - { path: 'examples/xai/grok_examples.py', name: 'xAI Grok' } + - { path: 'examples/xai/grok_vision_examples.py', name: 'xAI Grok Vision' } + + # CrewAI examples + - { path: 'examples/crewai/job_posting.py', name: 'CrewAI Job Posting' } + - { path: 'examples/crewai/markdown_validator.py', name: 'CrewAI Markdown' } + + # AutoGen examples + - { path: 'examples/autogen/AgentChat.py', name: 'AutoGen Agent Chat' } + - { path: 'examples/autogen/MathAgent.py', name: 'AutoGen Math Agent' } + + # AG2 examples + - { path: 'examples/ag2/async_human_input.py', name: 'AG2 Async Human Input' } + - { path: 'examples/ag2/tools_wikipedia_search.py', name: 'AG2 Wikipedia Search' } + + # Context Manager examples + - { path: 'examples/context_manager/basic_usage.py', name: 'Context Manager Basic' } + - { path: 'examples/context_manager/error_handling.py', name: 'Context Manager Errors' } + - { path: 'examples/context_manager/parallel_traces.py', name: 'Context Manager Parallel' } + - { path: 'examples/context_manager/production_patterns.py', name: 'Context Manager Production' } + + # Agno examples + - { path: 'examples/agno/agno_async_operations.py', name: 'Agno Async Operations' } + - { path: 'examples/agno/agno_basic_agents.py', name: 'Agno Basic Agents' } + - { path: 'examples/agno/agno_research_team.py', name: 'Agno Research Team' } + - { path: 'examples/agno/agno_tool_integrations.py', name: 'Agno Tool Integrations' } + - { path: 'examples/agno/agno_workflow_setup.py', name: 'Agno Workflow Setup' } + + # Google ADK examples + - { path: 'examples/google_adk/human_approval.py', name: 'Google ADK Human Approval' } + + # LlamaIndex examples + - { path: 'examples/llamaindex/llamaindex_example.py', name: 'LlamaIndex' } + + # Mem0 examples + - { path: 'examples/mem0/mem0_memoryclient_example.py', name: 'Mem0 Memory Client' } + + # Watsonx examples + - { path: 'examples/watsonx/watsonx-streaming.py', name: 'Watsonx Streaming' } + - { path: 'examples/watsonx/watsonx-text-chat.py', name: 'Watsonx Text Chat' } + - { path: 'examples/watsonx/watsonx-tokeniation-model.py', name: 'Watsonx Tokenization' } + + # LangGraph examples + - { path: 'examples/langgraph/langgraph_example.py', name: 'LangGraph' } + + # Smolagents examples + - { path: 'examples/smolagents/multi_smolagents_system.py', name: 'Smolagents Multi System' } + - { path: 'examples/smolagents/text_to_sql.py', name: 'Smolagents Text to SQL' } + + # OpenAI Agents examples + - { path: 'examples/openai_agents/agent_guardrails.py', name: 'OpenAI Agents Guardrails' } + - { path: 'examples/openai_agents/agent_patterns.py', name: 'OpenAI Agents Patterns' } + - { path: 'examples/openai_agents/agents_tools.py', name: 'OpenAI Agents Tools' } + - { path: 'examples/openai_agents/customer_service_agent.py', name: 'OpenAI Agents Customer Service' } + + # Add more examples as needed + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install AgentOps + run: | + pip install -e . + + - name: Install example dependencies + run: | + # Install common dependencies + pip install python-dotenv requests + + # Install provider-specific dependencies based on the example + if [[ "${{ matrix.example.path }}" == *"openai"* ]]; then + pip install openai + fi + + if [[ "${{ matrix.example.path }}" == *"anthropic"* ]]; then + pip install anthropic + fi + + if [[ "${{ matrix.example.path }}" == *"langchain"* ]]; then + pip install langchain langchain-openai + fi + + if [[ "${{ matrix.example.path }}" == *"litellm"* ]]; then + pip install litellm + fi + + if [[ "${{ matrix.example.path }}" == *"google_genai"* ]]; then + pip install google-generativeai + fi + + if [[ "${{ matrix.example.path }}" == *"xai"* ]]; then + pip install openai # xAI uses OpenAI client + fi + + if [[ "${{ matrix.example.path }}" == *"crewai"* ]]; then + pip install crewai crewai-tools + fi + + if [[ "${{ matrix.example.path }}" == *"autogen"* ]]; then + pip install pyautogen + fi + + if [[ "${{ matrix.example.path }}" == *"ag2"* ]]; then + pip install ag2 + fi + + if [[ "${{ matrix.example.path }}" == *"agno"* ]]; then + pip install agno + fi + + if [[ "${{ matrix.example.path }}" == *"watsonx"* ]]; then + pip install ibm-watsonx-ai + fi + + if [[ "${{ matrix.example.path }}" == *"mem0"* ]]; then + pip install mem0ai + fi + + if [[ "${{ matrix.example.path }}" == *"smolagents"* ]]; then + pip install smolagents + fi + + if [[ "${{ matrix.example.path }}" == *"llamaindex"* ]]; then + pip install llama-index + fi + + if [[ "${{ matrix.example.path }}" == *"langgraph"* ]]; then + pip install langgraph + fi + + if [[ "${{ matrix.example.path }}" == *"google_adk"* ]]; then + pip install google-genai nest-asyncio + fi + + if [[ "${{ matrix.example.path }}" == *"openai_agents"* ]]; then + pip install openai # OpenAI Agents SDK examples + fi + + - name: Run example - ${{ matrix.example.name }} + env: + AGENTOPS_API_KEY: ${{ secrets.AGENTOPS_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }} + WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} + WATSONX_URL: ${{ secrets.WATSONX_URL }} + MEM0_API_KEY: ${{ secrets.MEM0_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + AI21_API_KEY: ${{ secrets.AI21_API_KEY }} + TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} + EXA_API_KEY: ${{ secrets.EXA_API_KEY }} + LLAMA_API_KEY: ${{ secrets.LLAMA_API_KEY }} + PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} + REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }} + MULTION_API_KEY: ${{ secrets.MULTION_API_KEY }} + PYTHONPATH: ${{ github.workspace }} + run: | + echo "Running ${{ matrix.example.name }}..." + python "${{ matrix.example.path }}" || exit 1 + + - name: Check for errors + if: failure() + run: | + echo "Example ${{ matrix.example.name }} failed!" + echo "Path: ${{ matrix.example.path }}" + + # Show last 50 lines of any log files + if [ -f agentops.log ]; then + echo "=== AgentOps Log ===" + tail -n 50 agentops.log + fi + + summary: + needs: test-examples + runs-on: ubuntu-latest + if: always() + + steps: + - name: Summary + run: | + echo "## Examples Integration Test Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ needs.test-examples.result }}" == "success" ]; then + echo "āœ… All examples passed!" >> $GITHUB_STEP_SUMMARY + else + echo "āŒ Some examples failed. Check the logs above." >> $GITHUB_STEP_SUMMARY + fi \ No newline at end of file From 78868a73d5a21c67f2e8a9eaab64f0ae6b1e00e0 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 22:06:45 -0700 Subject: [PATCH 12/25] workflow secrets --- .github/workflows/examples-integration-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/examples-integration-test.yml b/.github/workflows/examples-integration-test.yml index 575a52c12..88649122c 100644 --- a/.github/workflows/examples-integration-test.yml +++ b/.github/workflows/examples-integration-test.yml @@ -217,7 +217,6 @@ jobs: LLAMA_API_KEY: ${{ secrets.LLAMA_API_KEY }} PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }} - MULTION_API_KEY: ${{ secrets.MULTION_API_KEY }} PYTHONPATH: ${{ github.workspace }} run: | echo "Running ${{ matrix.example.name }}..." From 58f56ff32e1a2ccebe895e8baca0a51181ba0fa8 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 22:27:47 -0700 Subject: [PATCH 13/25] requirements for each dir --- examples/ag2/requirements.txt | 3 +++ examples/agno/requirements.txt | 2 ++ examples/anthropic/requirements.txt | 1 + examples/autogen/requirements.txt | 1 + examples/context_manager/requirements.txt | 2 ++ examples/crewai/requirements.txt | 2 ++ examples/google_adk/requirements.txt | 2 ++ examples/google_genai/requirements.txt | 1 + examples/langchain/requirements.txt | 2 ++ examples/langgraph/requirements.txt | 2 ++ examples/litellm/requirements.txt | 1 + examples/llamaindex/requirements.txt | 3 +++ examples/mem0/requirements.txt | 1 + examples/openai/requirements.txt | 3 +++ examples/openai_agents/requirements.txt | 1 + examples/smolagents/requirements.txt | 3 +++ examples/watsonx/requirements.txt | 1 + examples/xai/requirements.txt | 1 + 18 files changed, 32 insertions(+) create mode 100644 examples/ag2/requirements.txt create mode 100644 examples/agno/requirements.txt create mode 100644 examples/anthropic/requirements.txt create mode 100644 examples/autogen/requirements.txt create mode 100644 examples/context_manager/requirements.txt create mode 100644 examples/crewai/requirements.txt create mode 100644 examples/google_adk/requirements.txt create mode 100644 examples/google_genai/requirements.txt create mode 100644 examples/langchain/requirements.txt create mode 100644 examples/langgraph/requirements.txt create mode 100644 examples/litellm/requirements.txt create mode 100644 examples/llamaindex/requirements.txt create mode 100644 examples/mem0/requirements.txt create mode 100644 examples/openai/requirements.txt create mode 100644 examples/openai_agents/requirements.txt create mode 100644 examples/smolagents/requirements.txt create mode 100644 examples/watsonx/requirements.txt create mode 100644 examples/xai/requirements.txt diff --git a/examples/ag2/requirements.txt b/examples/ag2/requirements.txt new file mode 100644 index 000000000..4359d0e30 --- /dev/null +++ b/examples/ag2/requirements.txt @@ -0,0 +1,3 @@ +ag2 +nest-asyncio +wikipedia-api \ No newline at end of file diff --git a/examples/agno/requirements.txt b/examples/agno/requirements.txt new file mode 100644 index 000000000..96ab8a51e --- /dev/null +++ b/examples/agno/requirements.txt @@ -0,0 +1,2 @@ +agno +aiohttp \ No newline at end of file diff --git a/examples/anthropic/requirements.txt b/examples/anthropic/requirements.txt new file mode 100644 index 000000000..d5d39b6df --- /dev/null +++ b/examples/anthropic/requirements.txt @@ -0,0 +1 @@ +anthropic \ No newline at end of file diff --git a/examples/autogen/requirements.txt b/examples/autogen/requirements.txt new file mode 100644 index 000000000..032705de1 --- /dev/null +++ b/examples/autogen/requirements.txt @@ -0,0 +1 @@ +pyautogen \ No newline at end of file diff --git a/examples/context_manager/requirements.txt b/examples/context_manager/requirements.txt new file mode 100644 index 000000000..434f10ac2 --- /dev/null +++ b/examples/context_manager/requirements.txt @@ -0,0 +1,2 @@ +# Context manager examples only need agentops +# python-dotenv and requests are installed as common deps \ No newline at end of file diff --git a/examples/crewai/requirements.txt b/examples/crewai/requirements.txt new file mode 100644 index 000000000..c74543d99 --- /dev/null +++ b/examples/crewai/requirements.txt @@ -0,0 +1,2 @@ +crewai +crewai-tools \ No newline at end of file diff --git a/examples/google_adk/requirements.txt b/examples/google_adk/requirements.txt new file mode 100644 index 000000000..37c88f3af --- /dev/null +++ b/examples/google_adk/requirements.txt @@ -0,0 +1,2 @@ +google-genai +nest-asyncio \ No newline at end of file diff --git a/examples/google_genai/requirements.txt b/examples/google_genai/requirements.txt new file mode 100644 index 000000000..fb48b1d3c --- /dev/null +++ b/examples/google_genai/requirements.txt @@ -0,0 +1 @@ +google-generativeai \ No newline at end of file diff --git a/examples/langchain/requirements.txt b/examples/langchain/requirements.txt new file mode 100644 index 000000000..5497821c0 --- /dev/null +++ b/examples/langchain/requirements.txt @@ -0,0 +1,2 @@ +langchain +langchain-openai \ No newline at end of file diff --git a/examples/langgraph/requirements.txt b/examples/langgraph/requirements.txt new file mode 100644 index 000000000..0450c9884 --- /dev/null +++ b/examples/langgraph/requirements.txt @@ -0,0 +1,2 @@ +langgraph +langchain-openai \ No newline at end of file diff --git a/examples/litellm/requirements.txt b/examples/litellm/requirements.txt new file mode 100644 index 000000000..e215fd144 --- /dev/null +++ b/examples/litellm/requirements.txt @@ -0,0 +1 @@ +litellm \ No newline at end of file diff --git a/examples/llamaindex/requirements.txt b/examples/llamaindex/requirements.txt new file mode 100644 index 000000000..bafc641c4 --- /dev/null +++ b/examples/llamaindex/requirements.txt @@ -0,0 +1,3 @@ +llama-index +llama-index-llms-openai +llama-index-instrumentation-agentops \ No newline at end of file diff --git a/examples/mem0/requirements.txt b/examples/mem0/requirements.txt new file mode 100644 index 000000000..2713cbc83 --- /dev/null +++ b/examples/mem0/requirements.txt @@ -0,0 +1 @@ +mem0ai \ No newline at end of file diff --git a/examples/openai/requirements.txt b/examples/openai/requirements.txt new file mode 100644 index 000000000..f066e4c31 --- /dev/null +++ b/examples/openai/requirements.txt @@ -0,0 +1,3 @@ +openai +pandas # for multi_tool_orchestration.py +tavily-python # for web_search.py \ No newline at end of file diff --git a/examples/openai_agents/requirements.txt b/examples/openai_agents/requirements.txt new file mode 100644 index 000000000..a82e2f0d8 --- /dev/null +++ b/examples/openai_agents/requirements.txt @@ -0,0 +1 @@ +openai \ No newline at end of file diff --git a/examples/smolagents/requirements.txt b/examples/smolagents/requirements.txt new file mode 100644 index 000000000..e84894a58 --- /dev/null +++ b/examples/smolagents/requirements.txt @@ -0,0 +1,3 @@ +smolagents +pandas +duckduckgo-search \ No newline at end of file diff --git a/examples/watsonx/requirements.txt b/examples/watsonx/requirements.txt new file mode 100644 index 000000000..b3e3ac418 --- /dev/null +++ b/examples/watsonx/requirements.txt @@ -0,0 +1 @@ +ibm-watsonx-ai \ No newline at end of file diff --git a/examples/xai/requirements.txt b/examples/xai/requirements.txt new file mode 100644 index 000000000..85a3cb06b --- /dev/null +++ b/examples/xai/requirements.txt @@ -0,0 +1 @@ +openai # xAI uses OpenAI client \ No newline at end of file From 229feab1478ce1d2cb9d4b29e8a3037d5b225e57 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 22:27:53 -0700 Subject: [PATCH 14/25] reqs --- .../workflows/examples-integration-test.yml | 74 ++----------------- 1 file changed, 7 insertions(+), 67 deletions(-) diff --git a/.github/workflows/examples-integration-test.yml b/.github/workflows/examples-integration-test.yml index 88649122c..ccddbd194 100644 --- a/.github/workflows/examples-integration-test.yml +++ b/.github/workflows/examples-integration-test.yml @@ -126,73 +126,13 @@ jobs: # Install common dependencies pip install python-dotenv requests - # Install provider-specific dependencies based on the example - if [[ "${{ matrix.example.path }}" == *"openai"* ]]; then - pip install openai - fi - - if [[ "${{ matrix.example.path }}" == *"anthropic"* ]]; then - pip install anthropic - fi - - if [[ "${{ matrix.example.path }}" == *"langchain"* ]]; then - pip install langchain langchain-openai - fi - - if [[ "${{ matrix.example.path }}" == *"litellm"* ]]; then - pip install litellm - fi - - if [[ "${{ matrix.example.path }}" == *"google_genai"* ]]; then - pip install google-generativeai - fi - - if [[ "${{ matrix.example.path }}" == *"xai"* ]]; then - pip install openai # xAI uses OpenAI client - fi - - if [[ "${{ matrix.example.path }}" == *"crewai"* ]]; then - pip install crewai crewai-tools - fi - - if [[ "${{ matrix.example.path }}" == *"autogen"* ]]; then - pip install pyautogen - fi - - if [[ "${{ matrix.example.path }}" == *"ag2"* ]]; then - pip install ag2 - fi - - if [[ "${{ matrix.example.path }}" == *"agno"* ]]; then - pip install agno - fi - - if [[ "${{ matrix.example.path }}" == *"watsonx"* ]]; then - pip install ibm-watsonx-ai - fi - - if [[ "${{ matrix.example.path }}" == *"mem0"* ]]; then - pip install mem0ai - fi - - if [[ "${{ matrix.example.path }}" == *"smolagents"* ]]; then - pip install smolagents - fi - - if [[ "${{ matrix.example.path }}" == *"llamaindex"* ]]; then - pip install llama-index - fi - - if [[ "${{ matrix.example.path }}" == *"langgraph"* ]]; then - pip install langgraph - fi - - if [[ "${{ matrix.example.path }}" == *"google_adk"* ]]; then - pip install google-genai nest-asyncio - fi - - if [[ "${{ matrix.example.path }}" == *"openai_agents"* ]]; then - pip install openai # OpenAI Agents SDK examples + # Install from requirements.txt in the example's directory + example_dir=$(dirname "${{ matrix.example.path }}") + if [ -f "$example_dir/requirements.txt" ]; then + echo "Installing dependencies from $example_dir/requirements.txt" + pip install -r "$example_dir/requirements.txt" + else + echo "No requirements.txt found in $example_dir" fi - name: Run example - ${{ matrix.example.name }} From 9b7c9e426861fa22d2c353cd7782c6a0a2cb661c Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 22:34:38 -0700 Subject: [PATCH 15/25] fix tests --- tests/unit/test_validation.py | 47 ++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py index f9a97308d..c7c12fdbd 100644 --- a/tests/unit/test_validation.py +++ b/tests/unit/test_validation.py @@ -108,9 +108,23 @@ class TestCheckLlmSpans: def test_check_llm_spans_found(self): """Test when LLM spans are found.""" spans = [ - {"span_name": "OpenAI Chat Completion"}, + { + "span_name": "OpenAI Chat Completion", + "span_attributes": { + "agentops.span.kind": "llm" + } + }, {"span_name": "Some other span"}, - {"span_name": "anthropic.messages.create"} + { + "span_name": "anthropic.messages.create", + "span_attributes": { + "agentops": { + "span": { + "kind": "llm" + } + } + } + } ] has_llm, llm_names = check_llm_spans(spans) @@ -148,7 +162,12 @@ def test_validate_trace_spans_success(self, mock_metrics, mock_details, mock_tok mock_token.return_value = "test-token" mock_details.return_value = { "spans": [ - {"span_name": "OpenAI Chat Completion"}, + { + "span_name": "OpenAI Chat Completion", + "span_attributes": { + "agentops.span.kind": "llm" + } + }, {"span_name": "Other span"} ] } @@ -187,7 +206,14 @@ def test_validate_trace_spans_retry(self, mock_details, mock_token): mock_details.side_effect = [ {"spans": []}, {"spans": []}, - {"spans": [{"span_name": "OpenAI Chat Completion"}]} + { + "spans": [{ + "span_name": "OpenAI Chat Completion", + "span_attributes": { + "agentops.span.kind": "llm" + } + }] + } ] result = validate_trace_spans( @@ -199,8 +225,12 @@ def test_validate_trace_spans_retry(self, mock_details, mock_token): assert result["span_count"] == 1 assert mock_details.call_count == 3 - def test_validate_trace_spans_no_trace_id(self): + @patch('opentelemetry.trace.get_current_span') + def test_validate_trace_spans_no_trace_id(self, mock_get_current_span): """Test validation without trace ID.""" + # Mock get_current_span to return None + mock_get_current_span.return_value = None + with pytest.raises(ValueError, match="No trace ID found"): validate_trace_spans() @@ -221,7 +251,12 @@ def test_validate_trace_spans_from_current_span(self, mock_metrics, mock_details mock_token.return_value = "test-token" mock_details.return_value = { - "spans": [{"span_name": "OpenAI Chat Completion"}] # Use an LLM span name + "spans": [{ + "span_name": "OpenAI Chat Completion", + "span_attributes": { + "agentops.span.kind": "llm" + } + }] } mock_metrics.return_value = { "total_tokens": 100, From 7e6451f747627f21ad33453e0f91809526ce1f13 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 22:49:42 -0700 Subject: [PATCH 16/25] fix tests? --- agentops/validation.py | 14 +++++++ tests/unit/test_validation.py | 70 +++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/agentops/validation.py b/agentops/validation.py index 9eca5005e..dd1942881 100644 --- a/agentops/validation.py +++ b/agentops/validation.py @@ -160,6 +160,20 @@ def check_llm_spans(spans: List[Dict[str, Any]]) -> Tuple[bool, List[str]]: if "prompt" in gen_ai_attrs or "completion" in gen_ai_attrs: is_llm_span = True + # Check for LLM_REQUEST_TYPE attribute (used by provider instrumentations) + if not is_llm_span and isinstance(span_attributes, dict): + from agentops.semconv import SpanAttributes, LLMRequestTypeValues + + # Check for LLM request type + llm_request_type = span_attributes.get(SpanAttributes.LLM_REQUEST_TYPE, "") + + # Check if it's a chat or completion request (the main LLM types) + if llm_request_type in [ + LLMRequestTypeValues.CHAT.value, + LLMRequestTypeValues.COMPLETION.value + ]: + is_llm_span = True + if is_llm_span: llm_spans.append(span_name) diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py index c7c12fdbd..095779376 100644 --- a/tests/unit/test_validation.py +++ b/tests/unit/test_validation.py @@ -150,6 +150,76 @@ def test_check_llm_spans_empty(self): assert has_llm is False assert len(llm_names) == 0 + def test_check_llm_spans_with_request_type(self): + """Test when LLM spans are identified by LLM_REQUEST_TYPE attribute.""" + from agentops.semconv import SpanAttributes, LLMRequestTypeValues + + spans = [ + { + "span_name": "openai.chat.completion", + "span_attributes": { + SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value + } + }, + { + "span_name": "anthropic.messages.create", + "span_attributes": { + SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value + } + }, + { + "span_name": "llm.completion", + "span_attributes": { + SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.COMPLETION.value + } + }, + { + "span_name": "embedding.create", + "span_attributes": { + SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.EMBEDDING.value + } + }, + {"span_name": "database.query"} + ] + + has_llm, llm_names = check_llm_spans(spans) + assert has_llm is True + assert len(llm_names) == 3 # Only chat and completion types count as LLM + assert "openai.chat.completion" in llm_names + assert "anthropic.messages.create" in llm_names + assert "llm.completion" in llm_names + assert "embedding.create" not in llm_names # Embeddings are not LLM spans + + def test_check_llm_spans_real_world(self): + """Test with real-world span structures from OpenAI and Anthropic.""" + from agentops.semconv import SpanAttributes, LLMRequestTypeValues + + # This simulates what we actually get from the OpenAI and Anthropic instrumentations + spans = [ + { + "span_name": "openai.chat.completion", + "span_attributes": { + SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value, + SpanAttributes.LLM_SYSTEM: "OpenAI", + SpanAttributes.LLM_REQUEST_MODEL: "gpt-4" + } + }, + { + "span_name": "anthropic.messages.create", + "span_attributes": { + SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value, + SpanAttributes.LLM_SYSTEM: "Anthropic", + SpanAttributes.LLM_REQUEST_MODEL: "claude-3-opus-20240229" + } + } + ] + + has_llm, llm_names = check_llm_spans(spans) + assert has_llm is True + assert len(llm_names) == 2 + assert "openai.chat.completion" in llm_names + assert "anthropic.messages.create" in llm_names + class TestValidateTraceSpans: """Test the main validation function.""" From 3fc5a1524d61261109df46f78c4eb45b30e2ac3a Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 22:52:58 -0700 Subject: [PATCH 17/25] add reqs --- examples/google_adk/requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/google_adk/requirements.txt b/examples/google_adk/requirements.txt index 37c88f3af..476e7f452 100644 --- a/examples/google_adk/requirements.txt +++ b/examples/google_adk/requirements.txt @@ -1,2 +1,5 @@ +google-adk google-genai -nest-asyncio \ No newline at end of file +nest-asyncio +agentops +python-dotenv \ No newline at end of file From 35ec7efe66e2b344a5b1c1ec6a1c00b7daa57df2 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 23:09:21 -0700 Subject: [PATCH 18/25] shorten test --- examples/openai/openai_example_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/openai/openai_example_sync.py b/examples/openai/openai_example_sync.py index 13501b59b..c5f9a2f5d 100644 --- a/examples/openai/openai_example_sync.py +++ b/examples/openai/openai_example_sync.py @@ -35,7 +35,7 @@ You are given a prompt and you need to generate a story based on the prompt. """ -user_prompt = "Write a story about a cyber-warrior trapped in the imperial time period." +user_prompt = "Write a very short story about a cyber-warrior trapped in the imperial time period." messages = [ {"role": "system", "content": system_prompt}, From ad4c49ba036a055a38393a3d7ab8b1428448423e Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 23:09:36 -0700 Subject: [PATCH 19/25] shorten test --- examples/openai/openai_example_async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/openai/openai_example_async.py b/examples/openai/openai_example_async.py index 7e18ed1a3..fb4e6e951 100644 --- a/examples/openai/openai_example_async.py +++ b/examples/openai/openai_example_async.py @@ -36,7 +36,7 @@ """ user_prompt = [ - {"type": "text", "text": "Write a mystery thriller story based on your understanding of the provided image."}, + {"type": "text", "text": "Write a very short mystery thriller story based on your understanding of the provided image."}, { "type": "image_url", "image_url": {"url": "https://www.cosy.sbg.ac.at/~pmeerw/Watermarking/lena_color.gif"}, From f49e05c1c2221b41ce0d840b33f1b0ae916b47da Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 23:09:55 -0700 Subject: [PATCH 20/25] tests --- agentops/validation.py | 56 ++++++++------ tests/unit/test_validation.py | 135 ++++++++++++++++++++++++++++++++-- 2 files changed, 165 insertions(+), 26 deletions(-) diff --git a/agentops/validation.py b/agentops/validation.py index dd1942881..a9926f2c8 100644 --- a/agentops/validation.py +++ b/agentops/validation.py @@ -164,8 +164,12 @@ def check_llm_spans(spans: List[Dict[str, Any]]) -> Tuple[bool, List[str]]: if not is_llm_span and isinstance(span_attributes, dict): from agentops.semconv import SpanAttributes, LLMRequestTypeValues - # Check for LLM request type + # Check for LLM request type - try both gen_ai.* and llm.* prefixes + # The instrumentation sets gen_ai.* but the API might return llm.* llm_request_type = span_attributes.get(SpanAttributes.LLM_REQUEST_TYPE, "") + if not llm_request_type: + # Try the llm.* prefix version + llm_request_type = span_attributes.get("llm.request.type", "") # Check if it's a chat or completion request (the main LLM types) if llm_request_type in [ @@ -261,18 +265,7 @@ def validate_trace_spans( "metrics": None } - # Check for LLM spans if requested - if check_llm: - has_llm_spans, llm_span_names = check_llm_spans(spans) - result["has_llm_spans"] = has_llm_spans - result["llm_span_names"] = llm_span_names - - if has_llm_spans: - logger.info(f"Found LLM spans: {', '.join(llm_span_names)}") - elif check_llm: - logger.warning("No LLM spans found in trace") - - # Get metrics + # Get metrics first - if we have token usage, we definitely have LLM spans try: metrics = get_trace_metrics(trace_id, jwt_token) result["metrics"] = metrics @@ -280,14 +273,31 @@ def validate_trace_spans( if metrics: logger.info(f"Trace metrics - Total tokens: {metrics.get('total_tokens', 0)}, " f"Cost: ${metrics.get('total_cost', '0.0000')}") + + # If we have token usage > 0, we definitely have LLM activity + if metrics.get('total_tokens', 0) > 0: + result["has_llm_spans"] = True + logger.info("LLM activity confirmed via token usage metrics") except Exception as e: logger.warning(f"Could not retrieve metrics: {e}") - # Validate based on requirements - if check_llm and not has_llm_spans: + # Check for LLM spans if requested and not already confirmed via metrics + if check_llm and not result["has_llm_spans"]: + has_llm_spans, llm_span_names = check_llm_spans(spans) + result["has_llm_spans"] = has_llm_spans + result["llm_span_names"] = llm_span_names + + if has_llm_spans: + logger.info(f"Found LLM spans: {', '.join(llm_span_names)}") + else: + logger.warning("No LLM spans found via attribute inspection") + + # Final validation + if check_llm and not result["has_llm_spans"]: raise ValidationError( - f"No LLM spans found in trace {trace_id}. " - f"Found spans: {[s.get('span_name', 'unnamed') for s in spans]}" + f"No LLM activity detected in trace {trace_id}. " + f"Found spans: {[s.get('span_name', 'unnamed') for s in spans]}, " + f"Token usage: {result.get('metrics', {}).get('total_tokens', 0)}" ) return result @@ -305,7 +315,7 @@ def validate_trace_spans( raise ValidationError( f"Validation failed for trace {trace_id} after {max_retries} attempts. " f"Expected at least {min_spans} spans" + - (", including LLM spans" if check_llm else "") + + (", including LLM activity" if check_llm else "") + ". Please check that tracking is properly configured." ) @@ -324,9 +334,13 @@ def print_validation_summary(result: Dict[str, Any]) -> None: print(f"āœ… Found {result['span_count']} span(s) in trace") if result.get('has_llm_spans'): - print(f"āœ… Found LLM spans: {', '.join(result['llm_span_names'])}") - elif result.get('llm_span_names') is not None: - print("āš ļø No LLM spans detected") + if result.get('llm_span_names'): + print(f"āœ… Found LLM spans: {', '.join(result['llm_span_names'])}") + else: + # LLM activity confirmed via metrics + print("āœ… LLM activity confirmed via token usage metrics") + elif 'has_llm_spans' in result: + print("āš ļø No LLM activity detected") if result.get('metrics'): metrics = result['metrics'] diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py index 095779376..d24811820 100644 --- a/tests/unit/test_validation.py +++ b/tests/unit/test_validation.py @@ -251,24 +251,59 @@ def test_validate_trace_spans_success(self, mock_metrics, mock_details, mock_tok assert result["trace_id"] == "test-trace" assert result["span_count"] == 2 assert result["has_llm_spans"] is True - assert "OpenAI Chat Completion" in result["llm_span_names"] + # LLM activity can be confirmed via metrics or span inspection assert result["metrics"]["total_tokens"] == 100 @patch('agentops.validation.get_jwt_token') @patch('agentops.validation.get_trace_details') - def test_validate_trace_spans_no_llm(self, mock_details, mock_token): - """Test validation failure when no LLM spans found.""" + @patch('agentops.validation.get_trace_metrics') + def test_validate_trace_spans_success_via_metrics(self, mock_metrics, mock_details, mock_token): + """Test successful validation when LLM activity is confirmed via metrics.""" + mock_token.return_value = "test-token" + mock_details.return_value = { + "spans": [ + { + "span_name": "openai.chat.completion", + "span_attributes": {} # No specific LLM attributes + }, + {"span_name": "Other span"} + ] + } + # But we have token usage, proving LLM activity + mock_metrics.return_value = { + "total_tokens": 1066, + "total_cost": "0.0006077" + } + + result = validate_trace_spans(trace_id="test-trace") + + assert result["trace_id"] == "test-trace" + assert result["span_count"] == 2 + assert result["has_llm_spans"] is True # Confirmed via metrics + assert result["metrics"]["total_tokens"] == 1066 + + @patch('agentops.validation.get_jwt_token') + @patch('agentops.validation.get_trace_details') + @patch('agentops.validation.get_trace_metrics') + def test_validate_trace_spans_no_llm(self, mock_metrics, mock_details, mock_token): + """Test validation failure when no LLM spans found and no token usage.""" mock_token.return_value = "test-token" mock_details.return_value = { "spans": [{"span_name": "database.query"}] } + # No token usage either + mock_metrics.return_value = { + "total_tokens": 0, + "total_cost": "0.0000" + } - with pytest.raises(ValidationError, match="No LLM spans found"): + with pytest.raises(ValidationError, match="No LLM activity detected"): validate_trace_spans(trace_id="test-trace", check_llm=True) @patch('agentops.validation.get_jwt_token') @patch('agentops.validation.get_trace_details') - def test_validate_trace_spans_retry(self, mock_details, mock_token): + @patch('agentops.validation.get_trace_metrics') + def test_validate_trace_spans_retry(self, mock_metrics, mock_details, mock_token): """Test validation with retries.""" mock_token.return_value = "test-token" @@ -286,6 +321,12 @@ def test_validate_trace_spans_retry(self, mock_details, mock_token): } ] + # Mock metrics for the successful attempt + mock_metrics.return_value = { + "total_tokens": 100, + "total_cost": "0.0025" + } + result = validate_trace_spans( trace_id="test-trace", max_retries=3, @@ -362,3 +403,87 @@ def test_print_validation_summary(self, capsys): assert "Total tokens: 150" in captured.out assert "Total cost: $0.0030" in captured.out assert "āœ… Validation successful!" in captured.out + + def test_print_validation_summary_metrics_only(self, capsys): + """Test printing validation summary when LLM activity confirmed via metrics only.""" + result = { + "span_count": 2, + "has_llm_spans": True, + "llm_span_names": [], # No specific LLM span names found + "metrics": { + "total_tokens": 1066, + "prompt_tokens": 800, + "completion_tokens": 266, + "total_cost": "0.0006077" + } + } + + print_validation_summary(result) + + captured = capsys.readouterr() + assert "Found 2 span(s)" in captured.out + assert "LLM activity confirmed via token usage metrics" in captured.out + assert "Total tokens: 1066" in captured.out + assert "Total cost: $0.0006077" in captured.out + assert "āœ… Validation successful!" in captured.out + + def test_print_validation_summary_llm_prefix(self, capsys): + """Test with spans using llm.* prefix (as returned by API).""" + result = { + "span_count": 1, + "has_llm_spans": True, + "llm_span_names": ["openai.chat.completion"], + "metrics": { + "total_tokens": 150, + "prompt_tokens": 100, + "completion_tokens": 50, + "total_cost": "0.0030" + } + } + + print_validation_summary(result) + + captured = capsys.readouterr() + assert "Found 1 span(s)" in captured.out + assert "openai.chat.completion" in captured.out + assert "āœ… Validation successful!" in captured.out + + +class TestCheckLlmSpansWithLlmPrefix: + """Test LLM span checking with llm.* prefix attributes.""" + + def test_check_llm_spans_with_llm_prefix(self): + """Test when spans use llm.request.type instead of gen_ai.request.type.""" + spans = [ + { + "span_name": "openai.chat.completion", + "span_attributes": { + "llm.request.type": "chat", + "llm.system": "OpenAI", + "llm.request.model": "gpt-4", + "llm.usage.total_tokens": 150 + } + }, + { + "span_name": "anthropic.messages.create", + "span_attributes": { + "llm.request.type": "chat", + "llm.system": "Anthropic", + "llm.request.model": "claude-3-opus", + "llm.usage.total_tokens": 300 + } + }, + { + "span_name": "embedding.create", + "span_attributes": { + "llm.request.type": "embedding" + } + }, + {"span_name": "database.query"} + ] + + has_llm, llm_names = check_llm_spans(spans) + assert has_llm is True + assert len(llm_names) == 2 # Only chat types + assert "openai.chat.completion" in llm_names + assert "anthropic.messages.create" in llm_names From 7fae675ed29ebc95a564df04c8fef96f8f1db326 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 23:24:05 -0700 Subject: [PATCH 21/25] add requirements and fix broken notebooks --- examples/autogen/AgentChat.py | 7 ------- examples/autogen/MathAgent.py | 6 ------ examples/crewai/job_posting.py | 10 ++++------ examples/crewai/requirements.txt | 3 ++- examples/google_genai/requirements.txt | 3 ++- examples/openai_agents/requirements.txt | 3 ++- examples/smolagents/requirements.txt | 3 ++- 7 files changed, 12 insertions(+), 23 deletions(-) diff --git a/examples/autogen/AgentChat.py b/examples/autogen/AgentChat.py index cbe6c4a10..783f6e5a8 100644 --- a/examples/autogen/AgentChat.py +++ b/examples/autogen/AgentChat.py @@ -9,9 +9,6 @@ # Then import them import os from dotenv import load_dotenv -from IPython.core.error import ( - StdinNotImplementedError, -) import asyncio import agentops @@ -82,10 +79,6 @@ async def main(): await Console().run(stream) agentops.end_trace(tracer, end_state="Success") - except StdinNotImplementedError: - print("StdinNotImplementedError: This typically happens in non-interactive environments.") - print("Skipping interactive part of chat for automation.") - agentops.end_trace(tracer, end_state="Indeterminate") except Exception as e: print(f"An error occurred: {e}") agentops.end_trace(tracer, end_state="Error") diff --git a/examples/autogen/MathAgent.py b/examples/autogen/MathAgent.py index 8672bcc45..14184f426 100644 --- a/examples/autogen/MathAgent.py +++ b/examples/autogen/MathAgent.py @@ -11,9 +11,6 @@ import asyncio import os from dotenv import load_dotenv -from IPython.core.error import ( - StdinNotImplementedError, -) import agentops @@ -97,9 +94,6 @@ async def main(): agentops.end_trace(tracer, end_state="Success") - except StdinNotImplementedError: - print("StdinNotImplementedError: This typically happens in non-interactive environments.") - agentops.end_trace(tracer, end_state="Indeterminate") except Exception as e: print(f"An error occurred: {e}") agentops.end_trace(tracer, end_state="Error") diff --git a/examples/crewai/job_posting.py b/examples/crewai/job_posting.py index 99853c494..d3752fb80 100644 --- a/examples/crewai/job_posting.py +++ b/examples/crewai/job_posting.py @@ -133,10 +133,10 @@ def industry_analysis_task(self, agent, company_domain, company_description): tracer = agentops.start_trace(trace_name="CrewAI Job Posting", tags=["crew-job-posting-example", "agentops-example"]) tasks = Tasks() agents = Agents() -company_description = input("What is the company description?\n") -company_domain = input("What is the company domain?\n") -hiring_needs = input("What are the hiring needs?\n") -specific_benefits = input("What are specific_benefits you offer?\n") +company_description = "We are a software company that builds AI-powered tools for businesses." +company_domain = "https://www.agentops.ai" +hiring_needs = "We are looking for a software engineer with 3 years of experience in Python and Django." +specific_benefits = "We offer a competitive salary, health insurance, and a 401k plan." # Create Agents researcher_agent = agents.research_agent() @@ -182,5 +182,3 @@ def industry_analysis_task(self, agent, company_domain, company_description): except agentops.ValidationError as e: print(f"\nāŒ Error validating spans: {e}") raise - - diff --git a/examples/crewai/requirements.txt b/examples/crewai/requirements.txt index c74543d99..f2e17dad0 100644 --- a/examples/crewai/requirements.txt +++ b/examples/crewai/requirements.txt @@ -1,2 +1,3 @@ crewai -crewai-tools \ No newline at end of file +crewai-tools +pymarkdown \ No newline at end of file diff --git a/examples/google_genai/requirements.txt b/examples/google_genai/requirements.txt index fb48b1d3c..c969438c1 100644 --- a/examples/google_genai/requirements.txt +++ b/examples/google_genai/requirements.txt @@ -1 +1,2 @@ -google-generativeai \ No newline at end of file +google-generativeai +google-genai \ No newline at end of file diff --git a/examples/openai_agents/requirements.txt b/examples/openai_agents/requirements.txt index a82e2f0d8..11a0a416d 100644 --- a/examples/openai_agents/requirements.txt +++ b/examples/openai_agents/requirements.txt @@ -1 +1,2 @@ -openai \ No newline at end of file +openai +openai-agents \ No newline at end of file diff --git a/examples/smolagents/requirements.txt b/examples/smolagents/requirements.txt index e84894a58..9a3d792c5 100644 --- a/examples/smolagents/requirements.txt +++ b/examples/smolagents/requirements.txt @@ -1,3 +1,4 @@ smolagents pandas -duckduckgo-search \ No newline at end of file +duckduckgo-search +sqlalchemy \ No newline at end of file From 50af52429062e1a3999ffce7b026f9136b049188 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 23:36:47 -0700 Subject: [PATCH 22/25] add tags --- examples/context_manager/basic_usage.py | 8 ++++---- examples/context_manager/error_handling.py | 2 +- examples/crewai/job_posting.py | 2 +- examples/google_adk/human_approval.py | 3 ++- examples/langgraph/langgraph_example.py | 2 +- examples/llamaindex/llamaindex_example.py | 2 +- examples/mem0/mem0_memory_example.py | 4 ++-- examples/openai/multi_tool_orchestration.py | 3 +-- examples/openai/openai_example_async.py | 2 +- examples/openai/openai_example_sync.py | 2 +- examples/openai/web_search.py | 3 +-- examples/openai_agents/agent_patterns.py | 3 +-- examples/openai_agents/agents_tools.py | 2 +- examples/xai/grok_examples.py | 3 +-- examples/xai/grok_vision_examples.py | 3 +-- 15 files changed, 20 insertions(+), 24 deletions(-) diff --git a/examples/context_manager/basic_usage.py b/examples/context_manager/basic_usage.py index 8a50c7677..0dce207e3 100644 --- a/examples/context_manager/basic_usage.py +++ b/examples/context_manager/basic_usage.py @@ -40,7 +40,7 @@ def basic_context_manager_example(): print("Basic Context Manager Example") # Initialize AgentOps - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "agentops-example"]) # Use native TraceContext context manager with agentops.start_trace("basic_example", tags=["basic", "demo"]): @@ -58,7 +58,7 @@ def multiple_parallel_traces(): """Example showing multiple parallel traces.""" print("\nMultiple Parallel Traces") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "agentops-example"]) # First trace with agentops.start_trace("task_1", tags=["parallel", "task-1"]): @@ -81,7 +81,7 @@ def error_handling_example(): """Example showing error handling with context manager.""" print("\nError Handling Example") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "agentops-example"]) try: with agentops.start_trace("error_example", tags=["error-handling"]): @@ -103,7 +103,7 @@ def nested_traces_example(): """Example showing nested traces (which are parallel, not parent-child).""" print("\nNested Traces Example") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "agentops-example"]) # Outer trace with agentops.start_trace("main_workflow", tags=["workflow", "main"]): diff --git a/examples/context_manager/error_handling.py b/examples/context_manager/error_handling.py index 299a16ee3..b8306aa96 100644 --- a/examples/context_manager/error_handling.py +++ b/examples/context_manager/error_handling.py @@ -66,7 +66,7 @@ def basic_exception_handling(): """Basic example of exception handling with context managers.""" print("Basic Exception Handling") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "error-handling", "agentops-example"]) error_types = ["value_error", "type_error", "runtime_error", "success"] diff --git a/examples/crewai/job_posting.py b/examples/crewai/job_posting.py index d3752fb80..6b1935008 100644 --- a/examples/crewai/job_posting.py +++ b/examples/crewai/job_posting.py @@ -20,7 +20,7 @@ os.environ["SERPER_API_KEY"] = os.getenv("SERPER_API_KEY", "your_serper_api_key_here") # Initialize AgentOps client -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, tags=["crewai", "job-posting", "agentops-example"]) web_search_tool = WebsiteSearchTool() serper_dev_tool = SerperDevTool() diff --git a/examples/google_adk/human_approval.py b/examples/google_adk/human_approval.py index c9be0f75f..6ff142f02 100644 --- a/examples/google_adk/human_approval.py +++ b/examples/google_adk/human_approval.py @@ -31,7 +31,8 @@ AGENTOPS_API_KEY = os.getenv("AGENTOPS_API_KEY") or "your_agentops_api_key_here" # Initialize AgentOps - Just 2 lines! -agentops.init(AGENTOPS_API_KEY, trace_name="adk-human-approval-notebook", auto_start_session=False) +agentops.init(AGENTOPS_API_KEY, trace_name="adk-human-approval-notebook", + auto_start_session=False, tags=["google-adk", "human-approval", "agentops-example"]) # Define some constants for our application. APP_NAME = "human_approval_app_notebook" diff --git a/examples/langgraph/langgraph_example.py b/examples/langgraph/langgraph_example.py index 0ce06535e..a84c080c5 100644 --- a/examples/langgraph/langgraph_example.py +++ b/examples/langgraph/langgraph_example.py @@ -10,7 +10,7 @@ load_dotenv() -agentops.init(os.getenv("AGENTOPS_API_KEY")) +agentops.init(os.getenv("AGENTOPS_API_KEY"), tags=["langgraph", "tool-usage", "agentops-example"]) @tool diff --git a/examples/llamaindex/llamaindex_example.py b/examples/llamaindex/llamaindex_example.py index f1aa2c750..df23899bc 100644 --- a/examples/llamaindex/llamaindex_example.py +++ b/examples/llamaindex/llamaindex_example.py @@ -8,7 +8,7 @@ from llama_index.embeddings.huggingface import HuggingFaceEmbedding from llama_index.llms.huggingface import HuggingFaceLLM -handler = AgentOpsHandler() +handler = AgentOpsHandler(tags=["llamaindex", "rag", "agentops-example"]) handler.init() load_dotenv() diff --git a/examples/mem0/mem0_memory_example.py b/examples/mem0/mem0_memory_example.py index 0573e06d6..00b6953d1 100644 --- a/examples/mem0/mem0_memory_example.py +++ b/examples/mem0/mem0_memory_example.py @@ -123,7 +123,7 @@ async def demonstrate_async_memory(local_config, sample_messages, sample_prefere by running multiple memory operations in parallel. """ - tracer = agentops.start_trace("mem0_memory_async_example") + tracer = agentops.start_trace("mem0_memory_async_example", tags=["mem0", "async", "memory-management"]) try: # Initialize async Memory with configuration async_memory = await AsyncMemory.from_config(local_config) @@ -186,7 +186,7 @@ async def search_memory(query): # Initialize AgentOps -agentops.init() +agentops.init(tags=["mem0", "memory-management", "agentops-example"]) # Configuration for local memory (Memory) # This configuration specifies the LLM provider and model settings diff --git a/examples/openai/multi_tool_orchestration.py b/examples/openai/multi_tool_orchestration.py index d0ded8974..991516531 100644 --- a/examples/openai/multi_tool_orchestration.py +++ b/examples/openai/multi_tool_orchestration.py @@ -28,7 +28,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") os.environ["PINECONE_API_KEY"] = os.getenv("PINECONE_API_KEY", "your_pinecone_api_key_here") -agentops.init(auto_start_session=True) +agentops.init(auto_start_session=True, tags=["openai", "multi-tool", "agentops-example"]) tracer = agentops.start_trace( trace_name="Multi-Tool Orchestration with RAG", tags=["multi-tool-orchestration-rag-demo", "openai-responses", "agentops-example"], @@ -354,4 +354,3 @@ def query_pinecone_index(client, index, model, query_text): # Here, we have seen how to utilize OpenAI's Responses API to implement a Retrieval-Augmented Generation (RAG) approach with multi-tool calling capabilities. It showcases an example where the model selects the appropriate tool based on the input query: general questions may be handled by built-in tools such as web-search, while specific medical inquiries related to internal knowledge are addressed by retrieving context from a vector database (such as Pinecone) via function calls. Additonally, we have showcased how multiple tool calls can be sequentially combined to generate a final response based on our instructions provided to responses API. Happy coding! - diff --git a/examples/openai/openai_example_async.py b/examples/openai/openai_example_async.py index fb4e6e951..2b1ef22ab 100644 --- a/examples/openai/openai_example_async.py +++ b/examples/openai/openai_example_async.py @@ -20,7 +20,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=True) +agentops.init(auto_start_session=True, tags=["openai", "async", "agentops-example"]) tracer = agentops.start_trace( trace_name="OpenAI Async Example", tags=["openai-async-example", "openai", "agentops-example"] ) diff --git a/examples/openai/openai_example_sync.py b/examples/openai/openai_example_sync.py index c5f9a2f5d..3a26ee7f4 100644 --- a/examples/openai/openai_example_sync.py +++ b/examples/openai/openai_example_sync.py @@ -20,7 +20,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=True) +agentops.init(auto_start_session=True, tags=["openai", "sync", "agentops-example"]) tracer = agentops.start_trace( trace_name="OpenAI Sync Example", tags=["openai-sync-example", "openai", "agentops-example"] ) diff --git a/examples/openai/web_search.py b/examples/openai/web_search.py index 2aa051f1d..a5efd9be7 100644 --- a/examples/openai/web_search.py +++ b/examples/openai/web_search.py @@ -26,7 +26,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") -agentops.init(auto_start_session=True) +agentops.init(auto_start_session=True, tags=["openai", "web-search", "agentops-example"]) tracer = agentops.start_trace( trace_name="OpenAI Responses Example", tags=["openai-responses-example", "openai", "agentops-example"] ) @@ -131,4 +131,3 @@ # 3ļøāƒ£ Re-submit tool results for summarization ā†’ another request # # We are very excited for you to try out the Responses API and see how it can simplify your code and make it easier to build complex, multimodal, tool-augmented interactions! - diff --git a/examples/openai_agents/agent_patterns.py b/examples/openai_agents/agent_patterns.py index f2357649e..2390e7001 100644 --- a/examples/openai_agents/agent_patterns.py +++ b/examples/openai_agents/agent_patterns.py @@ -78,7 +78,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") # Initialize AgentOps -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, tags=["openai-agents", "patterns", "agentops-example"]) # Note: tracer will be defined in each section's cell for clarity, using the specific tags for that pattern. # ## 1. Agents as Tools Pattern # @@ -748,4 +748,3 @@ async def run_streaming_guardrails_demo(): # - **Workflow optimization** - Identifying bottlenecks and improving agent coordination # # Visit [app.agentops.ai](https://app.agentops.ai) to explore your agent sessions and gain deeper insights into your AI application's behavior. - diff --git a/examples/openai_agents/agents_tools.py b/examples/openai_agents/agents_tools.py index b76bea472..a89051034 100644 --- a/examples/openai_agents/agents_tools.py +++ b/examples/openai_agents/agents_tools.py @@ -48,7 +48,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False, tags=["agentops-example"]) +agentops.init(auto_start_session=False, tags=["openai-agents", "tools", "agentops-example"]) # ## 1. Code Interpreter Tool # diff --git a/examples/xai/grok_examples.py b/examples/xai/grok_examples.py index 420f42a51..bb385f3a6 100644 --- a/examples/xai/grok_examples.py +++ b/examples/xai/grok_examples.py @@ -17,7 +17,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, tags=["xai", "grok", "agentops-example"]) tracer = agentops.start_trace(trace_name="XAI Example", tags=["xai-example", "grok", "agentops-example"]) # And we are all set! Note the seesion url above. We will use it to track the chatbot. @@ -89,4 +89,3 @@ # We end the session with a success state and a success reason. This is useful if you want to track the success or failure of the chatbot. In that case you can set the end state to failure and provide a reason. By default the session will have an indeterminate end state. - diff --git a/examples/xai/grok_vision_examples.py b/examples/xai/grok_vision_examples.py index 8ceb56efb..570911ca3 100644 --- a/examples/xai/grok_vision_examples.py +++ b/examples/xai/grok_vision_examples.py @@ -18,7 +18,7 @@ os.environ["XAI_API_KEY"] = os.getenv("XAI_API_KEY", "your_xai_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, tags=["xai", "grok-vision", "agentops-example"]) tracer = agentops.start_trace(trace_name="XAI Vision Example", tags=["xai-example", "grok-vision", "agentops-example"]) # And we are all set! Note the seesion url above. We will use it to track the program's performance. @@ -72,4 +72,3 @@ # We end the session with a success state and a success reason. This is useful if you want to track the success or failure of the chatbot. In that case you can set the end state to failure and provide a reason. By default the session will have an indeterminate end state. - From d985e46a2646ad1b1bfa23412abc0379c9b176e4 Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 23:52:04 -0700 Subject: [PATCH 23/25] add trace naming --- examples/context_manager/basic_usage.py | 4 +-- examples/context_manager/error_handling.py | 2 +- examples/mem0/mem0_memory_example.py | 4 +-- examples/mem0/mem0_memoryclient_example.py | 4 +-- examples/openai/multi_tool_orchestration.py | 2 +- examples/openai_agents/agent_patterns.py | 25 ++++++++++++------- examples/openai_agents/agents_tools.py | 8 +++--- .../openai_agents/customer_service_agent.py | 7 +++--- .../smolagents/multi_smolagents_system.py | 3 +-- examples/smolagents/text_to_sql.py | 3 +-- examples/xai/grok_examples.py | 2 +- examples/xai/grok_vision_examples.py | 3 ++- 12 files changed, 36 insertions(+), 31 deletions(-) diff --git a/examples/context_manager/basic_usage.py b/examples/context_manager/basic_usage.py index 0dce207e3..07ac61cb0 100644 --- a/examples/context_manager/basic_usage.py +++ b/examples/context_manager/basic_usage.py @@ -43,7 +43,7 @@ def basic_context_manager_example(): agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "agentops-example"]) # Use native TraceContext context manager - with agentops.start_trace("basic_example", tags=["basic", "demo"]): + with agentops.start_trace("Context Manager Basic Example", tags=["basic", "demo"]): print("Trace started") # Create and use agent @@ -61,7 +61,7 @@ def multiple_parallel_traces(): agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "agentops-example"]) # First trace - with agentops.start_trace("task_1", tags=["parallel", "task-1"]): + with agentops.start_trace("Context Manager Task 1", tags=["parallel", "task-1"]): print("Task 1 started") agent1 = SimpleAgent("Agent1") result1 = agent1.process_data("task 1 data") diff --git a/examples/context_manager/error_handling.py b/examples/context_manager/error_handling.py index b8306aa96..fb1b5e42b 100644 --- a/examples/context_manager/error_handling.py +++ b/examples/context_manager/error_handling.py @@ -72,7 +72,7 @@ def basic_exception_handling(): for error_type in error_types: try: - with agentops.start_trace(f"basic_{error_type}", tags=["basic", "error-handling"]): + with agentops.start_trace(f"Context Manager Basic {error_type}", tags=["basic", "error-handling"]): print(f"Started trace for {error_type}") agent = ErrorProneAgent(f"BasicAgent_{error_type}") diff --git a/examples/mem0/mem0_memory_example.py b/examples/mem0/mem0_memory_example.py index 00b6953d1..55cc43cb5 100644 --- a/examples/mem0/mem0_memory_example.py +++ b/examples/mem0/mem0_memory_example.py @@ -57,7 +57,7 @@ def demonstrate_sync_memory(local_config, sample_messages, sample_preferences, u must complete before the next one begins. """ - tracer = agentops.start_trace("mem0_memory_example", tags=["mem0_memory_example"]) + tracer = agentops.start_trace("Mem0 Memory Example", tags=["mem0_memory_example"]) try: # Initialize sync Memory with local configuration memory = Memory.from_config(local_config) @@ -123,7 +123,7 @@ async def demonstrate_async_memory(local_config, sample_messages, sample_prefere by running multiple memory operations in parallel. """ - tracer = agentops.start_trace("mem0_memory_async_example", tags=["mem0", "async", "memory-management"]) + tracer = agentops.start_trace("Mem0 Memory Async Example", tags=["mem0", "async", "memory-management"]) try: # Initialize async Memory with configuration async_memory = await AsyncMemory.from_config(local_config) diff --git a/examples/mem0/mem0_memoryclient_example.py b/examples/mem0/mem0_memoryclient_example.py index 79f85d962..5fa6e6a40 100644 --- a/examples/mem0/mem0_memoryclient_example.py +++ b/examples/mem0/mem0_memoryclient_example.py @@ -54,7 +54,7 @@ def demonstrate_sync_memory_client(sample_messages, sample_preferences, user_id) Cloud benefit: All memory operations are handled by Mem0's infrastructure, providing scalability and persistence without local storage management. """ - agentops.start_trace("mem0_memoryclient_example", tags=["mem0_memoryclient_example"]) + agentops.start_trace("Mem0 MemoryClient Example", tags=["mem0_memoryclient_example"]) try: # Initialize sync MemoryClient with API key for cloud access client = MemoryClient(api_key=mem0_api_key) @@ -106,7 +106,7 @@ async def demonstrate_async_memory_client(sample_messages, sample_preferences, u concurrently, significantly reducing total execution time compared to sequential calls. This is especially beneficial when dealing with network I/O to cloud services. """ - agentops.start_trace("mem0_memoryclient_example", tags=["mem0_memoryclient_example"]) + agentops.start_trace("Mem0 MemoryClient Async Example", tags=["mem0_memoryclient_example"]) try: # Initialize async MemoryClient for concurrent cloud operations async_client = AsyncMemoryClient(api_key=mem0_api_key) diff --git a/examples/openai/multi_tool_orchestration.py b/examples/openai/multi_tool_orchestration.py index 991516531..3077fff83 100644 --- a/examples/openai/multi_tool_orchestration.py +++ b/examples/openai/multi_tool_orchestration.py @@ -30,7 +30,7 @@ agentops.init(auto_start_session=True, tags=["openai", "multi-tool", "agentops-example"]) tracer = agentops.start_trace( - trace_name="Multi-Tool Orchestration with RAG", + trace_name="OpenAI Multi-Tool Orchestration with RAG", tags=["multi-tool-orchestration-rag-demo", "openai-responses", "agentops-example"], ) client = OpenAI() diff --git a/examples/openai_agents/agent_patterns.py b/examples/openai_agents/agent_patterns.py index 2390e7001..cfc552dd5 100644 --- a/examples/openai_agents/agent_patterns.py +++ b/examples/openai_agents/agent_patterns.py @@ -88,7 +88,7 @@ # # This pattern demonstrates using agents as callable tools within other agents. The orchestrator agent receives a user message and then picks which specialized agents to call as tools. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Agents as Tools Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents as Tools Pattern", tags=["agentops-example", "openai-agents"]) # Define specialized translation agents spanish_agent = Agent( @@ -174,7 +174,8 @@ async def run_agents_as_tools_demo(): # # This pattern demonstrates breaking down a complex task into a series of smaller, sequential steps. Each step is performed by an agent, and the output of one agent is used as input to the next. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Deterministic Flow Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents Deterministic Flow Pattern", + tags=["agentops-example", "openai-agents"]) # Define the story generation workflow story_outline_agent = Agent( @@ -246,7 +247,8 @@ async def run_deterministic_flow_demo(): # # For this demo, we'll allow the user to choose which tool use behavior to test: # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Forcing Tool Use Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents Forcing Tool Use Pattern", + tags=["agentops-example", "openai-agents"]) # Define the weather tool and agent @@ -325,7 +327,8 @@ async def run_forcing_tool_use_demo(tool_use_behavior: str): # # This pattern demonstrates how to use input guardrails to validate user inputs before they reach the main agent. Guardrails can prevent inappropriate or off-topic requests from being processed. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Input Guardrails Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents Input Guardrails Pattern", + tags=["agentops-example", "openai-agents"]) # Define the guardrail @@ -388,7 +391,8 @@ async def run_input_guardrails_demo(): # # This pattern shows how to use one LLM to evaluate and improve the output of another. The first agent generates content, and the second agent judges the quality and provides feedback for improvement. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="LLM as a Judge Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents LLM as a Judge Pattern", + tags=["agentops-example", "openai-agents"]) # Define the story generation and evaluation agents story_outline_generator = Agent( @@ -468,7 +472,8 @@ async def run_llm_as_judge_demo(): # # This pattern demonstrates how to use output guardrails to validate agent outputs after they are generated. This can help prevent sensitive information from being shared or ensure outputs meet quality standards. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Output Guardrails Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents Output Guardrails Pattern", + tags=["agentops-example", "openai-agents"]) # The agent's output type @@ -535,7 +540,8 @@ async def run_output_guardrails_demo(): # # This pattern shows how to run multiple agents in parallel to improve latency or generate multiple options to choose from. In this example, we run translation agents multiple times and pick the best result. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Output Guardrails Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents Parallelization Pattern", + tags=["agentops-example", "openai-agents"]) # Define agents for parallelization spanish_translation_agent = Agent( @@ -592,7 +598,7 @@ async def run_parallelization_demo(): # # This pattern demonstrates handoffs and routing between specialized agents. The triage agent receives the first message and hands off to the appropriate agent based on the language of the request. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Routing Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents Routing Pattern", tags=["agentops-example", "openai-agents"]) # Define language-specific agents french_routing_agent = Agent( @@ -645,7 +651,8 @@ async def run_routing_demo(): # # This pattern shows how to use guardrails during streaming to provide real-time validation. Unlike output guardrails that run after completion, streaming guardrails can interrupt the generation process early. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Streaming Guardrails Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents Streaming Guardrails Pattern", + tags=["agentops-example", "openai-agents"]) # Define streaming guardrail agent streaming_agent = Agent( diff --git a/examples/openai_agents/agents_tools.py b/examples/openai_agents/agents_tools.py index a89051034..79460903e 100644 --- a/examples/openai_agents/agents_tools.py +++ b/examples/openai_agents/agents_tools.py @@ -61,7 +61,7 @@ # - Handle data processing tasks # Start the AgentOps trace session tracer = agentops.start_trace( - trace_name="Code Interpreter Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] + trace_name="OpenAI Agents Code Interpreter Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] ) @@ -111,7 +111,7 @@ async def run_code_interpreter_demo(): # **Note:** This example requires a pre-configured vector store ID. # Start the AgentOps trace session tracer = agentops.start_trace( - trace_name="File Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] + trace_name="OpenAI Agents File Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] ) @@ -157,7 +157,7 @@ async def run_file_search_demo(): # - Automatic image saving and display # Start the AgentOps trace session tracer = agentops.start_trace( - trace_name="Image Generation Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] + trace_name="OpenAI Agents Image Generation Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] ) @@ -222,7 +222,7 @@ async def run_image_generation_demo(): # - Configurable search parameters # Start the AgentOps trace session tracer = agentops.start_trace( - trace_name="Web Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] + trace_name="OpenAI Agents Web Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] ) diff --git a/examples/openai_agents/customer_service_agent.py b/examples/openai_agents/customer_service_agent.py index 67b19f56b..208b789a2 100644 --- a/examples/openai_agents/customer_service_agent.py +++ b/examples/openai_agents/customer_service_agent.py @@ -54,7 +54,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") agentops.init(tags=["customer-service-agent", "openai-agents", "agentops-example"], auto_start_session=False) -tracer = agentops.start_trace(trace_name="Customer Service Agent") +tracer = agentops.start_trace(trace_name="OpenAI Agents Customer Service Agent") # Context model for the airline agent @@ -102,13 +102,13 @@ async def update_seat(context: RunContextWrapper[AirlineAgentContext], confirmat return f"Updated seat to {new_seat} for confirmation number {confirmation_number}" -### HOOKS +# HOOKS async def on_seat_booking_handoff(context: RunContextWrapper[AirlineAgentContext]) -> None: flight_number = f"FLT-{random.randint(100, 999)}" context.context.flight_number = flight_number -### AGENTS +# AGENTS faq_agent = Agent[AirlineAgentContext]( name="FAQ Agent", handoff_description="A helpful agent that can answer questions about the airline.", @@ -208,4 +208,3 @@ async def main(): # - Scale your AI applications with confidence in tool reliability # # Visit [app.agentops.ai](https://app.agentops.ai) to explore your tool usage sessions and gain deeper insights into your AI application's tool interactions. - diff --git a/examples/smolagents/multi_smolagents_system.py b/examples/smolagents/multi_smolagents_system.py index e223c1291..a09e0a194 100644 --- a/examples/smolagents/multi_smolagents_system.py +++ b/examples/smolagents/multi_smolagents_system.py @@ -49,7 +49,7 @@ agentops.init(auto_start_session=False) tracer = agentops.start_trace( - trace_name="Orchestrate a Multi-Agent System", tags=["smolagents", "example", "multi-agent", "agentops-example"] + trace_name="Smolagents Multi-Agent System Orchestration", tags=["smolagents", "example", "multi-agent", "agentops-example"] ) model = LiteLLMModel("openai/gpt-4o-mini") # ## Create a Web Search Tool @@ -123,4 +123,3 @@ def visit_webpage(url: str) -> str: raise # You can view the session in the [AgentOps dashboard](https://app.agentops.ai/sessions) by clicking the link provided after ending the session. - diff --git a/examples/smolagents/text_to_sql.py b/examples/smolagents/text_to_sql.py index e532c0695..a8db8477e 100644 --- a/examples/smolagents/text_to_sql.py +++ b/examples/smolagents/text_to_sql.py @@ -101,7 +101,7 @@ def sql_engine(query: str) -> str: agentops.init(auto_start_session=False) tracer = agentops.start_trace( - trace_name="Text-to-SQL", tags=["smolagents", "example", "text-to-sql", "agentops-example"] + trace_name="Smolagents Text-to-SQL", tags=["smolagents", "example", "text-to-sql", "agentops-example"] ) model = LiteLLMModel("openai/gpt-4o-mini") agent = CodeAgent( @@ -169,4 +169,3 @@ def sql_engine(query: str) -> str: raise # You can view the session in the [AgentOps dashboard](https://app.agentops.ai/sessions) by clicking the link provided after ending the session. - diff --git a/examples/xai/grok_examples.py b/examples/xai/grok_examples.py index bb385f3a6..6b03eed0c 100644 --- a/examples/xai/grok_examples.py +++ b/examples/xai/grok_examples.py @@ -18,7 +18,7 @@ # Next we initialize the AgentOps client. agentops.init(auto_start_session=False, tags=["xai", "grok", "agentops-example"]) -tracer = agentops.start_trace(trace_name="XAI Example", tags=["xai-example", "grok", "agentops-example"]) +tracer = agentops.start_trace(trace_name="XAI Grok Example", tags=["xai-example", "grok", "agentops-example"]) # And we are all set! Note the seesion url above. We will use it to track the chatbot. # diff --git a/examples/xai/grok_vision_examples.py b/examples/xai/grok_vision_examples.py index 570911ca3..54c778fc4 100644 --- a/examples/xai/grok_vision_examples.py +++ b/examples/xai/grok_vision_examples.py @@ -19,7 +19,8 @@ # Next we initialize the AgentOps client. agentops.init(auto_start_session=False, tags=["xai", "grok-vision", "agentops-example"]) -tracer = agentops.start_trace(trace_name="XAI Vision Example", tags=["xai-example", "grok-vision", "agentops-example"]) +tracer = agentops.start_trace(trace_name="XAI Grok Vision Example", tags=[ + "xai-example", "grok-vision", "agentops-example"]) # And we are all set! Note the seesion url above. We will use it to track the program's performance. # From 4741c126e0952769f052366406a504fd3062663c Mon Sep 17 00:00:00 2001 From: reibs Date: Mon, 7 Jul 2025 23:58:48 -0700 Subject: [PATCH 24/25] pre-commit checks --- agentops/__init__.py | 6 +- agentops/validation.py | 66 +++--- examples/ag2/async_human_input.py | 4 +- examples/ag2/tools_wikipedia_search.py | 4 +- examples/agno/agno_async_operations.py | 2 +- examples/agno/agno_basic_agents.py | 2 +- examples/agno/agno_research_team.py | 2 +- examples/agno/agno_tool_integrations.py | 30 +-- examples/agno/agno_workflow_setup.py | 2 +- .../agentops-anthropic-understanding-tools.py | 3 +- examples/anthropic/anthropic-example-async.py | 3 +- examples/anthropic/anthropic-example-sync.py | 4 +- examples/autogen/AgentChat.py | 2 +- examples/autogen/MathAgent.py | 2 +- examples/context_manager/basic_usage.py | 2 +- examples/context_manager/error_handling.py | 2 +- .../context_manager/production_patterns.py | 2 +- examples/crewai/job_posting.py | 2 +- examples/crewai/markdown_validator.py | 2 +- examples/google_adk/human_approval.py | 10 +- examples/google_genai/gemini_example.py | 2 +- examples/langchain/langchain_examples.py | 3 +- examples/langgraph/langgraph_example.py | 2 +- examples/litellm/litellm_example.py | 2 +- examples/llamaindex/llamaindex_example.py | 3 +- examples/mem0/mem0_memory_example.py | 2 +- examples/mem0/mem0_memoryclient_example.py | 2 +- examples/openai/multi_tool_orchestration.py | 2 +- examples/openai/openai_example_async.py | 8 +- examples/openai/openai_example_sync.py | 2 +- examples/openai/web_search.py | 2 +- examples/openai_agents/agent_guardrails.py | 4 +- examples/openai_agents/agent_patterns.py | 37 +-- examples/openai_agents/agents_tools.py | 2 +- .../openai_agents/customer_service_agent.py | 2 +- .../smolagents/multi_smolagents_system.py | 5 +- examples/smolagents/text_to_sql.py | 3 +- examples/watsonx/watsonx-streaming.py | 2 +- examples/watsonx/watsonx-text-chat.py | 2 +- examples/watsonx/watsonx-tokeniation-model.py | 2 +- examples/xai/grok_examples.py | 2 +- examples/xai/grok_vision_examples.py | 7 +- tests/unit/test_validation.py | 212 ++++++------------ 43 files changed, 187 insertions(+), 275 deletions(-) diff --git a/agentops/__init__.py b/agentops/__init__.py index d7ad52930..a03089318 100755 --- a/agentops/__init__.py +++ b/agentops/__init__.py @@ -35,11 +35,7 @@ import threading # Import validation functions -from agentops.validation import ( - validate_trace_spans, - print_validation_summary, - ValidationError -) +from agentops.validation import validate_trace_spans, print_validation_summary, ValidationError # Thread-safe client management _client_lock = threading.Lock() diff --git a/agentops/validation.py b/agentops/validation.py index a9926f2c8..7b9cda2e2 100644 --- a/agentops/validation.py +++ b/agentops/validation.py @@ -15,6 +15,7 @@ class ValidationError(Exception): """Raised when span validation fails.""" + pass @@ -33,20 +34,20 @@ def get_jwt_token(api_key: Optional[str] = None) -> str: """ if api_key is None: from agentops import get_client + client = get_client() if client and client.config.api_key: api_key = client.config.api_key else: import os + api_key = os.getenv("AGENTOPS_API_KEY") if not api_key: raise ValueError("No API key provided and AGENTOPS_API_KEY environment variable not set") try: response = requests.post( - "https://api.agentops.ai/public/v1/auth/access_token", - json={"api_key": api_key}, - timeout=10 + "https://api.agentops.ai/public/v1/auth/access_token", json={"api_key": api_key}, timeout=10 ) response.raise_for_status() return response.json()["bearer"] @@ -72,7 +73,7 @@ def get_trace_details(trace_id: str, jwt_token: str) -> Dict[str, Any]: response = requests.get( f"https://api.agentops.ai/public/v1/traces/{trace_id}", headers={"Authorization": f"Bearer {jwt_token}"}, - timeout=10 + timeout=10, ) response.raise_for_status() return response.json() @@ -98,7 +99,7 @@ def get_trace_metrics(trace_id: str, jwt_token: str) -> Dict[str, Any]: response = requests.get( f"https://api.agentops.ai/public/v1/traces/{trace_id}/metrics", headers={"Authorization": f"Bearer {jwt_token}"}, - timeout=10 + timeout=10, ) response.raise_for_status() return response.json() @@ -146,6 +147,7 @@ def check_llm_spans(spans: List[Dict[str, Any]]) -> Tuple[bool, List[str]]: # Structure 3: Look for SpanAttributes.AGENTOPS_SPAN_KIND if not span_kind and isinstance(span_attributes, dict): from agentops.semconv import SpanAttributes + span_kind = span_attributes.get(SpanAttributes.AGENTOPS_SPAN_KIND, "") # Check if this is an LLM span by span kind @@ -172,10 +174,7 @@ def check_llm_spans(spans: List[Dict[str, Any]]) -> Tuple[bool, List[str]]: llm_request_type = span_attributes.get("llm.request.type", "") # Check if it's a chat or completion request (the main LLM types) - if llm_request_type in [ - LLMRequestTypeValues.CHAT.value, - LLMRequestTypeValues.COMPLETION.value - ]: + if llm_request_type in [LLMRequestTypeValues.CHAT.value, LLMRequestTypeValues.COMPLETION.value]: is_llm_span = True if is_llm_span: @@ -191,7 +190,7 @@ def validate_trace_spans( retry_delay: float = 1.0, check_llm: bool = True, min_spans: int = 1, - api_key: Optional[str] = None + api_key: Optional[str] = None, ) -> Dict[str, Any]: """ Validate that spans have been sent to AgentOps. @@ -217,10 +216,11 @@ def validate_trace_spans( # Try to get from current span try: from opentelemetry.trace import get_current_span + current_span = get_current_span() - if current_span and hasattr(current_span, 'get_span_context'): + if current_span and hasattr(current_span, "get_span_context"): span_context = current_span.get_span_context() - if hasattr(span_context, 'trace_id') and span_context.trace_id: + if hasattr(span_context, "trace_id") and span_context.trace_id: if isinstance(span_context.trace_id, int): trace_id = format(span_context.trace_id, "032x") else: @@ -230,9 +230,9 @@ def validate_trace_spans( elif trace_context is not None and trace_id is None: # Extract from TraceContext - if hasattr(trace_context, 'span') and trace_context.span: + if hasattr(trace_context, "span") and trace_context.span: span_context = trace_context.span.get_span_context() - if hasattr(span_context, 'trace_id'): + if hasattr(span_context, "trace_id"): if isinstance(span_context.trace_id, int): trace_id = format(span_context.trace_id, "032x") else: @@ -262,7 +262,7 @@ def validate_trace_spans( "spans": spans, "has_llm_spans": False, "llm_span_names": [], - "metrics": None + "metrics": None, } # Get metrics first - if we have token usage, we definitely have LLM spans @@ -271,11 +271,13 @@ def validate_trace_spans( result["metrics"] = metrics if metrics: - logger.info(f"Trace metrics - Total tokens: {metrics.get('total_tokens', 0)}, " - f"Cost: ${metrics.get('total_cost', '0.0000')}") + logger.info( + f"Trace metrics - Total tokens: {metrics.get('total_tokens', 0)}, " + f"Cost: ${metrics.get('total_cost', '0.0000')}" + ) # If we have token usage > 0, we definitely have LLM activity - if metrics.get('total_tokens', 0) > 0: + if metrics.get("total_tokens", 0) > 0: result["has_llm_spans"] = True logger.info("LLM activity confirmed via token usage metrics") except Exception as e: @@ -303,8 +305,10 @@ def validate_trace_spans( return result else: - logger.debug(f"Only {len(spans)} spans found, expected at least {min_spans}. " - f"Retrying... ({attempt + 1}/{max_retries})") + logger.debug( + f"Only {len(spans)} spans found, expected at least {min_spans}. " + f"Retrying... ({attempt + 1}/{max_retries})" + ) except ApiServerException as e: logger.debug(f"API error during validation: {e}. Retrying... ({attempt + 1}/{max_retries})") @@ -314,9 +318,9 @@ def validate_trace_spans( raise ValidationError( f"Validation failed for trace {trace_id} after {max_retries} attempts. " - f"Expected at least {min_spans} spans" + - (", including LLM activity" if check_llm else "") + - ". Please check that tracking is properly configured." + f"Expected at least {min_spans} spans" + + (", including LLM activity" if check_llm else "") + + ". Please check that tracking is properly configured." ) @@ -327,24 +331,24 @@ def print_validation_summary(result: Dict[str, Any]) -> None: Args: result: Validation result dictionary from validate_trace_spans """ - print("\n" + "="*50) + print("\n" + "=" * 50) print("šŸ” AgentOps Span Validation Results") - print("="*50) + print("=" * 50) print(f"āœ… Found {result['span_count']} span(s) in trace") - if result.get('has_llm_spans'): - if result.get('llm_span_names'): + if result.get("has_llm_spans"): + if result.get("llm_span_names"): print(f"āœ… Found LLM spans: {', '.join(result['llm_span_names'])}") else: # LLM activity confirmed via metrics print("āœ… LLM activity confirmed via token usage metrics") - elif 'has_llm_spans' in result: + elif "has_llm_spans" in result: print("āš ļø No LLM activity detected") - if result.get('metrics'): - metrics = result['metrics'] - print(f"\nšŸ“Š Trace Metrics:") + if result.get("metrics"): + metrics = result["metrics"] + print("\nšŸ“Š Trace Metrics:") print(f" - Total tokens: {metrics.get('total_tokens', 0)}") print(f" - Prompt tokens: {metrics.get('prompt_tokens', 0)}") print(f" - Completion tokens: {metrics.get('completion_tokens', 0)}") diff --git a/examples/ag2/async_human_input.py b/examples/ag2/async_human_input.py index 4b6265d64..03112ab1a 100644 --- a/examples/ag2/async_human_input.py +++ b/examples/ag2/async_human_input.py @@ -106,7 +106,7 @@ async def main(): agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) @@ -114,5 +114,3 @@ async def main(): except agentops.ValidationError as e: print(f"\nāŒ Error validating spans: {e}") raise - - diff --git a/examples/ag2/tools_wikipedia_search.py b/examples/ag2/tools_wikipedia_search.py index 391de9f82..028d7139d 100644 --- a/examples/ag2/tools_wikipedia_search.py +++ b/examples/ag2/tools_wikipedia_search.py @@ -72,7 +72,7 @@ agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) @@ -80,5 +80,3 @@ except agentops.ValidationError as e: print(f"\nāŒ Error validating spans: {e}") raise - - diff --git a/examples/agno/agno_async_operations.py b/examples/agno/agno_async_operations.py index c88e30481..ae6f9cf7f 100644 --- a/examples/agno/agno_async_operations.py +++ b/examples/agno/agno_async_operations.py @@ -76,7 +76,7 @@ async def task3(): agentops.end_trace(tracer, end_state="Error") # Let's check programmatically that spans were recorded in AgentOps - print("\n" + "="*50) + print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/agno/agno_basic_agents.py b/examples/agno/agno_basic_agents.py index eeb430114..2dcfb43ac 100644 --- a/examples/agno/agno_basic_agents.py +++ b/examples/agno/agno_basic_agents.py @@ -89,7 +89,7 @@ def demonstrate_basic_agents(): agentops.end_trace(tracer, end_state="Error") # Let's check programmatically that spans were recorded in AgentOps - print("\n" + "="*50) + print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/agno/agno_research_team.py b/examples/agno/agno_research_team.py index 9b96d5442..5338922b5 100644 --- a/examples/agno/agno_research_team.py +++ b/examples/agno/agno_research_team.py @@ -205,7 +205,7 @@ def demonstrate_research_team(): agentops.end_trace(tracer, end_state="Error") # Let's check programmatically that spans were recorded in AgentOps - print("\n" + "="*50) + print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/agno/agno_tool_integrations.py b/examples/agno/agno_tool_integrations.py index db5c29648..709f5e575 100644 --- a/examples/agno/agno_tool_integrations.py +++ b/examples/agno/agno_tool_integrations.py @@ -32,10 +32,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_agentops_api_key_here") # Initialize AgentOps -agentops.init( - auto_start_session=False, - tags=["agno-tools", "tool-integration", "demo"] -) +agentops.init(auto_start_session=False, tags=["agno-tools", "tool-integration", "demo"]) def demonstrate_tool_integration(): @@ -56,7 +53,7 @@ def demonstrate_tool_integration(): role="Research information using Google Search", model=OpenAIChat(id="gpt-4o-mini"), tools=[GoogleSearchTools()], - instructions="You are a research assistant. Use Google Search to find accurate, up-to-date information." + instructions="You are a research assistant. Use Google Search to find accurate, up-to-date information.", ) response = search_agent.run("What are the latest developments in AI agents?") @@ -70,15 +67,11 @@ def demonstrate_tool_integration(): name="Research Agent", role="Comprehensive research using multiple tools", model=OpenAIChat(id="gpt-4o-mini"), - tools=[ - GoogleSearchTools(), - ArxivTools(), - DuckDuckGoTools() - ], + tools=[GoogleSearchTools(), ArxivTools(), DuckDuckGoTools()], instructions="""You are a comprehensive research assistant. Use Google Search for general information, Arxiv for academic papers, and DuckDuckGo as an alternative search engine. - Provide well-researched, balanced information from multiple sources.""" + Provide well-researched, balanced information from multiple sources.""", ) response = research_agent.run( @@ -96,12 +89,10 @@ def demonstrate_tool_integration(): role="Find and summarize academic papers", model=OpenAIChat(id="gpt-4o-mini"), tools=[ArxivTools()], - instructions="You are an academic research assistant. Use Arxiv to find relevant papers and provide concise summaries." + instructions="You are an academic research assistant. Use Arxiv to find relevant papers and provide concise summaries.", ) - response = academic_agent.run( - "Find recent papers about tool augmented language models" - ) + response = academic_agent.run("Find recent papers about tool augmented language models") print(f"Academic Agent Response:\n{response.content}") # Example 4: Comparing Search Tools @@ -112,13 +103,10 @@ def demonstrate_tool_integration(): name="Comparison Agent", role="Compare results from different search engines", model=OpenAIChat(id="gpt-4o-mini"), - tools=[ - GoogleSearchTools(), - DuckDuckGoTools() - ], + tools=[GoogleSearchTools(), DuckDuckGoTools()], instructions="""Compare search results from Google and DuckDuckGo. Note any differences in results, ranking, or information quality. - Be objective in your comparison.""" + Be objective in your comparison.""", ) response = comparison_agent.run( @@ -145,7 +133,7 @@ def demonstrate_tool_integration(): raise # Let's check programmatically that spans were recorded in AgentOps - print("\n" + "="*50) + print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/agno/agno_workflow_setup.py b/examples/agno/agno_workflow_setup.py index 5b8c87560..bcfd8ad70 100644 --- a/examples/agno/agno_workflow_setup.py +++ b/examples/agno/agno_workflow_setup.py @@ -114,7 +114,7 @@ def demonstrate_workflows(): agentops.end_trace(tracer, end_state="Error") # Let's check programmatically that spans were recorded in AgentOps - print("\n" + "="*50) + print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/anthropic/agentops-anthropic-understanding-tools.py b/examples/anthropic/agentops-anthropic-understanding-tools.py index b15a3d78c..9a5863c30 100644 --- a/examples/anthropic/agentops-anthropic-understanding-tools.py +++ b/examples/anthropic/agentops-anthropic-understanding-tools.py @@ -374,7 +374,7 @@ def inventoryscan(): # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) @@ -382,4 +382,3 @@ def inventoryscan(): except agentops.ValidationError as e: print(f"\nāŒ Error validating spans: {e}") raise - diff --git a/examples/anthropic/anthropic-example-async.py b/examples/anthropic/anthropic-example-async.py index 5d5b63d7d..e305d652c 100644 --- a/examples/anthropic/anthropic-example-async.py +++ b/examples/anthropic/anthropic-example-async.py @@ -111,7 +111,7 @@ async def main(): # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) @@ -119,4 +119,3 @@ async def main(): except agentops.ValidationError as e: print(f"\nāŒ Error validating spans: {e}") raise - diff --git a/examples/anthropic/anthropic-example-sync.py b/examples/anthropic/anthropic-example-sync.py index 06bd10d23..972024a73 100644 --- a/examples/anthropic/anthropic-example-sync.py +++ b/examples/anthropic/anthropic-example-sync.py @@ -112,7 +112,7 @@ agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) @@ -120,5 +120,3 @@ except agentops.ValidationError as e: print(f"\nāŒ Error validating spans: {e}") raise - - diff --git a/examples/autogen/AgentChat.py b/examples/autogen/AgentChat.py index 783f6e5a8..d6c5b9938 100644 --- a/examples/autogen/AgentChat.py +++ b/examples/autogen/AgentChat.py @@ -86,7 +86,7 @@ async def main(): await model_client.close() # Let's check programmatically that spans were recorded in AgentOps - print("\n" + "="*50) + print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/autogen/MathAgent.py b/examples/autogen/MathAgent.py index 14184f426..52be4796b 100644 --- a/examples/autogen/MathAgent.py +++ b/examples/autogen/MathAgent.py @@ -101,7 +101,7 @@ async def main(): await model_client.close() # Let's check programmatically that spans were recorded in AgentOps - print("\n" + "="*50) + print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/context_manager/basic_usage.py b/examples/context_manager/basic_usage.py index 07ac61cb0..817806acb 100644 --- a/examples/context_manager/basic_usage.py +++ b/examples/context_manager/basic_usage.py @@ -140,7 +140,7 @@ def nested_traces_example(): print("\nAll examples completed!") # Let's check programmatically that spans were recorded in AgentOps - print("\n" + "="*50) + print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) diff --git a/examples/context_manager/error_handling.py b/examples/context_manager/error_handling.py index fb1b5e42b..d991dd243 100644 --- a/examples/context_manager/error_handling.py +++ b/examples/context_manager/error_handling.py @@ -317,7 +317,7 @@ def exception_chaining_example(): print("\nAll error handling examples completed!") # Let's check programmatically that spans were recorded in AgentOps - print("\n" + "="*50) + print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) diff --git a/examples/context_manager/production_patterns.py b/examples/context_manager/production_patterns.py index 5da0d06e0..8ba7c69dd 100644 --- a/examples/context_manager/production_patterns.py +++ b/examples/context_manager/production_patterns.py @@ -340,7 +340,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): print("\nAll production pattern examples completed!") # Let's check programmatically that spans were recorded in AgentOps - print("\n" + "="*50) + print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) diff --git a/examples/crewai/job_posting.py b/examples/crewai/job_posting.py index 6b1935008..7285e8b20 100644 --- a/examples/crewai/job_posting.py +++ b/examples/crewai/job_posting.py @@ -174,7 +174,7 @@ def industry_analysis_task(self, agent, company_domain, company_description): agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/crewai/markdown_validator.py b/examples/crewai/markdown_validator.py index 19184aa8b..49b59b69c 100644 --- a/examples/crewai/markdown_validator.py +++ b/examples/crewai/markdown_validator.py @@ -107,7 +107,7 @@ def markdown_validation_tool(file_path: str) -> str: # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) diff --git a/examples/google_adk/human_approval.py b/examples/google_adk/human_approval.py index 6ff142f02..7bf088449 100644 --- a/examples/google_adk/human_approval.py +++ b/examples/google_adk/human_approval.py @@ -31,8 +31,12 @@ AGENTOPS_API_KEY = os.getenv("AGENTOPS_API_KEY") or "your_agentops_api_key_here" # Initialize AgentOps - Just 2 lines! -agentops.init(AGENTOPS_API_KEY, trace_name="adk-human-approval-notebook", - auto_start_session=False, tags=["google-adk", "human-approval", "agentops-example"]) +agentops.init( + AGENTOPS_API_KEY, + trace_name="adk-human-approval-notebook", + auto_start_session=False, + tags=["google-adk", "human-approval", "agentops-example"], +) # Define some constants for our application. APP_NAME = "human_approval_app_notebook" @@ -212,7 +216,7 @@ async def main_notebook(): agentops.end_trace(end_state="Error") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) diff --git a/examples/google_genai/gemini_example.py b/examples/google_genai/gemini_example.py index 401fb112c..7a53505d5 100644 --- a/examples/google_genai/gemini_example.py +++ b/examples/google_genai/gemini_example.py @@ -48,7 +48,7 @@ print(f"Token count: {token_response.total_tokens}") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) diff --git a/examples/langchain/langchain_examples.py b/examples/langchain/langchain_examples.py index 36de4d6b7..1c1307882 100644 --- a/examples/langchain/langchain_examples.py +++ b/examples/langchain/langchain_examples.py @@ -79,10 +79,11 @@ def find_movie(genre: str) -> str: # Finally, check your run on [AgentOps](https://app.agentops.ai). You will see a session recorded with the LLM calls and tool usage. # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: import agentops + agentops.validate_trace_spans(trace_context=None) print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") except ImportError: diff --git a/examples/langgraph/langgraph_example.py b/examples/langgraph/langgraph_example.py index a84c080c5..6db661793 100644 --- a/examples/langgraph/langgraph_example.py +++ b/examples/langgraph/langgraph_example.py @@ -119,7 +119,7 @@ def run_example(): # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that we have enough spans tracked properly...") try: # LangGraph doesn't emit LLM spans in the same format, so we just check span count diff --git a/examples/litellm/litellm_example.py b/examples/litellm/litellm_example.py index 67bcde0c0..0a226200a 100644 --- a/examples/litellm/litellm_example.py +++ b/examples/litellm/litellm_example.py @@ -52,7 +52,7 @@ agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/llamaindex/llamaindex_example.py b/examples/llamaindex/llamaindex_example.py index df23899bc..d416f1636 100644 --- a/examples/llamaindex/llamaindex_example.py +++ b/examples/llamaindex/llamaindex_example.py @@ -66,10 +66,11 @@ print("šŸ”— The session link should be printed above by AgentOps.") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: import agentops + agentops.validate_trace_spans(trace_context=None) print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") except agentops.ValidationError as e: diff --git a/examples/mem0/mem0_memory_example.py b/examples/mem0/mem0_memory_example.py index 55cc43cb5..c8f8da7c4 100644 --- a/examples/mem0/mem0_memory_example.py +++ b/examples/mem0/mem0_memory_example.py @@ -229,7 +229,7 @@ async def search_memory(query): asyncio.run(demonstrate_async_memory(local_config, sample_messages, sample_preferences, user_id)) # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: # Note: Using trace_id since we ran multiple traces diff --git a/examples/mem0/mem0_memoryclient_example.py b/examples/mem0/mem0_memoryclient_example.py index 5fa6e6a40..74d307ac7 100644 --- a/examples/mem0/mem0_memoryclient_example.py +++ b/examples/mem0/mem0_memoryclient_example.py @@ -185,7 +185,7 @@ async def demonstrate_async_memory_client(sample_messages, sample_preferences, u asyncio.run(demonstrate_async_memory_client(sample_messages, sample_preferences, user_id)) # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) diff --git a/examples/openai/multi_tool_orchestration.py b/examples/openai/multi_tool_orchestration.py index 3077fff83..0b7fed626 100644 --- a/examples/openai/multi_tool_orchestration.py +++ b/examples/openai/multi_tool_orchestration.py @@ -343,7 +343,7 @@ def query_pinecone_index(client, index, model, query_text): agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/openai/openai_example_async.py b/examples/openai/openai_example_async.py index 2b1ef22ab..43d5e4f51 100644 --- a/examples/openai/openai_example_async.py +++ b/examples/openai/openai_example_async.py @@ -14,6 +14,7 @@ import os import asyncio from dotenv import load_dotenv + # Next, we'll grab our API keys. You can use dotenv like below or however else you like to load environment variables load_dotenv() os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") @@ -36,7 +37,10 @@ """ user_prompt = [ - {"type": "text", "text": "Write a very short mystery thriller story based on your understanding of the provided image."}, + { + "type": "text", + "text": "Write a very short mystery thriller story based on your understanding of the provided image.", + }, { "type": "image_url", "image_url": {"url": "https://www.cosy.sbg.ac.at/~pmeerw/Watermarking/lena_color.gif"}, @@ -78,7 +82,7 @@ async def main_stream(): agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/openai/openai_example_sync.py b/examples/openai/openai_example_sync.py index 3a26ee7f4..37ff5249f 100644 --- a/examples/openai/openai_example_sync.py +++ b/examples/openai/openai_example_sync.py @@ -65,7 +65,7 @@ agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: result = agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/openai/web_search.py b/examples/openai/web_search.py index a5efd9be7..1e90f596b 100644 --- a/examples/openai/web_search.py +++ b/examples/openai/web_search.py @@ -102,7 +102,7 @@ agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/openai_agents/agent_guardrails.py b/examples/openai_agents/agent_guardrails.py index ace92488f..1296245b6 100644 --- a/examples/openai_agents/agent_guardrails.py +++ b/examples/openai_agents/agent_guardrails.py @@ -10,7 +10,6 @@ # Import dependencies from pydantic import BaseModel from agents import ( - Agent, GuardrailFunctionOutput, InputGuardrailTripwireTriggered, @@ -85,7 +84,7 @@ async def main(): # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) @@ -93,4 +92,3 @@ async def main(): except agentops.ValidationError as e: print(f"\nāŒ Error validating spans: {e}") raise - diff --git a/examples/openai_agents/agent_patterns.py b/examples/openai_agents/agent_patterns.py index cfc552dd5..db51e513e 100644 --- a/examples/openai_agents/agent_patterns.py +++ b/examples/openai_agents/agent_patterns.py @@ -174,8 +174,9 @@ async def run_agents_as_tools_demo(): # # This pattern demonstrates breaking down a complex task into a series of smaller, sequential steps. Each step is performed by an agent, and the output of one agent is used as input to the next. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="OpenAI Agents Deterministic Flow Pattern", - tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Deterministic Flow Pattern", tags=["agentops-example", "openai-agents"] +) # Define the story generation workflow story_outline_agent = Agent( @@ -247,8 +248,9 @@ async def run_deterministic_flow_demo(): # # For this demo, we'll allow the user to choose which tool use behavior to test: # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="OpenAI Agents Forcing Tool Use Pattern", - tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Forcing Tool Use Pattern", tags=["agentops-example", "openai-agents"] +) # Define the weather tool and agent @@ -327,8 +329,9 @@ async def run_forcing_tool_use_demo(tool_use_behavior: str): # # This pattern demonstrates how to use input guardrails to validate user inputs before they reach the main agent. Guardrails can prevent inappropriate or off-topic requests from being processed. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="OpenAI Agents Input Guardrails Pattern", - tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Input Guardrails Pattern", tags=["agentops-example", "openai-agents"] +) # Define the guardrail @@ -391,8 +394,9 @@ async def run_input_guardrails_demo(): # # This pattern shows how to use one LLM to evaluate and improve the output of another. The first agent generates content, and the second agent judges the quality and provides feedback for improvement. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="OpenAI Agents LLM as a Judge Pattern", - tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents LLM as a Judge Pattern", tags=["agentops-example", "openai-agents"] +) # Define the story generation and evaluation agents story_outline_generator = Agent( @@ -472,8 +476,9 @@ async def run_llm_as_judge_demo(): # # This pattern demonstrates how to use output guardrails to validate agent outputs after they are generated. This can help prevent sensitive information from being shared or ensure outputs meet quality standards. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="OpenAI Agents Output Guardrails Pattern", - tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Output Guardrails Pattern", tags=["agentops-example", "openai-agents"] +) # The agent's output type @@ -540,8 +545,9 @@ async def run_output_guardrails_demo(): # # This pattern shows how to run multiple agents in parallel to improve latency or generate multiple options to choose from. In this example, we run translation agents multiple times and pick the best result. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="OpenAI Agents Parallelization Pattern", - tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Parallelization Pattern", tags=["agentops-example", "openai-agents"] +) # Define agents for parallelization spanish_translation_agent = Agent( @@ -651,8 +657,9 @@ async def run_routing_demo(): # # This pattern shows how to use guardrails during streaming to provide real-time validation. Unlike output guardrails that run after completion, streaming guardrails can interrupt the generation process early. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="OpenAI Agents Streaming Guardrails Pattern", - tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Streaming Guardrails Pattern", tags=["agentops-example", "openai-agents"] +) # Define streaming guardrail agent streaming_agent = Agent( @@ -732,7 +739,7 @@ async def run_streaming_guardrails_demo(): agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/openai_agents/agents_tools.py b/examples/openai_agents/agents_tools.py index 79460903e..4045a135f 100644 --- a/examples/openai_agents/agents_tools.py +++ b/examples/openai_agents/agents_tools.py @@ -249,7 +249,7 @@ async def run_web_search_demo(): agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/openai_agents/customer_service_agent.py b/examples/openai_agents/customer_service_agent.py index 208b789a2..cf7634887 100644 --- a/examples/openai_agents/customer_service_agent.py +++ b/examples/openai_agents/customer_service_agent.py @@ -188,7 +188,7 @@ async def main(): agentops.end_trace(tracer, status="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/smolagents/multi_smolagents_system.py b/examples/smolagents/multi_smolagents_system.py index a09e0a194..02f4e1549 100644 --- a/examples/smolagents/multi_smolagents_system.py +++ b/examples/smolagents/multi_smolagents_system.py @@ -49,7 +49,8 @@ agentops.init(auto_start_session=False) tracer = agentops.start_trace( - trace_name="Smolagents Multi-Agent System Orchestration", tags=["smolagents", "example", "multi-agent", "agentops-example"] + trace_name="Smolagents Multi-Agent System Orchestration", + tags=["smolagents", "example", "multi-agent", "agentops-example"], ) model = LiteLLMModel("openai/gpt-4o-mini") # ## Create a Web Search Tool @@ -113,7 +114,7 @@ def visit_webpage(url: str) -> str: agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/smolagents/text_to_sql.py b/examples/smolagents/text_to_sql.py index a8db8477e..00fcd754d 100644 --- a/examples/smolagents/text_to_sql.py +++ b/examples/smolagents/text_to_sql.py @@ -15,7 +15,6 @@ # %pip install agentops # ## Setting up the SQL Table from sqlalchemy import ( - create_engine, MetaData, Table, @@ -159,7 +158,7 @@ def sql_engine(query: str) -> str: agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/watsonx/watsonx-streaming.py b/examples/watsonx/watsonx-streaming.py index 10167d88b..d11f07ad6 100644 --- a/examples/watsonx/watsonx-streaming.py +++ b/examples/watsonx/watsonx-streaming.py @@ -119,7 +119,7 @@ chat_model.close_persistent_connection() # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) diff --git a/examples/watsonx/watsonx-text-chat.py b/examples/watsonx/watsonx-text-chat.py index dbdd8ba60..ad4fc23ac 100644 --- a/examples/watsonx/watsonx-text-chat.py +++ b/examples/watsonx/watsonx-text-chat.py @@ -78,7 +78,7 @@ chat_model.close_persistent_connection() # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) diff --git a/examples/watsonx/watsonx-tokeniation-model.py b/examples/watsonx/watsonx-tokeniation-model.py index 4a8da8580..f79907330 100644 --- a/examples/watsonx/watsonx-tokeniation-model.py +++ b/examples/watsonx/watsonx-tokeniation-model.py @@ -97,7 +97,7 @@ llama_model.close_persistent_connection() # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=None) diff --git a/examples/xai/grok_examples.py b/examples/xai/grok_examples.py index 6b03eed0c..6c2a287e9 100644 --- a/examples/xai/grok_examples.py +++ b/examples/xai/grok_examples.py @@ -78,7 +78,7 @@ agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/examples/xai/grok_vision_examples.py b/examples/xai/grok_vision_examples.py index 54c778fc4..71f6f5030 100644 --- a/examples/xai/grok_vision_examples.py +++ b/examples/xai/grok_vision_examples.py @@ -19,8 +19,9 @@ # Next we initialize the AgentOps client. agentops.init(auto_start_session=False, tags=["xai", "grok-vision", "agentops-example"]) -tracer = agentops.start_trace(trace_name="XAI Grok Vision Example", tags=[ - "xai-example", "grok-vision", "agentops-example"]) +tracer = agentops.start_trace( + trace_name="XAI Grok Vision Example", tags=["xai-example", "grok-vision", "agentops-example"] +) # And we are all set! Note the seesion url above. We will use it to track the program's performance. # @@ -62,7 +63,7 @@ agentops.end_trace(tracer, end_state="Success") # Let's check programmatically that spans were recorded in AgentOps -print("\n" + "="*50) +print("\n" + "=" * 50) print("Now let's verify that our LLM calls were tracked properly...") try: agentops.validate_trace_spans(trace_context=tracer) diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py index d24811820..ac717358f 100644 --- a/tests/unit/test_validation.py +++ b/tests/unit/test_validation.py @@ -3,13 +3,12 @@ """ import pytest -from unittest.mock import patch, Mock, MagicMock +from unittest.mock import patch, Mock import requests from agentops.validation import ( get_jwt_token, get_trace_details, - get_trace_metrics, check_llm_spans, validate_trace_spans, ValidationError, @@ -21,7 +20,7 @@ class TestGetJwtToken: """Test JWT token exchange functionality.""" - @patch('agentops.validation.requests.post') + @patch("agentops.validation.requests.post") def test_get_jwt_token_success(self, mock_post): """Test successful JWT token retrieval.""" mock_response = Mock() @@ -33,12 +32,10 @@ def test_get_jwt_token_success(self, mock_post): assert token == "test-token" mock_post.assert_called_once_with( - "https://api.agentops.ai/public/v1/auth/access_token", - json={"api_key": "test-api-key"}, - timeout=10 + "https://api.agentops.ai/public/v1/auth/access_token", json={"api_key": "test-api-key"}, timeout=10 ) - @patch('agentops.validation.requests.post') + @patch("agentops.validation.requests.post") def test_get_jwt_token_failure(self, mock_post): """Test JWT token retrieval failure.""" mock_response = Mock() @@ -48,9 +45,9 @@ def test_get_jwt_token_failure(self, mock_post): with pytest.raises(ApiServerException, match="Failed to get JWT token"): get_jwt_token("invalid-api-key") - @patch('os.getenv') - @patch('agentops.get_client') - @patch('agentops.validation.requests.post') + @patch("os.getenv") + @patch("agentops.get_client") + @patch("agentops.validation.requests.post") def test_get_jwt_token_from_env(self, mock_post, mock_get_client, mock_getenv): """Test JWT token retrieval using environment variable.""" mock_get_client.return_value = None @@ -70,15 +67,12 @@ def test_get_jwt_token_from_env(self, mock_post, mock_get_client, mock_getenv): class TestGetTraceDetails: """Test trace details retrieval.""" - @patch('agentops.validation.requests.get') + @patch("agentops.validation.requests.get") def test_get_trace_details_success(self, mock_get): """Test successful trace details retrieval.""" mock_response = Mock() mock_response.status_code = 200 - mock_response.json.return_value = { - "trace_id": "test-trace", - "spans": [{"span_name": "test-span"}] - } + mock_response.json.return_value = {"trace_id": "test-trace", "spans": [{"span_name": "test-span"}]} mock_get.return_value = mock_response details = get_trace_details("test-trace", "test-token") @@ -88,10 +82,10 @@ def test_get_trace_details_success(self, mock_get): mock_get.assert_called_once_with( "https://api.agentops.ai/public/v1/traces/test-trace", headers={"Authorization": "Bearer test-token"}, - timeout=10 + timeout=10, ) - @patch('agentops.validation.requests.get') + @patch("agentops.validation.requests.get") def test_get_trace_details_failure(self, mock_get): """Test trace details retrieval failure.""" mock_response = Mock() @@ -108,23 +102,9 @@ class TestCheckLlmSpans: def test_check_llm_spans_found(self): """Test when LLM spans are found.""" spans = [ - { - "span_name": "OpenAI Chat Completion", - "span_attributes": { - "agentops.span.kind": "llm" - } - }, + {"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}}, {"span_name": "Some other span"}, - { - "span_name": "anthropic.messages.create", - "span_attributes": { - "agentops": { - "span": { - "kind": "llm" - } - } - } - } + {"span_name": "anthropic.messages.create", "span_attributes": {"agentops": {"span": {"kind": "llm"}}}}, ] has_llm, llm_names = check_llm_spans(spans) @@ -135,10 +115,7 @@ def test_check_llm_spans_found(self): def test_check_llm_spans_not_found(self): """Test when no LLM spans are found.""" - spans = [ - {"span_name": "database.query"}, - {"span_name": "http.request"} - ] + spans = [{"span_name": "database.query"}, {"span_name": "http.request"}] has_llm, llm_names = check_llm_spans(spans) assert has_llm is False @@ -157,29 +134,21 @@ def test_check_llm_spans_with_request_type(self): spans = [ { "span_name": "openai.chat.completion", - "span_attributes": { - SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value - } + "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value}, }, { "span_name": "anthropic.messages.create", - "span_attributes": { - SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value - } + "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value}, }, { "span_name": "llm.completion", - "span_attributes": { - SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.COMPLETION.value - } + "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.COMPLETION.value}, }, { "span_name": "embedding.create", - "span_attributes": { - SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.EMBEDDING.value - } + "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.EMBEDDING.value}, }, - {"span_name": "database.query"} + {"span_name": "database.query"}, ] has_llm, llm_names = check_llm_spans(spans) @@ -201,17 +170,17 @@ def test_check_llm_spans_real_world(self): "span_attributes": { SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value, SpanAttributes.LLM_SYSTEM: "OpenAI", - SpanAttributes.LLM_REQUEST_MODEL: "gpt-4" - } + SpanAttributes.LLM_REQUEST_MODEL: "gpt-4", + }, }, { "span_name": "anthropic.messages.create", "span_attributes": { SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value, SpanAttributes.LLM_SYSTEM: "Anthropic", - SpanAttributes.LLM_REQUEST_MODEL: "claude-3-opus-20240229" - } - } + SpanAttributes.LLM_REQUEST_MODEL: "claude-3-opus-20240229", + }, + }, ] has_llm, llm_names = check_llm_spans(spans) @@ -224,27 +193,19 @@ def test_check_llm_spans_real_world(self): class TestValidateTraceSpans: """Test the main validation function.""" - @patch('agentops.validation.get_jwt_token') - @patch('agentops.validation.get_trace_details') - @patch('agentops.validation.get_trace_metrics') + @patch("agentops.validation.get_jwt_token") + @patch("agentops.validation.get_trace_details") + @patch("agentops.validation.get_trace_metrics") def test_validate_trace_spans_success(self, mock_metrics, mock_details, mock_token): """Test successful validation.""" mock_token.return_value = "test-token" mock_details.return_value = { "spans": [ - { - "span_name": "OpenAI Chat Completion", - "span_attributes": { - "agentops.span.kind": "llm" - } - }, - {"span_name": "Other span"} + {"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}}, + {"span_name": "Other span"}, ] } - mock_metrics.return_value = { - "total_tokens": 100, - "total_cost": "0.0025" - } + mock_metrics.return_value = {"total_tokens": 100, "total_cost": "0.0025"} result = validate_trace_spans(trace_id="test-trace") @@ -254,9 +215,9 @@ def test_validate_trace_spans_success(self, mock_metrics, mock_details, mock_tok # LLM activity can be confirmed via metrics or span inspection assert result["metrics"]["total_tokens"] == 100 - @patch('agentops.validation.get_jwt_token') - @patch('agentops.validation.get_trace_details') - @patch('agentops.validation.get_trace_metrics') + @patch("agentops.validation.get_jwt_token") + @patch("agentops.validation.get_trace_details") + @patch("agentops.validation.get_trace_metrics") def test_validate_trace_spans_success_via_metrics(self, mock_metrics, mock_details, mock_token): """Test successful validation when LLM activity is confirmed via metrics.""" mock_token.return_value = "test-token" @@ -264,16 +225,13 @@ def test_validate_trace_spans_success_via_metrics(self, mock_metrics, mock_detai "spans": [ { "span_name": "openai.chat.completion", - "span_attributes": {} # No specific LLM attributes + "span_attributes": {}, # No specific LLM attributes }, - {"span_name": "Other span"} + {"span_name": "Other span"}, ] } # But we have token usage, proving LLM activity - mock_metrics.return_value = { - "total_tokens": 1066, - "total_cost": "0.0006077" - } + mock_metrics.return_value = {"total_tokens": 1066, "total_cost": "0.0006077"} result = validate_trace_spans(trace_id="test-trace") @@ -282,27 +240,22 @@ def test_validate_trace_spans_success_via_metrics(self, mock_metrics, mock_detai assert result["has_llm_spans"] is True # Confirmed via metrics assert result["metrics"]["total_tokens"] == 1066 - @patch('agentops.validation.get_jwt_token') - @patch('agentops.validation.get_trace_details') - @patch('agentops.validation.get_trace_metrics') + @patch("agentops.validation.get_jwt_token") + @patch("agentops.validation.get_trace_details") + @patch("agentops.validation.get_trace_metrics") def test_validate_trace_spans_no_llm(self, mock_metrics, mock_details, mock_token): """Test validation failure when no LLM spans found and no token usage.""" mock_token.return_value = "test-token" - mock_details.return_value = { - "spans": [{"span_name": "database.query"}] - } + mock_details.return_value = {"spans": [{"span_name": "database.query"}]} # No token usage either - mock_metrics.return_value = { - "total_tokens": 0, - "total_cost": "0.0000" - } + mock_metrics.return_value = {"total_tokens": 0, "total_cost": "0.0000"} with pytest.raises(ValidationError, match="No LLM activity detected"): validate_trace_spans(trace_id="test-trace", check_llm=True) - @patch('agentops.validation.get_jwt_token') - @patch('agentops.validation.get_trace_details') - @patch('agentops.validation.get_trace_metrics') + @patch("agentops.validation.get_jwt_token") + @patch("agentops.validation.get_trace_details") + @patch("agentops.validation.get_trace_metrics") def test_validate_trace_spans_retry(self, mock_metrics, mock_details, mock_token): """Test validation with retries.""" mock_token.return_value = "test-token" @@ -311,32 +264,18 @@ def test_validate_trace_spans_retry(self, mock_metrics, mock_details, mock_token mock_details.side_effect = [ {"spans": []}, {"spans": []}, - { - "spans": [{ - "span_name": "OpenAI Chat Completion", - "span_attributes": { - "agentops.span.kind": "llm" - } - }] - } + {"spans": [{"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}}]}, ] # Mock metrics for the successful attempt - mock_metrics.return_value = { - "total_tokens": 100, - "total_cost": "0.0025" - } + mock_metrics.return_value = {"total_tokens": 100, "total_cost": "0.0025"} - result = validate_trace_spans( - trace_id="test-trace", - max_retries=3, - retry_delay=0.01 - ) + result = validate_trace_spans(trace_id="test-trace", max_retries=3, retry_delay=0.01) assert result["span_count"] == 1 assert mock_details.call_count == 3 - @patch('opentelemetry.trace.get_current_span') + @patch("opentelemetry.trace.get_current_span") def test_validate_trace_spans_no_trace_id(self, mock_get_current_span): """Test validation without trace ID.""" # Mock get_current_span to return None @@ -345,10 +284,10 @@ def test_validate_trace_spans_no_trace_id(self, mock_get_current_span): with pytest.raises(ValueError, match="No trace ID found"): validate_trace_spans() - @patch('opentelemetry.trace.get_current_span') - @patch('agentops.validation.get_jwt_token') - @patch('agentops.validation.get_trace_details') - @patch('agentops.validation.get_trace_metrics') + @patch("opentelemetry.trace.get_current_span") + @patch("agentops.validation.get_jwt_token") + @patch("agentops.validation.get_trace_details") + @patch("agentops.validation.get_trace_metrics") def test_validate_trace_spans_from_current_span(self, mock_metrics, mock_details, mock_token, mock_get_span): """Test extracting trace ID from current span.""" # Mock the current span @@ -362,17 +301,9 @@ def test_validate_trace_spans_from_current_span(self, mock_metrics, mock_details mock_token.return_value = "test-token" mock_details.return_value = { - "spans": [{ - "span_name": "OpenAI Chat Completion", - "span_attributes": { - "agentops.span.kind": "llm" - } - }] - } - mock_metrics.return_value = { - "total_tokens": 100, - "total_cost": "0.0025" + "spans": [{"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}}] } + mock_metrics.return_value = {"total_tokens": 100, "total_cost": "0.0025"} result = validate_trace_spans() assert result["trace_id"] == "0000000000000000ab54a98ceb1f0ad2" # hex format of trace ID @@ -387,12 +318,7 @@ def test_print_validation_summary(self, capsys): "span_count": 3, "has_llm_spans": True, "llm_span_names": ["OpenAI Chat", "Claude Message"], - "metrics": { - "total_tokens": 150, - "prompt_tokens": 100, - "completion_tokens": 50, - "total_cost": "0.0030" - } + "metrics": {"total_tokens": 150, "prompt_tokens": 100, "completion_tokens": 50, "total_cost": "0.0030"}, } print_validation_summary(result) @@ -414,8 +340,8 @@ def test_print_validation_summary_metrics_only(self, capsys): "total_tokens": 1066, "prompt_tokens": 800, "completion_tokens": 266, - "total_cost": "0.0006077" - } + "total_cost": "0.0006077", + }, } print_validation_summary(result) @@ -433,12 +359,7 @@ def test_print_validation_summary_llm_prefix(self, capsys): "span_count": 1, "has_llm_spans": True, "llm_span_names": ["openai.chat.completion"], - "metrics": { - "total_tokens": 150, - "prompt_tokens": 100, - "completion_tokens": 50, - "total_cost": "0.0030" - } + "metrics": {"total_tokens": 150, "prompt_tokens": 100, "completion_tokens": 50, "total_cost": "0.0030"}, } print_validation_summary(result) @@ -461,8 +382,8 @@ def test_check_llm_spans_with_llm_prefix(self): "llm.request.type": "chat", "llm.system": "OpenAI", "llm.request.model": "gpt-4", - "llm.usage.total_tokens": 150 - } + "llm.usage.total_tokens": 150, + }, }, { "span_name": "anthropic.messages.create", @@ -470,16 +391,11 @@ def test_check_llm_spans_with_llm_prefix(self): "llm.request.type": "chat", "llm.system": "Anthropic", "llm.request.model": "claude-3-opus", - "llm.usage.total_tokens": 300 - } - }, - { - "span_name": "embedding.create", - "span_attributes": { - "llm.request.type": "embedding" - } + "llm.usage.total_tokens": 300, + }, }, - {"span_name": "database.query"} + {"span_name": "embedding.create", "span_attributes": {"llm.request.type": "embedding"}}, + {"span_name": "database.query"}, ] has_llm, llm_names = check_llm_spans(spans) From 6c5cfd33a989eea811ea681f063fe89eca452536 Mon Sep 17 00:00:00 2001 From: reibs Date: Tue, 8 Jul 2025 00:34:50 -0700 Subject: [PATCH 25/25] add trace naming --- examples/ag2/async_human_input.py | 2 +- examples/ag2/tools_wikipedia_search.py | 2 +- examples/agno/agno_async_operations.py | 2 +- examples/agno/agno_basic_agents.py | 4 +- examples/agno/agno_research_team.py | 2 +- examples/agno/agno_tool_integrations.py | 4 +- examples/agno/agno_workflow_setup.py | 2 +- .../agentops-anthropic-understanding-tools.py | 2 +- examples/anthropic/anthropic-example-async.py | 2 +- examples/anthropic/anthropic-example-sync.py | 2 +- examples/autogen/AgentChat.py | 2 +- examples/autogen/MathAgent.py | 2 +- examples/context_manager/basic_usage.py | 24 ++++++++-- examples/context_manager/error_handling.py | 48 +++++++++++++++---- examples/context_manager/parallel_traces.py | 28 +++++++++-- .../context_manager/production_patterns.py | 24 ++++++++-- examples/crewai/job_posting.py | 4 +- examples/crewai/markdown_validator.py | 2 +- examples/google_genai/gemini_example.py | 2 +- examples/langgraph/langgraph_example.py | 6 ++- examples/litellm/litellm_example.py | 2 +- examples/mem0/mem0_memory_example.py | 2 +- examples/openai/multi_tool_orchestration.py | 6 ++- examples/openai/openai_example_async.py | 2 +- examples/openai/openai_example_sync.py | 2 +- examples/openai/web_search.py | 4 +- examples/openai_agents/agent_guardrails.py | 2 +- examples/openai_agents/agent_patterns.py | 6 ++- examples/openai_agents/agents_tools.py | 6 ++- .../openai_agents/customer_service_agent.py | 6 ++- .../smolagents/multi_smolagents_system.py | 2 +- examples/smolagents/text_to_sql.py | 2 +- examples/watsonx/watsonx-streaming.py | 2 +- examples/watsonx/watsonx-text-chat.py | 2 +- examples/watsonx/watsonx-tokeniation-model.py | 2 +- examples/xai/grok_examples.py | 2 +- examples/xai/grok_vision_examples.py | 4 +- 37 files changed, 166 insertions(+), 54 deletions(-) diff --git a/examples/ag2/async_human_input.py b/examples/ag2/async_human_input.py index 03112ab1a..a534b4758 100644 --- a/examples/ag2/async_human_input.py +++ b/examples/ag2/async_human_input.py @@ -25,7 +25,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="AG2 Async Human Input") tracer = agentops.start_trace( trace_name="AG2 Agent chat with Async Human Inputs", tags=["ag2-chat-async-human-inputs", "agentops-example"] ) diff --git a/examples/ag2/tools_wikipedia_search.py b/examples/ag2/tools_wikipedia_search.py index 028d7139d..c5a3c5e9f 100644 --- a/examples/ag2/tools_wikipedia_search.py +++ b/examples/ag2/tools_wikipedia_search.py @@ -22,7 +22,7 @@ # ### Agent Configuration # # Configure an assistant agent and user proxy to be used for LLM recommendation and execution respectively. -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="AG2 Wikipedia Search Tools") tracer = agentops.start_trace( trace_name="AG2 Wikipedia Search Tools", tags=["ag2-wikipedia-search-tools", "agentops-example"] ) diff --git a/examples/agno/agno_async_operations.py b/examples/agno/agno_async_operations.py index ae6f9cf7f..4ab778b02 100644 --- a/examples/agno/agno_async_operations.py +++ b/examples/agno/agno_async_operations.py @@ -26,7 +26,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_agentops_api_key_here") -agentops.init(auto_start_session=False, tags=["agno-example", "async-operation"]) +agentops.init(auto_start_session=False, trace_name="Agno Async Operations", tags=["agno-example", "async-operation"]) async def demonstrate_async_operations(): diff --git a/examples/agno/agno_basic_agents.py b/examples/agno/agno_basic_agents.py index 2dcfb43ac..25a0cd41f 100644 --- a/examples/agno/agno_basic_agents.py +++ b/examples/agno/agno_basic_agents.py @@ -35,7 +35,9 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_agentops_api_key_here") -agentops.init(auto_start_session=False, tags=["agno-example", "basics", "agents-and-teams"]) +agentops.init( + auto_start_session=False, trace_name="Agno Basic Agents", tags=["agno-example", "basics", "agents-and-teams"] +) def demonstrate_basic_agents(): diff --git a/examples/agno/agno_research_team.py b/examples/agno/agno_research_team.py index 5338922b5..8fbb89a7a 100644 --- a/examples/agno/agno_research_team.py +++ b/examples/agno/agno_research_team.py @@ -67,7 +67,7 @@ load_dotenv() # Initialize AgentOps for monitoring and analytics -agentops.init(auto_start_session=False, tags=["agno-example", "research-team"]) +agentops.init(auto_start_session=False, trace_name="Agno Research Team", tags=["agno-example", "research-team"]) def demonstrate_research_team(): diff --git a/examples/agno/agno_tool_integrations.py b/examples/agno/agno_tool_integrations.py index 709f5e575..859e92f04 100644 --- a/examples/agno/agno_tool_integrations.py +++ b/examples/agno/agno_tool_integrations.py @@ -32,7 +32,9 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_agentops_api_key_here") # Initialize AgentOps -agentops.init(auto_start_session=False, tags=["agno-tools", "tool-integration", "demo"]) +agentops.init( + auto_start_session=False, trace_name="Agno Tool Integrations", tags=["agno-tools", "tool-integration", "demo"] +) def demonstrate_tool_integration(): diff --git a/examples/agno/agno_workflow_setup.py b/examples/agno/agno_workflow_setup.py index bcfd8ad70..42f44d647 100644 --- a/examples/agno/agno_workflow_setup.py +++ b/examples/agno/agno_workflow_setup.py @@ -27,7 +27,7 @@ load_dotenv() -agentops.init(auto_start_session=False, tags=["agno-example", "workflow-setup"]) +agentops.init(auto_start_session=False, trace_name="Agno Workflow Setup", tags=["agno-example", "workflow-setup"]) class CacheWorkflow(Workflow): diff --git a/examples/anthropic/agentops-anthropic-understanding-tools.py b/examples/anthropic/agentops-anthropic-understanding-tools.py index 9a5863c30..ed6898250 100644 --- a/examples/anthropic/agentops-anthropic-understanding-tools.py +++ b/examples/anthropic/agentops-anthropic-understanding-tools.py @@ -17,7 +17,7 @@ os.environ["ANTHROPIC_API_KEY"] = os.getenv("ANTHROPIC_API_KEY", "your_anthropic_api_key_here") # # Now let's set the client as Anthropic and make an AgentOps trace -agentops.init(tags=["anthropic-example-tool-tutorials", "agentops-example"]) +agentops.init(trace_name="Anthropic Understanding Tools", tags=["anthropic-example-tool-tutorials", "agentops-example"]) client = Anthropic() # Now to create a simple dummy tool! We are going to make a tool that will tell us about the demon infestation levels for 3 areas. From there, we will have VEGA, our AI determine the best place for the Doom Slayer to attack. locations = [ diff --git a/examples/anthropic/anthropic-example-async.py b/examples/anthropic/anthropic-example-async.py index e305d652c..26f6b2b65 100644 --- a/examples/anthropic/anthropic-example-async.py +++ b/examples/anthropic/anthropic-example-async.py @@ -22,7 +22,7 @@ # # Now let's set the client as Anthropic and open an agentops trace! client = Anthropic() -agentops.init(tags=["anthropic-async", "agentops-example"]) +agentops.init(trace_name="Anthropic Async Example", tags=["anthropic-async", "agentops-example"]) # Now we create three personality presets; # # Legion is a relentless and heavy-hitting Titan that embodies brute strength and defensive firepower, Northstar is a precise and agile sniper that excels in long-range combat and flight, while Ronin is a swift and aggressive melee specialist who thrives on close-quarters hit-and-run tactics. diff --git a/examples/anthropic/anthropic-example-sync.py b/examples/anthropic/anthropic-example-sync.py index 972024a73..08db9058d 100644 --- a/examples/anthropic/anthropic-example-sync.py +++ b/examples/anthropic/anthropic-example-sync.py @@ -21,7 +21,7 @@ os.environ["ANTHROPIC_API_KEY"] = os.getenv("ANTHROPIC_API_KEY", "your_anthropic_api_key_here") # Now let's set the client as Anthropic and an AgentOps session! client = Anthropic() -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="Anthropic Sync Example") tracer = agentops.start_trace(trace_name="Anthropic Sync Example", tags=["anthropic-example", "agentops-example"]) # Remember that story we made earlier? As of writing, claude-3-5-sonnet-20240620 (the version we will be using) has a 150k word, 680k character length. We also get an 8192 context length. This is great because we can actually set an example for the script! # diff --git a/examples/autogen/AgentChat.py b/examples/autogen/AgentChat.py index d6c5b9938..5958e90d8 100644 --- a/examples/autogen/AgentChat.py +++ b/examples/autogen/AgentChat.py @@ -32,7 +32,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") # When initializing AgentOps, you can pass in optional tags to help filter sessions -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="Autogen Agent Chat Example") tracer = agentops.start_trace( trace_name="Microsoft Agent Chat Example", tags=["autogen-chat", "microsoft-autogen", "agentops-example"] ) diff --git a/examples/autogen/MathAgent.py b/examples/autogen/MathAgent.py index 52be4796b..c594f05c1 100644 --- a/examples/autogen/MathAgent.py +++ b/examples/autogen/MathAgent.py @@ -29,7 +29,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="Autogen Math Agent Example") tracer = agentops.start_trace( trace_name="Microsoft Autogen Tool Example", tags=["autogen-tool", "microsoft-autogen", "agentops-example"] ) diff --git a/examples/context_manager/basic_usage.py b/examples/context_manager/basic_usage.py index 817806acb..94d57c5af 100644 --- a/examples/context_manager/basic_usage.py +++ b/examples/context_manager/basic_usage.py @@ -40,7 +40,11 @@ def basic_context_manager_example(): print("Basic Context Manager Example") # Initialize AgentOps - agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "agentops-example"]) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Basic Example", + tags=["context-manager", "agentops-example"], + ) # Use native TraceContext context manager with agentops.start_trace("Context Manager Basic Example", tags=["basic", "demo"]): @@ -58,7 +62,11 @@ def multiple_parallel_traces(): """Example showing multiple parallel traces.""" print("\nMultiple Parallel Traces") - agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "agentops-example"]) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Multiple Parallel Traces", + tags=["context-manager", "agentops-example"], + ) # First trace with agentops.start_trace("Context Manager Task 1", tags=["parallel", "task-1"]): @@ -81,7 +89,11 @@ def error_handling_example(): """Example showing error handling with context manager.""" print("\nError Handling Example") - agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "agentops-example"]) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Error Handling", + tags=["context-manager", "agentops-example"], + ) try: with agentops.start_trace("error_example", tags=["error-handling"]): @@ -103,7 +115,11 @@ def nested_traces_example(): """Example showing nested traces (which are parallel, not parent-child).""" print("\nNested Traces Example") - agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "agentops-example"]) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Nested Traces", + tags=["context-manager", "agentops-example"], + ) # Outer trace with agentops.start_trace("main_workflow", tags=["workflow", "main"]): diff --git a/examples/context_manager/error_handling.py b/examples/context_manager/error_handling.py index d991dd243..f42d0a8ec 100644 --- a/examples/context_manager/error_handling.py +++ b/examples/context_manager/error_handling.py @@ -66,7 +66,11 @@ def basic_exception_handling(): """Basic example of exception handling with context managers.""" print("Basic Exception Handling") - agentops.init(api_key=AGENTOPS_API_KEY, tags=["context-manager", "error-handling", "agentops-example"]) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Basic Exception Handling", + tags=["context-manager", "error-handling", "agentops-example"], + ) error_types = ["value_error", "type_error", "runtime_error", "success"] @@ -93,7 +97,11 @@ def nested_exception_handling(): """Example of exception handling in nested traces.""" print("\nNested Exception Handling") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Nested Exception Handling", + tags=["context-manager", "error-handling", "agentops-example"], + ) try: with agentops.start_trace("outer_operation", tags=["nested", "outer"]): @@ -124,7 +132,11 @@ def retry_pattern(): """Example of retry pattern with context managers.""" print("\nRetry Pattern") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Retry Pattern", + tags=["context-manager", "error-handling", "agentops-example"], + ) max_retries = 3 for attempt in range(max_retries): @@ -160,7 +172,11 @@ def graceful_degradation(): """Example of graceful degradation pattern.""" print("\nGraceful Degradation") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Graceful Degradation", + tags=["context-manager", "error-handling", "agentops-example"], + ) try: with agentops.start_trace("primary_service", tags=["degradation", "primary"]): @@ -188,7 +204,11 @@ def partial_success_handling(): """Example of partial success handling.""" print("\nPartial Success Handling") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Partial Success Handling", + tags=["context-manager", "error-handling", "agentops-example"], + ) steps = ["step1", "step2", "fail", "step4"] @@ -210,7 +230,11 @@ def custom_exception_handling(): """Example of handling custom exceptions.""" print("\nCustom Exception Handling") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Custom Exception Handling", + tags=["context-manager", "error-handling", "agentops-example"], + ) try: with agentops.start_trace("custom_exception", tags=["custom", "exception"]): @@ -232,7 +256,11 @@ def finally_blocks_example(): """Example of exception handling with finally blocks.""" print("\nFinally Blocks Example") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Finally Blocks", + tags=["context-manager", "error-handling", "agentops-example"], + ) cleanup_actions = [] @@ -263,7 +291,11 @@ def exception_chaining_example(): """Example of exception chaining and context preservation.""" print("\nException Chaining Example") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Exception Chaining", + tags=["context-manager", "error-handling", "agentops-example"], + ) try: with agentops.start_trace("exception_chaining", tags=["chaining", "context"]): diff --git a/examples/context_manager/parallel_traces.py b/examples/context_manager/parallel_traces.py index 1e05a26af..f8436a44a 100644 --- a/examples/context_manager/parallel_traces.py +++ b/examples/context_manager/parallel_traces.py @@ -51,7 +51,11 @@ def sequential_parallel_traces(): """Example of sequential parallel traces - each trace is independent.""" print("Sequential Parallel Traces") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Sequential Parallel Traces", + tags=["context-manager", "agentops-example"], + ) tasks = ["task_1", "task_2", "task_3"] results = [] @@ -75,7 +79,11 @@ def nested_parallel_traces(): """Example showing that nested context managers create parallel traces.""" print("\nNested Parallel Traces") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Nested Parallel Traces", + tags=["context-manager", "agentops-example"], + ) # Outer trace for the overall workflow with agentops.start_trace("workflow_main", tags=["workflow", "main"]): @@ -110,7 +118,11 @@ def concurrent_traces_with_threads(): """Example of truly concurrent traces using threading.""" print("\nConcurrent Traces with Threading") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Concurrent Threads", + tags=["context-manager", "agentops-example"], + ) def worker_function(worker_id: int, task_data: str): """Function to run in a separate thread with its own trace.""" @@ -149,7 +161,11 @@ def concurrent_traces_with_executor(): """Example using ThreadPoolExecutor for concurrent traces.""" print("\nConcurrent Traces with ThreadPoolExecutor") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Concurrent Executor", + tags=["context-manager", "agentops-example"], + ) def process_with_trace(task_id: int, data: str) -> str: """Process data within its own trace context.""" @@ -195,7 +211,9 @@ def trace_with_different_tag_types(): """Example showing different ways to tag parallel traces.""" print("\nTraces with Different Tag Types") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, trace_name="Context Manager Tag Types", tags=["context-manager", "agentops-example"] + ) # Trace with list tags with agentops.start_trace("list_tags_trace", tags=["list", "example", "demo"]): diff --git a/examples/context_manager/production_patterns.py b/examples/context_manager/production_patterns.py index 8ba7c69dd..8cb131f64 100644 --- a/examples/context_manager/production_patterns.py +++ b/examples/context_manager/production_patterns.py @@ -125,7 +125,11 @@ def api_endpoint_pattern(): """Example of using context managers in API endpoint pattern.""" print("API Endpoint Pattern") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager API Endpoint Pattern", + tags=["context-manager", "production", "agentops-example"], + ) # Simulate API requests requests = [ @@ -154,7 +158,11 @@ def batch_processing_pattern(): """Example of batch processing with context managers.""" print("\nBatch Processing Pattern") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Batch Processing Pattern", + tags=["context-manager", "production", "agentops-example"], + ) # Simulate batch data batch_data = [{"id": f"item_{i:03d}", "type": "data_record", "payload": {"value": i * 10}} for i in range(1, 6)] @@ -188,7 +196,11 @@ def microservice_pattern(): """Example of microservice communication pattern.""" print("\nMicroservice Communication Pattern") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Microservice Pattern", + tags=["context-manager", "production", "agentops-example"], + ) def authenticate_user(user_id: str) -> bool: """Simulate authentication service.""" @@ -239,7 +251,11 @@ def monitoring_pattern(): """Example of monitoring with context managers.""" print("\nMonitoring Pattern") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Monitoring Pattern", + tags=["context-manager", "production", "agentops-example"], + ) class AlertManager: """Simple alert manager for demonstration.""" diff --git a/examples/crewai/job_posting.py b/examples/crewai/job_posting.py index 7285e8b20..db0067c2d 100644 --- a/examples/crewai/job_posting.py +++ b/examples/crewai/job_posting.py @@ -20,7 +20,9 @@ os.environ["SERPER_API_KEY"] = os.getenv("SERPER_API_KEY", "your_serper_api_key_here") # Initialize AgentOps client -agentops.init(auto_start_session=False, tags=["crewai", "job-posting", "agentops-example"]) +agentops.init( + auto_start_session=False, trace_name="CrewAI Job Posting", tags=["crewai", "job-posting", "agentops-example"] +) web_search_tool = WebsiteSearchTool() serper_dev_tool = SerperDevTool() diff --git a/examples/crewai/markdown_validator.py b/examples/crewai/markdown_validator.py index 49b59b69c..1d56cb42e 100644 --- a/examples/crewai/markdown_validator.py +++ b/examples/crewai/markdown_validator.py @@ -24,7 +24,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") # The first step in any AgentOps integration is to call `agentops.init()` -agentops.init(tags=["markdown_validator", "agentops-example"]) +agentops.init(trace_name="CrewAI Markdown Validator", tags=["markdown_validator", "agentops-example"]) # Lets start by creating our markdown validator tool diff --git a/examples/google_genai/gemini_example.py b/examples/google_genai/gemini_example.py index 7a53505d5..62e4f019c 100644 --- a/examples/google_genai/gemini_example.py +++ b/examples/google_genai/gemini_example.py @@ -15,7 +15,7 @@ os.environ["GEMINI_API_KEY"] = os.getenv("GEMINI_API_KEY", "your_gemini_api_key_here") # Initialize AgentOps and Gemini client -agentops.init(tags=["gemini-example", "agentops-example"]) +agentops.init(trace_name="Google Gemini Example", tags=["gemini-example", "agentops-example"]) client = genai.Client() # Test synchronous generation diff --git a/examples/langgraph/langgraph_example.py b/examples/langgraph/langgraph_example.py index 6db661793..3a393001a 100644 --- a/examples/langgraph/langgraph_example.py +++ b/examples/langgraph/langgraph_example.py @@ -10,7 +10,11 @@ load_dotenv() -agentops.init(os.getenv("AGENTOPS_API_KEY"), tags=["langgraph", "tool-usage", "agentops-example"]) +agentops.init( + os.getenv("AGENTOPS_API_KEY"), + trace_name="LangGraph Tool Usage Example", + tags=["langgraph", "tool-usage", "agentops-example"], +) @tool diff --git a/examples/litellm/litellm_example.py b/examples/litellm/litellm_example.py index 0a226200a..2e4d51021 100644 --- a/examples/litellm/litellm_example.py +++ b/examples/litellm/litellm_example.py @@ -28,7 +28,7 @@ "OPENAI_API_KEY", "your_openai_api_key_here" ) # or the provider of your choosing -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="LiteLLM Example") tracer = agentops.start_trace(trace_name="LiteLLM Example", tags=["litellm-example", "agentops-example"]) # Note: AgentOps requires that you call LiteLLM completions differently than the LiteLLM's docs mention diff --git a/examples/mem0/mem0_memory_example.py b/examples/mem0/mem0_memory_example.py index c8f8da7c4..6f59f07e0 100644 --- a/examples/mem0/mem0_memory_example.py +++ b/examples/mem0/mem0_memory_example.py @@ -186,7 +186,7 @@ async def search_memory(query): # Initialize AgentOps -agentops.init(tags=["mem0", "memory-management", "agentops-example"]) +agentops.init(trace_name="Mem0 Memory Example", tags=["mem0", "memory-management", "agentops-example"]) # Configuration for local memory (Memory) # This configuration specifies the LLM provider and model settings diff --git a/examples/openai/multi_tool_orchestration.py b/examples/openai/multi_tool_orchestration.py index 0b7fed626..f3586a755 100644 --- a/examples/openai/multi_tool_orchestration.py +++ b/examples/openai/multi_tool_orchestration.py @@ -28,7 +28,11 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") os.environ["PINECONE_API_KEY"] = os.getenv("PINECONE_API_KEY", "your_pinecone_api_key_here") -agentops.init(auto_start_session=True, tags=["openai", "multi-tool", "agentops-example"]) +agentops.init( + auto_start_session=True, + trace_name="OpenAI Multi-Tool Orchestration", + tags=["openai", "multi-tool", "agentops-example"], +) tracer = agentops.start_trace( trace_name="OpenAI Multi-Tool Orchestration with RAG", tags=["multi-tool-orchestration-rag-demo", "openai-responses", "agentops-example"], diff --git a/examples/openai/openai_example_async.py b/examples/openai/openai_example_async.py index 43d5e4f51..0602f72f7 100644 --- a/examples/openai/openai_example_async.py +++ b/examples/openai/openai_example_async.py @@ -21,7 +21,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=True, tags=["openai", "async", "agentops-example"]) +agentops.init(auto_start_session=True, trace_name="OpenAI Async Example", tags=["openai", "async", "agentops-example"]) tracer = agentops.start_trace( trace_name="OpenAI Async Example", tags=["openai-async-example", "openai", "agentops-example"] ) diff --git a/examples/openai/openai_example_sync.py b/examples/openai/openai_example_sync.py index 37ff5249f..9c7f49cde 100644 --- a/examples/openai/openai_example_sync.py +++ b/examples/openai/openai_example_sync.py @@ -20,7 +20,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=True, tags=["openai", "sync", "agentops-example"]) +agentops.init(auto_start_session=True, trace_name="OpenAI Sync Example", tags=["openai", "sync", "agentops-example"]) tracer = agentops.start_trace( trace_name="OpenAI Sync Example", tags=["openai-sync-example", "openai", "agentops-example"] ) diff --git a/examples/openai/web_search.py b/examples/openai/web_search.py index 1e90f596b..627d79ced 100644 --- a/examples/openai/web_search.py +++ b/examples/openai/web_search.py @@ -26,7 +26,9 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") -agentops.init(auto_start_session=True, tags=["openai", "web-search", "agentops-example"]) +agentops.init( + auto_start_session=True, trace_name="OpenAI Web Search Example", tags=["openai", "web-search", "agentops-example"] +) tracer = agentops.start_trace( trace_name="OpenAI Responses Example", tags=["openai-responses-example", "openai", "agentops-example"] ) diff --git a/examples/openai_agents/agent_guardrails.py b/examples/openai_agents/agent_guardrails.py index 1296245b6..ebfa32ac9 100644 --- a/examples/openai_agents/agent_guardrails.py +++ b/examples/openai_agents/agent_guardrails.py @@ -33,7 +33,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(api_key=os.environ["AGENTOPS_API_KEY"], tags=["agentops-example"]) +agentops.init(api_key=os.environ["AGENTOPS_API_KEY"], trace_name="OpenAI Agents Guardrails", tags=["agentops-example"]) # OpenAI Agents SDK guardrail example with agentops guardrails decorator for observability diff --git a/examples/openai_agents/agent_patterns.py b/examples/openai_agents/agent_patterns.py index db51e513e..3e6af9026 100644 --- a/examples/openai_agents/agent_patterns.py +++ b/examples/openai_agents/agent_patterns.py @@ -78,7 +78,11 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") # Initialize AgentOps -agentops.init(auto_start_session=False, tags=["openai-agents", "patterns", "agentops-example"]) +agentops.init( + auto_start_session=False, + trace_name="OpenAI Agents Patterns", + tags=["openai-agents", "patterns", "agentops-example"], +) # Note: tracer will be defined in each section's cell for clarity, using the specific tags for that pattern. # ## 1. Agents as Tools Pattern # diff --git a/examples/openai_agents/agents_tools.py b/examples/openai_agents/agents_tools.py index 4045a135f..ed01e7520 100644 --- a/examples/openai_agents/agents_tools.py +++ b/examples/openai_agents/agents_tools.py @@ -48,7 +48,11 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False, tags=["openai-agents", "tools", "agentops-example"]) +agentops.init( + auto_start_session=False, + trace_name="OpenAI Agents Tools Examples", + tags=["openai-agents", "tools", "agentops-example"], +) # ## 1. Code Interpreter Tool # diff --git a/examples/openai_agents/customer_service_agent.py b/examples/openai_agents/customer_service_agent.py index cf7634887..dffad335f 100644 --- a/examples/openai_agents/customer_service_agent.py +++ b/examples/openai_agents/customer_service_agent.py @@ -53,7 +53,11 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(tags=["customer-service-agent", "openai-agents", "agentops-example"], auto_start_session=False) +agentops.init( + trace_name="OpenAI Agents Customer Service", + tags=["customer-service-agent", "openai-agents", "agentops-example"], + auto_start_session=False, +) tracer = agentops.start_trace(trace_name="OpenAI Agents Customer Service Agent") diff --git a/examples/smolagents/multi_smolagents_system.py b/examples/smolagents/multi_smolagents_system.py index 02f4e1549..acbb6965a 100644 --- a/examples/smolagents/multi_smolagents_system.py +++ b/examples/smolagents/multi_smolagents_system.py @@ -47,7 +47,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="Smolagents Multi-Agent System") tracer = agentops.start_trace( trace_name="Smolagents Multi-Agent System Orchestration", tags=["smolagents", "example", "multi-agent", "agentops-example"], diff --git a/examples/smolagents/text_to_sql.py b/examples/smolagents/text_to_sql.py index 00fcd754d..32282b6f8 100644 --- a/examples/smolagents/text_to_sql.py +++ b/examples/smolagents/text_to_sql.py @@ -98,7 +98,7 @@ def sql_engine(query: str) -> str: os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="Smolagents Text-to-SQL") tracer = agentops.start_trace( trace_name="Smolagents Text-to-SQL", tags=["smolagents", "example", "text-to-sql", "agentops-example"] ) diff --git a/examples/watsonx/watsonx-streaming.py b/examples/watsonx/watsonx-streaming.py index d11f07ad6..c62d27327 100644 --- a/examples/watsonx/watsonx-streaming.py +++ b/examples/watsonx/watsonx-streaming.py @@ -15,7 +15,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Initialize AgentOps -agentops.init(tags=["watsonx-streaming", "agentops-example"]) +agentops.init(trace_name="WatsonX Streaming Example", tags=["watsonx-streaming", "agentops-example"]) # ## Initialize IBM Watsonx AI Credentials # # To use IBM Watsonx AI, you need to set up your credentials and project ID. diff --git a/examples/watsonx/watsonx-text-chat.py b/examples/watsonx/watsonx-text-chat.py index ad4fc23ac..af49599a6 100644 --- a/examples/watsonx/watsonx-text-chat.py +++ b/examples/watsonx/watsonx-text-chat.py @@ -15,7 +15,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Initialize AgentOps -agentops.init(tags=["watsonx-text-chat", "agentops-example"]) +agentops.init(trace_name="WatsonX Text Chat Example", tags=["watsonx-text-chat", "agentops-example"]) # ## Initialize IBM Watsonx AI Credentials # # To use IBM Watsonx AI, you need to set up your credentials and project ID. diff --git a/examples/watsonx/watsonx-tokeniation-model.py b/examples/watsonx/watsonx-tokeniation-model.py index f79907330..4d039e528 100644 --- a/examples/watsonx/watsonx-tokeniation-model.py +++ b/examples/watsonx/watsonx-tokeniation-model.py @@ -15,7 +15,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Initialize AgentOps -agentops.init(tags=["watsonx-tokenization", "agentops-example"]) +agentops.init(trace_name="WatsonX Tokenization Model Example", tags=["watsonx-tokenization", "agentops-example"]) # ## Initialize IBM Watsonx AI Credentials # # To use IBM Watsonx AI, you need to set up your credentials and project ID. diff --git a/examples/xai/grok_examples.py b/examples/xai/grok_examples.py index 6c2a287e9..860f93b05 100644 --- a/examples/xai/grok_examples.py +++ b/examples/xai/grok_examples.py @@ -17,7 +17,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=False, tags=["xai", "grok", "agentops-example"]) +agentops.init(auto_start_session=False, trace_name="XAI Grok Example", tags=["xai", "grok", "agentops-example"]) tracer = agentops.start_trace(trace_name="XAI Grok Example", tags=["xai-example", "grok", "agentops-example"]) # And we are all set! Note the seesion url above. We will use it to track the chatbot. diff --git a/examples/xai/grok_vision_examples.py b/examples/xai/grok_vision_examples.py index 71f6f5030..151c3bdf1 100644 --- a/examples/xai/grok_vision_examples.py +++ b/examples/xai/grok_vision_examples.py @@ -18,7 +18,9 @@ os.environ["XAI_API_KEY"] = os.getenv("XAI_API_KEY", "your_xai_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=False, tags=["xai", "grok-vision", "agentops-example"]) +agentops.init( + auto_start_session=False, trace_name="XAI Grok Vision Example", tags=["xai", "grok-vision", "agentops-example"] +) tracer = agentops.start_trace( trace_name="XAI Grok Vision Example", tags=["xai-example", "grok-vision", "agentops-example"] )