diff --git a/tests/unit/vertexai/genai/replays/test_evaluate_instances.py b/tests/unit/vertexai/genai/replays/test_evaluate_instances.py index cf51bedf41..5e86485453 100644 --- a/tests/unit/vertexai/genai/replays/test_evaluate_instances.py +++ b/tests/unit/vertexai/genai/replays/test_evaluate_instances.py @@ -81,13 +81,19 @@ def test_rouge_metric(client): def test_pointwise_metric(client): """Tests the _evaluate_instances method with PointwiseMetricInput.""" - instance_dict = {"prompt": "What is the capital of France?", "response": "Paris"} + instance_dict = { + "prompt": "What is the capital of France?", + "response": "Paris", + } json_instance = json.dumps(instance_dict) test_input = types.PointwiseMetricInput( instance=types.PointwiseMetricInstance(json_instance=json_instance), metric_spec=genai_types.PointwiseMetricSpec( - metric_prompt_template="Evaluate if the response '{response}' correctly answers the prompt '{prompt}'." + metric_prompt_template=( + "Evaluate if the response '{response}' correctly answers the" + " prompt '{prompt}'." + ) ), ) response = client.evals.evaluate_instances( @@ -101,19 +107,20 @@ def test_pointwise_metric(client): def test_pointwise_metric_with_agent_data(client): """Tests the _evaluate_instances method with PointwiseMetricInput and agent_data.""" - instance_dict = {"prompt": "What is the capital of France?", "response": "Paris"} + instance_dict = { + "prompt": "What is the capital of France?", + "response": "Paris", + } json_instance = json.dumps(instance_dict) agent_data = types.evals.AgentData( agent_config=types.evals.AgentConfig( - tools=types.evals.Tools( - tool=[ - genai_types.Tool( - function_declarations=[ - genai_types.FunctionDeclaration(name="search") - ] - ) - ] - ), + tools=[ + genai_types.Tool( + function_declarations=[ + genai_types.FunctionDeclaration(name="search") + ] + ) + ], developer_instruction=types.evals.InstanceData(text="instruction"), ), events=types.evals.Events( @@ -129,7 +136,10 @@ def test_pointwise_metric_with_agent_data(client): test_input = types.PointwiseMetricInput( instance=types.PointwiseMetricInstance(json_instance=json_instance), metric_spec=genai_types.PointwiseMetricSpec( - metric_prompt_template="Evaluate if the response '{response}' correctly answers the prompt '{prompt}'." + metric_prompt_template=( + "Evaluate if the response '{response}' correctly answers the" + " prompt '{prompt}'." + ) ), ) response = client.evals.evaluate_instances( @@ -189,7 +199,10 @@ def test_pairwise_metric_with_autorater(client): test_input = types.PairwiseMetricInput( instance=types.PairwiseMetricInstance(json_instance=json_instance), metric_spec=genai_types.PairwiseMetricSpec( - metric_prompt_template="Which response is a better summary? Baseline: '{baseline_response}' or Candidate: '{candidate_response}'" + metric_prompt_template=( + "Which response is a better summary? Baseline:" + " '{baseline_response}' or Candidate: '{candidate_response}'" + ) ), ) autorater_config = genai_types.AutoraterConfig(sampling_count=2) @@ -240,7 +253,10 @@ def test_inference_with_prompt_template(client): def test_run_inference_with_agent(client): test_df = pd.DataFrame( - {"prompt": ["agent prompt"], "session_inputs": ['{"user_id": "user_123"}']} + { + "prompt": ["agent prompt"], + "session_inputs": ['{"user_id": "user_123"}'], + } ) inference_result = client.evals.run_inference( agent="projects/977012026409/locations/us-central1/reasoningEngines/7188347537655332864", diff --git a/tests/unit/vertexai/genai/test_evals.py b/tests/unit/vertexai/genai/test_evals.py index a9b62bebfe..7c4ed635f5 100644 --- a/tests/unit/vertexai/genai/test_evals.py +++ b/tests/unit/vertexai/genai/test_evals.py @@ -4056,7 +4056,7 @@ def test_eval_case_to_agent_data(self): ) assert agent_data.agent_config.developer_instruction.text == "instruction1" - assert agent_data.agent_config.tools.tool == [tool] + assert agent_data.agent_config.legacy_tools.tool == [tool] assert agent_data.events.event[0].parts[0].text == "intermediate event" def test_eval_case_to_agent_data_events_only(self): @@ -4164,7 +4164,7 @@ def test_eval_case_to_agent_data_agent_info_empty_tools(self): ) assert agent_data.agent_config.developer_instruction.text == "instruction1" - assert not agent_data.agent_config.tools.tool + assert not agent_data.agent_config.legacy_tools.tool def test_eval_case_to_agent_data_agent_info_empty(self): intermediate_events = [ diff --git a/vertexai/_genai/_evals_common.py b/vertexai/_genai/_evals_common.py index f33320324a..0e70a0a4c1 100644 --- a/vertexai/_genai/_evals_common.py +++ b/vertexai/_genai/_evals_common.py @@ -1299,21 +1299,36 @@ def _run_agent_internal( agent=agent, prompt_dataset=prompt_dataset, ) + + agent_obj = agent_engine if agent_engine else agent + processed_intermediate_events = [] processed_responses = [] - for resp_item in raw_responses: + processed_agent_data = [] # New column for AgentData + + for i, resp_item in enumerate(raw_responses): intermediate_events_row: list[dict[str, Any]] = [] response_row = None + + # --- Legacy Logic: Intermediate Events & Response --- if isinstance(resp_item, list): try: - response_row = resp_item[-1]["content"]["parts"][0]["text"] + # Attempt to extract final response text + if resp_item and "content" in resp_item[-1]: + # Basic extraction, assumes last message is model response + final_content = resp_item[-1]["content"] + if isinstance(final_content, dict) and "parts" in final_content: + response_row = final_content["parts"][0].get("text", "") + elif hasattr(final_content, "parts"): + response_row = final_content.parts[0].text + for intermediate_event in resp_item[:-1]: intermediate_events_row.append( { - "event_id": intermediate_event["id"], - "content": intermediate_event["content"], - "creation_timestamp": intermediate_event["timestamp"], - "author": intermediate_event["author"], + "event_id": intermediate_event.get("id"), + "content": intermediate_event.get("content"), + "creation_timestamp": intermediate_event.get("timestamp"), + "author": intermediate_event.get("author"), } ) except Exception as e: # pylint: disable=broad-exception-caught @@ -1335,6 +1350,33 @@ def _run_agent_internal( processed_intermediate_events.append(intermediate_events_row) processed_responses.append(response_row) + # --- New Logic: AgentData --- + agent_data_obj = None + try: + # 1. Get User Prompt for the current row + primary_prompt_column = ( + "request" if "request" in prompt_dataset.columns else "prompt" + ) + user_prompt_val = prompt_dataset.iloc[i][primary_prompt_column] + + # 2. Construct Full Session History (User Prompt + Agent Events) + # Normalize user prompt into a message dict structure + user_event = {"role": "user", "content": user_prompt_val} + + full_session_history = [user_event] + if isinstance(resp_item, list): + full_session_history.extend(resp_item) + + # 3. Create AgentData using the new factory method + agent_data_obj = types.evals.AgentData.from_session( + agent_obj, full_session_history + ) + except Exception as e: + logger.warning("Failed to adapt AgentData for row %d: %s", i, e) + # Proceed without AgentData; backend will fallback to legacy fields + + processed_agent_data.append(agent_data_obj) + if len(processed_responses) != len(prompt_dataset) or len( processed_responses ) != len(processed_intermediate_events): @@ -1353,6 +1395,7 @@ def _run_agent_internal( { _evals_constant.INTERMEDIATE_EVENTS: processed_intermediate_events, _evals_constant.RESPONSE: processed_responses, + "agent_data": processed_agent_data, # Populate agent_data } ) diff --git a/vertexai/_genai/_evals_metric_handlers.py b/vertexai/_genai/_evals_metric_handlers.py index 9f4494c74d..4b40a37e8e 100644 --- a/vertexai/_genai/_evals_metric_handlers.py +++ b/vertexai/_genai/_evals_metric_handlers.py @@ -879,6 +879,11 @@ def _eval_case_to_agent_data( eval_case: types.EvalCase, ) -> Optional[types.evals.AgentData]: """Converts an EvalCase object to an AgentData object.""" + # --- NEW LOGIC: Use the structured agent_data if present --- + if getattr(eval_case, "agent_data", None): + return eval_case.agent_data + + # --- LEGACY LOGIC: Fallback for older dataframes --- if not eval_case.agent_info and not eval_case.intermediate_events: return None tools = None @@ -899,7 +904,7 @@ def _eval_case_to_agent_data( if tools or developer_instruction: agent_config = types.evals.AgentConfig( - tools=tools, + legacy_tools=tools, developer_instruction=developer_instruction, ) diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index cbe817f42e..2ec662eded 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -1492,6 +1492,10 @@ class EvalCase(_common.BaseModel): default=None, description="""This field is experimental and may change in future versions. The agent info of the agent under evaluation. This can be extended for multi-agent evaluation.""", ) + agent_data: Optional[evals_types.AgentData] = Field( + default=None, + description="""This field is experimental and may change in future versions. The agent data of the agent under evaluation.""", + ) # Allow extra fields to support custom metric prompts and stay backward compatible. model_config = ConfigDict(frozen=True, extra="allow") @@ -1526,6 +1530,9 @@ class EvalCaseDict(TypedDict, total=False): agent_info: Optional[evals_types.AgentInfo] """This field is experimental and may change in future versions. The agent info of the agent under evaluation. This can be extended for multi-agent evaluation.""" + agent_data: Optional[evals_types.AgentData] + """This field is experimental and may change in future versions. The agent data of the agent under evaluation.""" + EvalCaseOrDict = Union[EvalCase, EvalCaseDict] diff --git a/vertexai/_genai/types/evals.py b/vertexai/_genai/types/evals.py index ce554d3110..c74a7d557c 100644 --- a/vertexai/_genai/types/evals.py +++ b/vertexai/_genai/types/evals.py @@ -36,6 +36,458 @@ class Importance(_common.CaseInSensitiveEnum): """Low importance.""" +class Tools(_common.BaseModel): + """This field is experimental and will be removed in future versions. + + Represents a list of tools for an agent. + """ + + tool: Optional[list[genai_types.Tool]] = Field( + default=None, + description="""List of tools: each tool can have multiple function declarations.""", + ) + + +class ToolsDict(TypedDict, total=False): + """This field is experimental and will be removed in future versions. + + Represents a list of tools for an agent. + """ + + tool: Optional[list[genai_types.ToolDict]] + """List of tools: each tool can have multiple function declarations.""" + + +ToolsOrDict = Union[Tools, ToolsDict] + + +class InstanceDataContents(_common.BaseModel): + """This field is experimental and will be removed in future versions. + + List of standard Content messages from Gemini API. + """ + + contents: Optional[list[genai_types.Content]] = Field( + default=None, description="""Repeated contents.""" + ) + + +class InstanceDataContentsDict(TypedDict, total=False): + """This field is experimental and will be removed in future versions. + + List of standard Content messages from Gemini API. + """ + + contents: Optional[list[genai_types.ContentDict]] + """Repeated contents.""" + + +InstanceDataContentsOrDict = Union[InstanceDataContents, InstanceDataContentsDict] + + +class InstanceData(_common.BaseModel): + """This field is experimental and will be removed in future versions. + + Instance data used to populate placeholders in a metric prompt template. + """ + + text: Optional[str] = Field(default=None, description="""Text data.""") + contents: Optional[InstanceDataContents] = Field( + default=None, description="""List of Gemini content data.""" + ) + + +class InstanceDataDict(TypedDict, total=False): + """This field is experimental and will be removed in future versions. + + Instance data used to populate placeholders in a metric prompt template. + """ + + text: Optional[str] + """Text data.""" + + contents: Optional[InstanceDataContentsDict] + """List of Gemini content data.""" + + +InstanceDataOrDict = Union[InstanceData, InstanceDataDict] + + +class AgentConfig(_common.BaseModel): + """Represents configuration for an Agent.""" + + agent_id: Optional[str] = Field( + default=None, + description="""Unique identifier of the agent. + This ID is used to refer to this agent, e.g., in AgentEvent.author, or in + the `sub_agents` field. It must be unique within the `agents` map.""", + ) + agent_type: Optional[str] = Field( + default=None, + description="""The type or class of the agent (e.g., "LlmAgent", "RouterAgent", + "ToolUseAgent"). Useful for the autorater to understand the expected + behavior of the agent.""", + ) + description: Optional[str] = Field( + default=None, + description="""A high-level description of the agent's role and responsibilities. + Critical for evaluating if the agent is routing tasks correctly.""", + ) + instruction: Optional[str] = Field( + default=None, + description="""The instructions for the LLM model, guiding the agent's behavior. + Can be static or dynamic. Dynamic instructions can contain placeholders + like {variable_name} that will be resolved at runtime using the + `AgentEvent.state_delta` field.""", + ) + tools: Optional[list[genai_types.Tool]] = Field( + default=None, description="""The list of tools available to this agent.""" + ) + sub_agents: Optional[list[str]] = Field( + default=None, + description="""The list of valid agent IDs that this agent can delegate to. + This defines the directed edges in the multi-agent system graph topology.""", + ) + tools_text: Optional[str] = Field( + default=None, + description="""A JSON string containing a list of tools available to an agent.""", + ) + legacy_tools: Optional[Tools] = Field( + default=None, description="""List of tools.""" + ) + developer_instruction: Optional[InstanceData] = Field( + default=None, + description="""A field containing instructions from the developer for the agent.""", + ) + + @classmethod + def from_agent(cls, agent: Any) -> "AgentConfig": + """Creates an AgentConfig from an agent object. + + Args: + agent: The agent object (e.g., LlmAgent or AgentEngine). + + Returns: + An AgentConfig object populated with the agent's metadata. + """ + agent_id = getattr(agent, "name", "default_agent") or "default_agent" + if not isinstance(agent_id, str): + agent_id = "default_agent" + + description = getattr(agent, "description", None) + instruction = getattr(agent, "instruction", None) + + tools = [] + if hasattr(agent, "tools") and agent.tools: + for tool in agent.tools: + try: + if callable(tool): + fd = genai_types.FunctionDeclaration.from_callable( + callable=tool, client=None + ) + tools.append(genai_types.Tool(function_declarations=[fd])) + elif isinstance(tool, genai_types.Tool): + tools.append(tool) + except Exception: + pass # Skip tools that can't be converted + + return cls( # pytype: disable=missing-parameter + agent_id=agent_id, + agent_type=agent.__class__.__name__, + description=description, + instruction=instruction, + tools=tools if tools else None, + ) + + +class AgentConfigDict(TypedDict, total=False): + """Represents configuration for an Agent.""" + + agent_id: Optional[str] + """Unique identifier of the agent. + This ID is used to refer to this agent, e.g., in AgentEvent.author, or in + the `sub_agents` field. It must be unique within the `agents` map.""" + + agent_type: Optional[str] + """The type or class of the agent (e.g., "LlmAgent", "RouterAgent", + "ToolUseAgent"). Useful for the autorater to understand the expected + behavior of the agent.""" + + description: Optional[str] + """A high-level description of the agent's role and responsibilities. + Critical for evaluating if the agent is routing tasks correctly.""" + + instruction: Optional[str] + """The instructions for the LLM model, guiding the agent's behavior. + Can be static or dynamic. Dynamic instructions can contain placeholders + like {variable_name} that will be resolved at runtime using the + `AgentEvent.state_delta` field.""" + + tools: Optional[list[genai_types.ToolDict]] + """The list of tools available to this agent.""" + + sub_agents: Optional[list[str]] + """The list of valid agent IDs that this agent can delegate to. + This defines the directed edges in the multi-agent system graph topology.""" + + tools_text: Optional[str] + """A JSON string containing a list of tools available to an agent.""" + + legacy_tools: Optional[ToolsDict] + """List of tools.""" + + developer_instruction: Optional[InstanceDataDict] + """A field containing instructions from the developer for the agent.""" + + +AgentConfigOrDict = Union[AgentConfig, AgentConfigDict] + + +class AgentEvent(_common.BaseModel): + """A single event in the execution trace.""" + + author: Optional[str] = Field( + default=None, + description="""The ID of the agent or entity that generated this event. + Use "user" to denote events generated by the end-user.""", + ) + content: Optional[genai_types.Content] = Field( + default=None, description="""The content of the event.""" + ) + event_time: Optional[datetime.datetime] = Field( + default=None, description="""The timestamp when the event occurred.""" + ) + state_delta: Optional[dict[str, Any]] = Field( + default=None, + description="""The change in the session state caused by this event. + This is a key-value map of fields that were modified or added by the event.""", + ) + active_tools: Optional[list[genai_types.Tool]] = Field( + default=None, + description="""The list of tools that were active/available to the agent at the + time of this event. This overrides the `AgentConfig.tools` if set.""", + ) + + +class AgentEventDict(TypedDict, total=False): + """A single event in the execution trace.""" + + author: Optional[str] + """The ID of the agent or entity that generated this event. + Use "user" to denote events generated by the end-user.""" + + content: Optional[genai_types.ContentDict] + """The content of the event.""" + + event_time: Optional[datetime.datetime] + """The timestamp when the event occurred.""" + + state_delta: Optional[dict[str, Any]] + """The change in the session state caused by this event. + This is a key-value map of fields that were modified or added by the event.""" + + active_tools: Optional[list[genai_types.ToolDict]] + """The list of tools that were active/available to the agent at the + time of this event. This overrides the `AgentConfig.tools` if set.""" + + +AgentEventOrDict = Union[AgentEvent, AgentEventDict] + + +class ConversationTurn(_common.BaseModel): + """Represents a single turn/invocation in the conversation.""" + + turn_index: Optional[int] = Field( + default=None, + description="""The 0-based index of the turn in the conversation sequence.""", + ) + turn_id: Optional[str] = Field( + default=None, description="""A unique identifier for the turn.""" + ) + events: Optional[list[AgentEvent]] = Field( + default=None, + description="""The list of events that occurred during this turn.""", + ) + + +class ConversationTurnDict(TypedDict, total=False): + """Represents a single turn/invocation in the conversation.""" + + turn_index: Optional[int] + """The 0-based index of the turn in the conversation sequence.""" + + turn_id: Optional[str] + """A unique identifier for the turn.""" + + events: Optional[list[AgentEventDict]] + """The list of events that occurred during this turn.""" + + +ConversationTurnOrDict = Union[ConversationTurn, ConversationTurnDict] + + +class Events(_common.BaseModel): + """This field is experimental and will be removed in future versions. + + Represents a list of events for an agent. + """ + + event: Optional[list[genai_types.Content]] = Field( + default=None, description="""A list of events.""" + ) + + +class EventsDict(TypedDict, total=False): + """This field is experimental and will be removed in future versions. + + Represents a list of events for an agent. + """ + + event: Optional[list[genai_types.ContentDict]] + """A list of events.""" + + +EventsOrDict = Union[Events, EventsDict] + + +class AgentData(_common.BaseModel): + """Represents data specific to multi-turn agent evaluations.""" + + agents: Optional[dict[str, AgentConfig]] = Field( + default=None, + description="""A map containing the static configurations for each agent in the system. + Key: agent_id (matches the `author` field in events). + Value: The static configuration of the agent.""", + ) + turns: Optional[list[ConversationTurn]] = Field( + default=None, + description="""A chronological list of conversation turns. + Each turn represents a logical execution cycle (e.g., User Input -> Agent + Response).""", + ) + agent_config: Optional[AgentConfig] = Field( + default=None, description="""Agent configuration.""" + ) + events_text: Optional[str] = Field( + default=None, description="""A JSON string containing a sequence of events.""" + ) + events: Optional[Events] = Field(default=None, description="""A list of events.""") + + @classmethod + def from_session(cls, agent: Any, session_history: list[Any]) -> "AgentData": + """Creates an AgentData object from a session history. + + Segments the flat list of session events into ConversationTurns. A new turn + is initiated by a User message. + + Args: + agent: The agent instance used in the session. + session_history: A list of raw events/messages from the session. + + Returns: + An AgentData object containing the segmented history and agent config. + """ + agent_config = AgentConfig.from_agent(agent) + agent_id = agent_config.agent_id or "default_agent" + agents_map = {agent_id: agent_config} + + turns = [] + current_turn_events = [] + + for event in session_history: + is_user = False + if isinstance(event, dict): + if event.get("role") == "user": + is_user = True + elif ( + isinstance(event.get("content"), dict) + and event["content"].get("role") == "user" + ): + is_user = True + elif hasattr(event, "role") and event.role == "user": + is_user = True + + if is_user and current_turn_events: + turns.append( + ConversationTurn( # pytype: disable=missing-parameter + turn_index=len(turns), + turn_id=f"turn_{len(turns)}", + events=current_turn_events, + ) + ) + current_turn_events = [] + + author = "user" if is_user else agent_id + + content = None + if isinstance(event, dict): + if "content" in event: + raw_content = event["content"] + if isinstance(raw_content, genai_types.Content): + content = raw_content + elif isinstance(raw_content, dict): + try: + content = genai_types.Content.model_validate(raw_content) + except Exception: + pass + elif isinstance(raw_content, str): + content = genai_types.Content( + parts=[genai_types.Part(text=raw_content)] + ) + elif "parts" in event: + try: + content = genai_types.Content.model_validate(event) + except Exception: + pass + elif hasattr(event, "content") and isinstance( + event.content, genai_types.Content + ): + content = event.content + + agent_event = AgentEvent( # pytype: disable=missing-parameter + author=author, + content=content, + ) + current_turn_events.append(agent_event) + + if current_turn_events: + turns.append( + ConversationTurn( # pytype: disable=missing-parameter + turn_index=len(turns), + turn_id=f"turn_{len(turns)}", + events=current_turn_events, + ) + ) + + return cls(agents=agents_map, turns=turns) # pytype: disable=missing-parameter + + +class AgentDataDict(TypedDict, total=False): + """Represents data specific to multi-turn agent evaluations.""" + + agents: Optional[dict[str, AgentConfigDict]] + """A map containing the static configurations for each agent in the system. + Key: agent_id (matches the `author` field in events). + Value: The static configuration of the agent.""" + + turns: Optional[list[ConversationTurnDict]] + """A chronological list of conversation turns. + Each turn represents a logical execution cycle (e.g., User Input -> Agent + Response).""" + + agent_config: Optional[AgentConfigDict] + """Agent configuration.""" + + events_text: Optional[str] + """A JSON string containing a sequence of events.""" + + events: Optional[EventsDict] + """A list of events.""" + + +AgentDataOrDict = Union[AgentData, AgentDataDict] + + class AgentInfo(_common.BaseModel): """The agent info of an agent, used for agent eval.""" @@ -443,138 +895,3 @@ class SessionInputDict(TypedDict, total=False): SessionInputOrDict = Union[SessionInput, SessionInputDict] - - -class Tools(_common.BaseModel): - """Represents a list of tools for an agent.""" - - tool: Optional[list[genai_types.Tool]] = Field( - default=None, - description="""List of tools: each tool can have multiple function declarations.""", - ) - - -class ToolsDict(TypedDict, total=False): - """Represents a list of tools for an agent.""" - - tool: Optional[list[genai_types.ToolDict]] - """List of tools: each tool can have multiple function declarations.""" - - -ToolsOrDict = Union[Tools, ToolsDict] - - -class InstanceDataContents(_common.BaseModel): - """List of standard Content messages from Gemini API.""" - - contents: Optional[list[genai_types.Content]] = Field( - default=None, description="""Repeated contents.""" - ) - - -class InstanceDataContentsDict(TypedDict, total=False): - """List of standard Content messages from Gemini API.""" - - contents: Optional[list[genai_types.ContentDict]] - """Repeated contents.""" - - -InstanceDataContentsOrDict = Union[InstanceDataContents, InstanceDataContentsDict] - - -class InstanceData(_common.BaseModel): - """Instance data used to populate placeholders in a metric prompt template.""" - - text: Optional[str] = Field(default=None, description="""Text data.""") - contents: Optional[InstanceDataContents] = Field( - default=None, description="""List of Gemini content data.""" - ) - - -class InstanceDataDict(TypedDict, total=False): - """Instance data used to populate placeholders in a metric prompt template.""" - - text: Optional[str] - """Text data.""" - - contents: Optional[InstanceDataContentsDict] - """List of Gemini content data.""" - - -InstanceDataOrDict = Union[InstanceData, InstanceDataDict] - - -class AgentConfig(_common.BaseModel): - """Configuration for an Agent.""" - - tools_text: Optional[str] = Field( - default=None, - description="""A JSON string containing a list of tools available to an agent.""", - ) - tools: Optional[Tools] = Field(default=None, description="""List of tools.""") - developer_instruction: Optional[InstanceData] = Field( - default=None, - description="""A field containing instructions from the developer for the agent.""", - ) - - -class AgentConfigDict(TypedDict, total=False): - """Configuration for an Agent.""" - - tools_text: Optional[str] - """A JSON string containing a list of tools available to an agent.""" - - tools: Optional[ToolsDict] - """List of tools.""" - - developer_instruction: Optional[InstanceDataDict] - """A field containing instructions from the developer for the agent.""" - - -AgentConfigOrDict = Union[AgentConfig, AgentConfigDict] - - -class Events(_common.BaseModel): - """Represents a list of events for an agent.""" - - event: Optional[list[genai_types.Content]] = Field( - default=None, description="""A list of events.""" - ) - - -class EventsDict(TypedDict, total=False): - """Represents a list of events for an agent.""" - - event: Optional[list[genai_types.ContentDict]] - """A list of events.""" - - -EventsOrDict = Union[Events, EventsDict] - - -class AgentData(_common.BaseModel): - """Contains data specific to agent evaluations.""" - - agent_config: Optional[AgentConfig] = Field( - default=None, description="""Agent configuration.""" - ) - events_text: Optional[str] = Field( - default=None, description="""A JSON string containing a sequence of events.""" - ) - events: Optional[Events] = Field(default=None, description="""A list of events.""") - - -class AgentDataDict(TypedDict, total=False): - """Contains data specific to agent evaluations.""" - - agent_config: Optional[AgentConfigDict] - """Agent configuration.""" - - events_text: Optional[str] - """A JSON string containing a sequence of events.""" - - events: Optional[EventsDict] - """A list of events.""" - - -AgentDataOrDict = Union[AgentData, AgentDataDict]