feat: GenAI Client(evals) - Add loading agent info until function

PiperOrigin-RevId: 826638428

Author: vertex-sdk-bot

tests/unit/vertexai/genai/replays/test_create_evaluation_run.py

@@ -76,7 +76,7 @@ def test_create_eval_run_data_source_evaluation_set(client):
             LLM_METRIC,
         ],
         agent_info=types.evals.AgentInfo(
-            agent="project/123/locations/us-central1/reasoningEngines/456",
+            agent_resource_name="project/123/locations/us-central1/reasoningEngines/456",
             name="agent-1",
             instruction="agent-1 instruction",
             tool_declarations=[tool],

tests/unit/vertexai/genai/test_evals.py

@@ -2782,6 +2782,40 @@ def test_agent_info_creation(self):
         assert agent_info.description == "description1"
         assert agent_info.tool_declarations == [tool]
 
+    @mock.patch.object(genai_types.FunctionDeclaration, "from_callable_with_api_option")
+    def test_load_from_agent(self, mock_from_callable):
+        def my_search_tool(query: str) -> str:
+            """Searches for information."""
+            return f"search result for {query}"
+
+        mock_function_declaration = mock.Mock(spec=genai_types.FunctionDeclaration)
+        mock_from_callable.return_value = mock_function_declaration
+
+        mock_agent = mock.Mock()
+        mock_agent.name = "mock_agent"
+        mock_agent.instruction = "mock instruction"
+        mock_agent.description = "mock description"
+        mock_agent.tools = [my_search_tool]
+
+        agent_info = vertexai_genai_types.evals.AgentInfo.load_from_agent(
+            agent=mock_agent,
+            agent_resource_name="projects/123/locations/abc/reasoningEngines/456",
+        )
+
+        assert agent_info.name == "mock_agent"
+        assert agent_info.instruction == "mock instruction"
+        assert agent_info.description == "mock description"
+        assert (
+            agent_info.agent_resource_name
+            == "projects/123/locations/abc/reasoningEngines/456"
+        )
+        assert len(agent_info.tool_declarations) == 1
+        assert isinstance(agent_info.tool_declarations[0], genai_types.Tool)
+        assert agent_info.tool_declarations[0].function_declarations == [
+            mock_function_declaration
+        ]
+        mock_from_callable.assert_called_once_with(callable=my_search_tool)
+
 
 class TestEvent:
     """Unit tests for the Event class."""
@@ -4867,7 +4901,9 @@ def test_execute_evaluation_adds_creation_timestamp(
         frozenset(["summarization_quality"]),
     )
     @mock.patch("time.sleep", return_value=None)
-    @mock.patch("vertexai._genai.evals.Evals._evaluate_instances")
+    @mock.patch(
+        "vertexai._genai.evals.Evals._evaluate_instances"
+    )
     def test_predefined_metric_retry_on_resource_exhausted(
         self,
         mock_private_evaluate_instances,
@@ -4920,7 +4956,9 @@ def test_predefined_metric_retry_on_resource_exhausted(
         frozenset(["summarization_quality"]),
     )
     @mock.patch("time.sleep", return_value=None)
-    @mock.patch("vertexai._genai.evals.Evals._evaluate_instances")
+    @mock.patch(
+        "vertexai._genai.evals.Evals._evaluate_instances"
+    )
     def test_predefined_metric_retry_fail_on_resource_exhausted(
         self,
         mock_private_evaluate_instances,

vertexai/_genai/evals.py

@@ -1397,6 +1397,15 @@ def create_evaluation_run(
         )
         inference_configs = {}
         if agent_info:
+            if isinstance(agent_info, dict):
+                agent_info = types.evals.AgentInfo.model_validate(agent_info)
+            if (
+                not agent_info.agent_resource_name
+                or len(agent_info.agent_resource_name.split("reasoningEngines/")) != 2
+            ):
+                raise ValueError(
+                    "agent_info.agent_resource_name cannot be empty. Please provide a valid reasoning engine resource name in the format of projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine}."
+                )
             inference_configs[agent_info.name] = types.EvaluationRunInferenceConfig(
                 agent_config=types.EvaluationRunAgentConfig(
                     developer_instruction=genai_types.Content(
@@ -1405,11 +1414,11 @@ def create_evaluation_run(
                     tools=agent_info.tool_declarations,
                 )
             )
-            if agent_info.agent:
+            if agent_info.agent_resource_name:
                 labels = labels or {}
-                labels["vertex-ai-evaluation-agent-engine-id"] = agent_info.agent.split(
-                    "reasoningEngines/"
-                )[-1]
+                labels["vertex-ai-evaluation-agent-engine-id"] = (
+                    agent_info.agent_resource_name.split("reasoningEngines/")[-1]
+                )
         if not name:
             name = f"evaluation_run_{uuid.uuid4()}"
 
@@ -2244,6 +2253,15 @@ async def create_evaluation_run(
         )
         inference_configs = {}
         if agent_info:
+            if isinstance(agent_info, dict):
+                agent_info = types.evals.AgentInfo.model_validate(agent_info)
+            if (
+                not agent_info.agent_resource_name
+                or len(agent_info.agent_resource_name.split("reasoningEngines/")) != 2
+            ):
+                raise ValueError(
+                    "agent_info.agent_resource_name cannot be empty. Please provide a valid reasoning engine resource name in the format of projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine}."
+                )
             inference_configs[agent_info.name] = types.EvaluationRunInferenceConfig(
                 agent_config=types.EvaluationRunAgentConfig(
                     developer_instruction=genai_types.Content(
@@ -2252,11 +2270,11 @@ async def create_evaluation_run(
                     tools=agent_info.tool_declarations,
                 )
             )
-            if agent_info.agent:
+            if agent_info.agent_resource_name:
                 labels = labels or {}
-                labels["vertex-ai-evaluation-agent-engine-id"] = agent_info.agent.split(
-                    "reasoningEngines/"
-                )[-1]
+                labels["vertex-ai-evaluation-agent-engine-id"] = (
+                    agent_info.agent_resource_name.split("reasoningEngines/")[-1]
+                )
         if not name:
             name = f"evaluation_run_{uuid.uuid4()}"
 

vertexai/_genai/types/evals.py

@@ -39,7 +39,7 @@ class Importance(_common.CaseInSensitiveEnum):
 class AgentInfo(_common.BaseModel):
     """The agent info of an agent, used for agent eval."""
 
-    agent: Optional[str] = Field(
+    agent_resource_name: Optional[str] = Field(
         default=None,
         description="""The agent engine used to run agent. Agent engine resource name in str type, with format
             `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`.""",
@@ -57,11 +57,55 @@ class AgentInfo(_common.BaseModel):
         default=None, description="""List of tools used by the Agent."""
     )
 
+    @staticmethod
+    def _get_tool_declarations_from_agent(agent: Any) -> genai_types.ToolListUnion:
+        """Get tool declarations from an agent.
+
+        Args:
+          agent: The agent to get the tool declarations from. Data type is google.adk.agents.LLMAgent type, use Any to avoid dependency on ADK.
+
+        Returns:
+          The tool declarations of the agent.
+        """
+        tool_declarations: genai_types.ToolListUnion = []
+        for tool in agent.tools:
+            tool_declarations.append(
+                {
+                    "function_declarations": [
+                        genai_types.FunctionDeclaration.from_callable_with_api_option(
+                            callable=tool
+                        )
+                    ]
+                }
+            )
+        return tool_declarations
+
+    @classmethod
+    def load_from_agent(
+        cls, agent: Any, agent_resource_name: Optional[str] = None
+    ) -> "AgentInfo":
+        """Load agent info from an agent.
+
+        Args:
+          agent: The agent to get the agent info from, data type is google.adk.agents.LLMAgent type, use Any to avoid dependency on ADK.
+          agent_resource_name: Optional. The agent engine resource name.
+
+        Returns:
+          The agent info of the agent.
+        """
+        return cls(  # pytype: disable=missing-parameter
+            name=agent.name,
+            agent_resource_name=agent_resource_name,
+            instruction=agent.instruction,
+            description=agent.description,
+            tool_declarations=AgentInfo._get_tool_declarations_from_agent(agent),
+        )
+
 
 class AgentInfoDict(TypedDict, total=False):
     """The agent info of an agent, used for agent eval."""
 
-    agent: Optional[str]
+    agent_resource_name: Optional[str]
     """The agent engine used to run agent. Agent engine resource name in str type, with format
             `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`."""