Fix: Use App (with plugins) for eval when available

- Extend LocalEvalService to accept optional App parameter
- Route evaluation through App so plugins are applied
- Add _generate_inferences_from_app() to EvaluationGenerator
- Update CLI eval command to load and pass App
- Add helper to load App from agent module

Fixes #3833

Author: ishanrajsingh

src/google/adk/cli/cli_tools_click.py

@@ -22,7 +22,11 @@
 import logging
 import os
 import tempfile
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..apps.app import App
+
 
 import click
 from click.core import ParameterSource
@@ -279,6 +283,34 @@ def cli_run(
       )
   )
 
+def _load_app_from_module(module_path: str) -> Optional['App']:
+    """Try to load an App instance from the agent module.
+    
+    Args:
+        module_path: Python module path (e.g., 'my_package.my_agent')
+        
+    Returns:
+        App instance if found, None otherwise
+    """
+    try:
+        import importlib
+        module = importlib.import_module(module_path)
+        
+        # Check for 'app' attribute (most common convention)
+        if hasattr(module, 'app'):
+            from ..apps.app import App
+            candidate = getattr(module, 'app')
+            if isinstance(candidate, App):
+                logger.info(f"Loaded App instance from {module_path}")
+                return candidate
+        
+        logger.debug(f"No App instance found in {module_path}")
+        
+    except (ImportError, AttributeError) as e:
+        logger.debug(f"Could not load App from module {module_path}: {e}")
+    
+    return None
+
 
 @main.command("eval", cls=HelpfulCommand)
 @click.argument(
@@ -471,10 +503,19 @@ def cli_eval(
       )
 
   try:
+    # Try to load App if available (for plugin support like ReflectAndRetryToolPlugin)
+    app = _load_app_from_module(agent_module_file_path)
+    
+    if app:
+        logger.info("Using App instance for evaluation (plugins will be applied)")
+    else:
+        logger.info("No App found, using root_agent directly")
+
     eval_service = LocalEvalService(
         root_agent=root_agent,
         eval_sets_manager=eval_sets_manager,
         eval_set_results_manager=eval_set_results_manager,
+        app=app, # NEW: Pass app if available
     )
 
     inference_results = asyncio.run(

src/google/adk/evaluation/evaluation_generator.py

@@ -15,8 +15,11 @@
 from __future__ import annotations
 
 import importlib
-from typing import Any
-from typing import Optional
+from typing import Any, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..apps.app import App
+
 import uuid
 
 from pydantic import BaseModel
@@ -220,6 +223,71 @@ async def _generate_inferences_from_root_agent(
       )
 
     return response_invocations
+  
+  @staticmethod
+  async def _generate_inferences_from_app(
+      invocations: list['Invocation'],
+      app: 'App',
+      initial_session: Optional['SessionInput'],
+      session_id: str,
+      session_service: 'BaseSessionService',
+      artifact_service: 'BaseArtifactService',
+  ) -> list['Invocation']:
+      """Generate inferences by invoking through App (preserving plugins)."""
+      
+      actual_invocations = []
+      
+      # Determine user_id consistently
+      user_id = 'test_user_id'
+      if initial_session and initial_session.user_id is not None:
+          user_id = initial_session.user_id
+      
+      # Initialize session if provided
+      if initial_session:
+          app_name = initial_session.app_name if initial_session.app_name else app.name
+          await session_service.create_session(
+              app_name=app_name,
+              user_id=user_id,
+              session_id=session_id,
+              state=initial_session.state if initial_session.state else {},
+          )
+      
+      # Run each invocation through the app
+      for expected_invocation in invocations:
+          user_content = expected_invocation.user_content
+          
+          # Invoke through App (this applies all plugins)
+          response = await app.run(
+              user_id=user_id,
+              session_id=session_id,
+              new_message=user_content,
+          )
+          
+          # Extract response similar to existing implementation
+          final_response = None
+          tool_uses = []
+          invocation_id = ""
+          
+          async for event in response:
+              invocation_id = invocation_id or event.invocation_id
+              
+              if event.is_final_response() and event.content and event.content.parts:
+                  final_response = event.content
+              elif event.get_function_calls():
+                  for call in event.get_function_calls():
+                      tool_uses.append(call)
+          
+          actual_invocations.append(
+              Invocation(
+                  invocation_id=invocation_id,
+                  user_content=user_content,
+                  final_response=final_response,
+                  intermediate_data=IntermediateData(tool_uses=tool_uses),
+              )
+          )
+      
+      return actual_invocations
+
 
   @staticmethod
   def _process_query_with_session(session_data, data):

src/google/adk/evaluation/local_eval_service.py

@@ -20,6 +20,10 @@
 from typing import AsyncGenerator
 from typing import Callable
 from typing import Optional
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..apps.app import App
 import uuid
 
 from typing_extensions import override
@@ -38,6 +42,7 @@
 from .base_eval_service import InferenceResult
 from .base_eval_service import InferenceStatus
 from .eval_case import Invocation
+from .eval_case import SessionInput
 from .eval_metrics import EvalMetric
 from .eval_metrics import EvalMetricResult
 from .eval_metrics import EvalMetricResultPerInvocation
@@ -73,9 +78,11 @@ def __init__(
       artifact_service: Optional[BaseArtifactService] = None,
       eval_set_results_manager: Optional[EvalSetResultsManager] = None,
       session_id_supplier: Callable[[], str] = _get_session_id,
+      app: Optional['App'] = None,
   ):
     self._root_agent = root_agent
     self._eval_sets_manager = eval_sets_manager
+    self._app = app
     metric_evaluator_registry = (
         metric_evaluator_registry or DEFAULT_METRIC_EVALUATOR_REGISTRY
     )
@@ -364,23 +371,37 @@ async def _perform_inference_sigle_eval_item(
     )
 
     try:
-      inferences = (
-          await EvaluationGenerator._generate_inferences_from_root_agent(
-              invocations=eval_case.conversation,
-              root_agent=root_agent,
-              initial_session=initial_session,
-              session_id=session_id,
-              session_service=self._session_service,
-              artifact_service=self._artifact_service,
+      # Use App if available (so plugins like ReflectAndRetryToolPlugin run)
+      if self._app is not None:
+          inferences = (
+              await EvaluationGenerator._generate_inferences_from_app(
+                  invocations=eval_case.conversation,
+                  app=self._app,
+                  initial_session=initial_session,
+                  session_id=session_id,
+                  session_service=self._session_service,
+                  artifact_service=self._artifact_service,
+              )
+          )
+      else:
+          # Fallback to direct root_agent usage (existing behavior)
+          inferences = (
+              await EvaluationGenerator._generate_inferences_from_root_agent(
+                  invocations=eval_case.conversation,
+                  root_agent=root_agent,
+                  initial_session=initial_session,
+                  session_id=session_id,
+                  session_service=self._session_service,
+                  artifact_service=self._artifact_service,
+              )
           )
-      )
 
       inference_result.inferences = inferences
       inference_result.status = InferenceStatus.SUCCESS
 
       return inference_result
     except Exception as e:
-      # We intentionally catch the Exception as we don't failures to affect
+      # We intentionally catch the Exception as we don't want failures to affect
       # other inferences.
       logger.error(
           'Inference failed for eval case `%s` with error %s',
@@ -389,4 +410,4 @@ async def _perform_inference_sigle_eval_item(
       )
       inference_result.status = InferenceStatus.FAILURE
       inference_result.error_message = str(e)
-      return inference_result
+      return inference_result