Handle None inferences in eval results for issue #2729

Author: ShaharKatz

src/google/adk/evaluation/local_eval_service.py

@@ -168,10 +168,43 @@ async def run_evaluation(inference_result):
             evaluate_config=evaluate_request.evaluate_config,
         )
 
-    evaluation_tasks = [
-        run_evaluation(inference_result)
-        for inference_result in evaluate_request.inference_results
-    ]
+    evaluation_tasks = []
+
+    for inference_result in evaluate_request.inference_results:
+      if inference_result.status == InferenceStatus.FAILURE:
+        logger.warning(
+            'Skipping evaluation for eval case `%s` because inference failed'
+            ' with status `%s`: %s',
+            inference_result.eval_case_id,
+            inference_result.status,
+            inference_result.error_message,
+        )
+        eval_case = self._eval_sets_manager.get_eval_case(
+            app_name=inference_result.app_name,
+            eval_set_id=inference_result.eval_set_id,
+            eval_case_id=inference_result.eval_case_id,
+        )
+        if eval_case is None:
+          raise NotFoundError(
+              f'Eval case with id {inference_result.eval_case_id} not found'
+              f' for app {inference_result.app_name} and eval set'
+              f' {inference_result.eval_set_id}.'
+          )
+        eval_case_result = await self._build_not_evaluated_eval_case_result(
+            inference_result=inference_result,
+            eval_case=eval_case,
+            reason='Inference failed',
+        )
+        if self._eval_set_results_manager:
+          self._eval_set_results_manager.save_eval_set_result(
+              app_name=inference_result.app_name,
+              eval_set_id=inference_result.eval_set_id,
+              eval_case_results=[eval_case_result],
+          )
+        yield eval_case_result
+        continue
+
+      evaluation_tasks.append(run_evaluation(inference_result))
 
     for evaluation_task in asyncio.as_completed(evaluation_tasks):
       inference_result, eval_case_result = await evaluation_task
@@ -226,6 +259,29 @@ async def _evaluate_single_inference_result(
         else 'test_user_id'
     )
 
+    if inference_result.status == InferenceStatus.FAILURE:
+      eval_case_result = await self._build_not_evaluated_eval_case_result(
+          inference_result=inference_result,
+          eval_case=eval_case,
+          user_id=user_id,
+          reason='Inference status indicates failure',
+      )
+      return (inference_result, eval_case_result)
+
+    if inference_result.inferences is None:
+      logger.warning(
+          'Inference result for eval case `%s` did not include inferences;'
+          ' marking as not evaluated.',
+          inference_result.eval_case_id,
+      )
+      eval_case_result = await self._build_not_evaluated_eval_case_result(
+          inference_result=inference_result,
+          eval_case=eval_case,
+          user_id=user_id,
+          reason='Missing inference payload',
+      )
+      return (inference_result, eval_case_result)
+
     if eval_case.conversation_scenario is None and len(
         inference_result.inferences
     ) != len(eval_case.conversation):
@@ -389,6 +445,61 @@ def _generate_final_eval_status(
 
     return final_eval_status
 
+  async def _build_not_evaluated_eval_case_result(
+      self,
+      *,
+      inference_result: InferenceResult,
+      eval_case: EvalCase,
+      user_id: Optional[str] = None,
+      reason: Optional[str] = None,
+  ) -> EvalCaseResult:
+    """Constructs an EvalCaseResult for cases that could not be evaluated."""
+    resolved_user_id = user_id or self._resolve_user_id(eval_case)
+    session_details = await self._safe_get_session_details(
+        app_name=inference_result.app_name,
+        user_id=resolved_user_id,
+        session_id=inference_result.session_id,
+    )
+    if reason:
+      logger.info(
+          'Eval case `%s` marked as not evaluated: %s',
+          inference_result.eval_case_id,
+          reason,
+      )
+    return EvalCaseResult(
+        eval_set_file=inference_result.eval_set_id,
+        eval_set_id=inference_result.eval_set_id,
+        eval_id=inference_result.eval_case_id,
+        final_eval_status=EvalStatus.NOT_EVALUATED,
+        overall_eval_metric_results=[],
+        eval_metric_result_per_invocation=[],
+        session_id=inference_result.session_id,
+        session_details=session_details,
+        user_id=resolved_user_id,
+    )
+
+  def _resolve_user_id(self, eval_case: EvalCase) -> str:
+    if eval_case.session_input and eval_case.session_input.user_id:
+      return eval_case.session_input.user_id
+    return 'test_user_id'
+
+  async def _safe_get_session_details(
+      self, *, app_name: str, user_id: str, session_id: str
+  ):
+    try:
+      return await self._session_service.get_session(
+          app_name=app_name, user_id=user_id, session_id=session_id
+      )
+    except NotFoundError:
+      logger.warning(
+          'Session `%s` for app `%s` and user `%s` not found while building'
+          ' eval result; continuing without session details.',
+          session_id,
+          app_name,
+          user_id,
+      )
+      return None
+
   async def _perform_inference_single_eval_item(
       self,
       app_name: str,

tests/unittests/evaluation/test_local_eval_service.py

@@ -407,6 +407,50 @@ async def test_evaluate_single_inference_result(
     assert metric_result.eval_status == EvalStatus.PASSED
 
 
+@pytest.mark.asyncio
+async def test_evaluate_single_inference_result_handles_failed_inference(
+    eval_service, mock_eval_sets_manager, mocker
+):
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="test user content.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="test final response.")]
+      ),
+  )
+  inference_result = InferenceResult(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case_id="case1",
+      inferences=None,
+      session_id="session1",
+      status=InferenceStatus.FAILURE,
+      error_message="simulated inference failure",
+  )
+  eval_metric = EvalMetric(metric_name="fake_metric", threshold=0.5)
+  evaluate_config = EvaluateConfig(eval_metrics=[eval_metric], parallelism=1)
+
+  mock_eval_case = mocker.MagicMock(spec=EvalCase)
+  mock_eval_case.conversation = [invocation.model_copy(deep=True)]
+  mock_eval_case.conversation_scenario = None
+  mock_eval_case.session_input = None
+  mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
+
+  _, result = await eval_service._evaluate_single_inference_result(
+      inference_result=inference_result, evaluate_config=evaluate_config
+  )
+
+  assert isinstance(result, EvalCaseResult)
+  assert result.eval_id == "case1"
+  assert result.final_eval_status == EvalStatus.NOT_EVALUATED
+  assert result.overall_eval_metric_results == []
+  assert result.eval_metric_result_per_invocation == []
+  mock_eval_sets_manager.get_eval_case.assert_called_once_with(
+      app_name="test_app", eval_set_id="test_eval_set", eval_case_id="case1"
+  )
+
+
 @pytest.mark.asyncio
 async def test_evaluate_single_inference_result_for_conversation_scenario(
     eval_service, mock_eval_sets_manager, mocker