feat: support 'same-as-agent' model option for legacy evaluators

Add support for the 'same-as-agent' model configuration in legacy LLM-based
evaluators. When an evaluator specifies 'same-as-agent' as its model, it now
resolves to the actual model from agent.json settings instead of throwing an
error.

Changes:
- Updated EvaluatorFactory to accept and pass agent_model parameter
- Added _get_agent_model() method to runtime to load model from agent.json
- Added logging for model resolution and evaluator creation
- Fixed error message in trajectory evaluator (was incorrectly saying "LLM evaluator")

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: Chibi Vikram

src/uipath/_cli/_evals/_evaluator_factory.py

@@ -1,10 +1,13 @@
 import importlib.util
+import logging
 import sys
 from pathlib import Path
 from typing import Any
 
 from pydantic import TypeAdapter
 
+logger = logging.getLogger(__name__)
+
 from uipath._cli._evals._helpers import (  # type: ignore # Remove after gnarly fix
     try_extract_file_and_class_name,
 )
@@ -106,12 +109,15 @@ def _prepare_evaluator_config(data: dict[str, Any]) -> dict[str, Any]:
 
     @classmethod
     def create_evaluator(
-        cls, data: dict[str, Any], evaluators_dir: Path | None = None
+        cls,
+        data: dict[str, Any],
+        evaluators_dir: Path | None = None,
+        agent_model: str | None = None,
     ) -> BaseEvaluator[Any, Any, Any]:
         if data.get("version", None) == "1.0":
             return cls._create_evaluator_internal(data, evaluators_dir)
         else:
-            return cls._create_legacy_evaluator_internal(data)
+            return cls._create_legacy_evaluator_internal(data, agent_model)
 
     @staticmethod
     def _create_evaluator_internal(
@@ -371,11 +377,14 @@ def _create_llm_judge_simulation_trajectory_evaluator(
     @staticmethod
     def _create_legacy_evaluator_internal(
         data: dict[str, Any],
+        agent_model: str | None = None,
     ) -> LegacyBaseEvaluator[Any]:
         """Create an evaluator instance from configuration data.
 
         Args:
             data: Dictionary containing evaluator configuration from JSON file
+            agent_model: Optional model name from agent settings for resolving
+                'same-as-agent' model configuration
 
         Returns:
             Appropriate evaluator instance based on category
@@ -391,9 +400,13 @@ def _create_legacy_evaluator_internal(
             case JsonSimilarityEvaluatorParams():
                 return EvaluatorFactory._create_legacy_json_similarity_evaluator(params)
             case LLMEvaluatorParams():
-                return EvaluatorFactory._create_legacy_llm_as_judge_evaluator(params)
+                return EvaluatorFactory._create_legacy_llm_as_judge_evaluator(
+                    params, agent_model
+                )
             case TrajectoryEvaluatorParams():
-                return EvaluatorFactory._create_legacy_trajectory_evaluator(params)
+                return EvaluatorFactory._create_legacy_trajectory_evaluator(
+                    params, agent_model
+                )
             case _:
                 raise ValueError(f"Unknown evaluator category: {params}")
 
@@ -414,33 +427,59 @@ def _create_legacy_json_similarity_evaluator(
     @staticmethod
     def _create_legacy_llm_as_judge_evaluator(
         params: LLMEvaluatorParams,
+        agent_model: str | None = None,
     ) -> LegacyLlmAsAJudgeEvaluator:
         """Create an LLM-as-a-judge evaluator."""
         if not params.prompt:
             raise ValueError("LLM evaluator must include 'prompt' field")
 
         if not params.model:
             raise ValueError("LLM evaluator must include 'model' field")
+
+        # Resolve 'same-as-agent' to actual agent model
         if params.model == "same-as-agent":
-            raise ValueError(
-                "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
+            if not agent_model:
+                raise ValueError(
+                    "'same-as-agent' model option requires agent settings. "
+                    "Ensure agent.json contains valid model settings."
+                )
+            logger.info(
+                f"Resolving 'same-as-agent' to agent model: {agent_model} "
+                f"for evaluator '{params.name}'"
             )
+            params = params.model_copy(update={"model": agent_model})
 
+        logger.info(
+            f"Creating LLM-as-judge evaluator '{params.name}' with model: {params.model}"
+        )
         return LegacyLlmAsAJudgeEvaluator(**params.model_dump(), config={})
 
     @staticmethod
     def _create_legacy_trajectory_evaluator(
         params: TrajectoryEvaluatorParams,
+        agent_model: str | None = None,
     ) -> LegacyTrajectoryEvaluator:
         """Create a trajectory evaluator."""
         if not params.prompt:
             raise ValueError("Trajectory evaluator must include 'prompt' field")
 
         if not params.model:
-            raise ValueError("LLM evaluator must include 'model' field")
+            raise ValueError("Trajectory evaluator must include 'model' field")
+
+        # Resolve 'same-as-agent' to actual agent model
         if params.model == "same-as-agent":
-            raise ValueError(
-                "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
+            if not agent_model:
+                raise ValueError(
+                    "'same-as-agent' model option requires agent settings. "
+                    "Ensure agent.json contains valid model settings."
+                )
+            logger.info(
+                f"Resolving 'same-as-agent' to agent model: {agent_model} "
+                f"for evaluator '{params.name}'"
             )
+            params = params.model_copy(update={"model": agent_model})
 
+        logger.info(
+            f"Creating trajectory evaluator '{params.name}' with model: {params.model}"
+        )
         return LegacyTrajectoryEvaluator(**params.model_dump(), config={})

src/uipath/_cli/_evals/_runtime.py

@@ -601,6 +601,22 @@ async def run_evaluator(
 
         return result
 
+    def _get_agent_model(self) -> str | None:
+        """Load agent model from agent.json.
+
+        Returns:
+            The model name from agent settings, or None if not found.
+        """
+        agent_json = Path.cwd() / "agent.json"
+        if agent_json.exists():
+            try:
+                with open(agent_json, "r", encoding="utf-8") as f:
+                    data = json.load(f)
+                return data.get("settings", {}).get("model")
+            except (json.JSONDecodeError, OSError):
+                return None
+        return None
+
     def _load_evaluators(
         self, evaluation_set: EvaluationSet
     ) -> list[BaseEvaluator[Any, Any, Any]]:
@@ -611,6 +627,9 @@ def _load_evaluators(
             raise ValueError("eval_set cannot be None")
         evaluators_dir = Path(eval_set).parent.parent / "evaluators"
 
+        # Load agent model for 'same-as-agent' resolution in legacy evaluators
+        agent_model = self._get_agent_model()
+
         # If evaluatorConfigs is specified, use that (new field with weights)
         # Otherwise, fall back to evaluatorRefs (old field without weights)
         if (
@@ -638,7 +657,9 @@ def _load_evaluators(
             try:
                 evaluator_id = data.get("id")
                 if evaluator_id in evaluator_ref_ids:
-                    evaluator = EvaluatorFactory.create_evaluator(data, evaluators_dir)
+                    evaluator = EvaluatorFactory.create_evaluator(
+                        data, evaluators_dir, agent_model=agent_model
+                    )
                     evaluators.append(evaluator)
                     found_evaluator_ids.add(evaluator_id)
             except Exception as e: