feat(NewEvaluators): context precision and faithfulness

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.4.6"
+version = "2.4.7"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

src/uipath/_cli/_evals/_evaluator_factory.py

@@ -22,7 +22,9 @@
 from uipath.eval.evaluators import (
     BaseEvaluator,
     LegacyBaseEvaluator,
+    LegacyContextPrecisionEvaluator,
     LegacyExactMatchEvaluator,
+    LegacyFaithfulnessEvaluator,
     LegacyJsonSimilarityEvaluator,
     LegacyLlmAsAJudgeEvaluator,
     LegacyTrajectoryEvaluator,
@@ -68,6 +70,7 @@
     ToolCallOutputEvaluator,
     ToolCallOutputEvaluatorConfig,
 )
+from uipath.eval.models import LegacyEvaluatorType
 
 logger = logging.getLogger(__name__)
 
@@ -428,11 +431,8 @@ def _create_legacy_json_similarity_evaluator(
     def _create_legacy_llm_as_judge_evaluator(
         params: LLMEvaluatorParams,
         agent_model: str | None = None,
-    ) -> LegacyLlmAsAJudgeEvaluator:
-        """Create an LLM-as-a-judge evaluator."""
-        if not params.prompt:
-            raise ValueError("LLM evaluator must include 'prompt' field")
-
+    ) -> LegacyBaseEvaluator[Any]:
+        """Create an LLM-as-a-judge evaluator or context precision evaluator based on type."""
         if not params.model:
             raise ValueError("LLM evaluator must include 'model' field")
 
@@ -449,10 +449,16 @@ def _create_legacy_llm_as_judge_evaluator(
             )
             params = params.model_copy(update={"model": agent_model})
 
-        logger.info(
-            f"Creating LLM-as-judge evaluator '{params.name}' with model: {params.model}"
-        )
-        return LegacyLlmAsAJudgeEvaluator(**params.model_dump(), config={})
+        # Check evaluator type to determine which evaluator to create
+        if params.evaluator_type == LegacyEvaluatorType.ContextPrecision:
+            return LegacyContextPrecisionEvaluator(**params.model_dump(), config={})
+        elif params.evaluator_type == LegacyEvaluatorType.Faithfulness:
+            return LegacyFaithfulnessEvaluator(**params.model_dump(), config={})
+        else:
+            if not params.prompt:
+                raise ValueError("LLM evaluator must include 'prompt' field")
+
+            return LegacyLlmAsAJudgeEvaluator(**params.model_dump(), config={})
 
     @staticmethod
     def _create_legacy_trajectory_evaluator(

src/uipath/eval/evaluators/__init__.py

@@ -10,7 +10,9 @@
 
 # Legacy evaluators
 from .legacy_base_evaluator import LegacyBaseEvaluator
+from .legacy_context_precision_evaluator import LegacyContextPrecisionEvaluator
 from .legacy_exact_match_evaluator import LegacyExactMatchEvaluator
+from .legacy_faithfulness_evaluator import LegacyFaithfulnessEvaluator
 from .legacy_json_similarity_evaluator import LegacyJsonSimilarityEvaluator
 from .legacy_llm_as_judge_evaluator import LegacyLlmAsAJudgeEvaluator
 from .legacy_trajectory_evaluator import LegacyTrajectoryEvaluator
@@ -46,7 +48,9 @@
 __all__ = [
     # Legacy evaluators
     "LegacyBaseEvaluator",
+    "LegacyContextPrecisionEvaluator",
     "LegacyExactMatchEvaluator",
+    "LegacyFaithfulnessEvaluator",
     "LegacyJsonSimilarityEvaluator",
     "LegacyLlmAsAJudgeEvaluator",
     "LegacyTrajectoryEvaluator",