Merge pull request #1021 from UiPath/fix/evaluator-folder

fix: use new eval folder

Author: radu-mocanu

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.2.30"
+version = "2.2.31"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

src/uipath/_cli/_evals/_evaluator_factory.py

@@ -17,7 +17,7 @@
     TrajectoryEvaluatorParams,
 )
 from uipath._cli._evals._models._evaluator_base_params import EvaluatorBaseParams
-from uipath._cli._utils._constants import EVALS_DIRECTORY_NAME
+from uipath._utils.constants import EVALS_FOLDER
 from uipath.eval.evaluators import (
     BaseEvaluator,
     LegacyBaseEvaluator,
@@ -206,7 +206,7 @@ def _create_coded_evaluator_internal(
                     # Fall back to the old behavior
                     file_path = (
                         Path.cwd()
-                        / EVALS_DIRECTORY_NAME
+                        / EVALS_FOLDER
                         / "evaluators"
                         / "custom"
                         / file_path_str

src/uipath/_cli/_evals/_helpers.py

@@ -11,8 +11,7 @@
 import click
 
 from uipath._cli._utils._console import ConsoleLogger
-from uipath._cli._utils._constants import EVALS_DIRECTORY_NAME
-from uipath._utils.constants import CUSTOM_EVALUATOR_PREFIX
+from uipath._utils.constants import CUSTOM_EVALUATOR_PREFIX, EVALS_FOLDER
 
 logger = logging.getLogger(__name__)
 console = ConsoleLogger().get_instance()
@@ -37,7 +36,7 @@ def to_kebab_case(text: str) -> str:
 
 def find_evaluator_file(filename: str) -> Path | None:
     """Find the evaluator file in evals/evaluators/custom folder."""
-    custom_evaluators_path = Path.cwd() / EVALS_DIRECTORY_NAME / "evaluators" / "custom"
+    custom_evaluators_path = Path.cwd() / EVALS_FOLDER / "evaluators" / "custom"
 
     if not custom_evaluators_path.exists():
         return None
@@ -124,7 +123,9 @@ def register_evaluator(filename: str) -> tuple[str, str]:
         filename = filename + ".py"
     file_path = find_evaluator_file(filename)
     if file_path is None:
-        console.error(f"Could not find '{filename}' in evals/evaluators/custom folder")
+        console.error(
+            f"Could not find '{filename}' in {EVALS_FOLDER}/evaluators/custom folder"
+        )
 
     relative_path = f"evals/evaluators/custom/{filename}"
     console.info(
@@ -151,7 +152,7 @@ def register_evaluator(filename: str) -> tuple[str, str]:
     evaluator_config = generate_evaluator_config(evaluator_class, class_name)
     evaluator_json_type = evaluator_class.generate_json_type()
 
-    evaluators_dir = Path.cwd() / EVALS_DIRECTORY_NAME / "evaluators"
+    evaluators_dir = Path.cwd() / EVALS_FOLDER / "evaluators"
     evaluators_dir.mkdir(parents=True, exist_ok=True)
 
     evaluator_types_dir = evaluators_dir / "custom" / "types"
@@ -167,7 +168,7 @@ def register_evaluator(filename: str) -> tuple[str, str]:
         json.dump(evaluator_json_type, f, indent=2)
 
     relative_output_path = (
-        f"evals/evaluators/custom/types/{output_file_evaluator_types}"
+        f"{EVALS_FOLDER}/evaluators/custom/types/{output_file_evaluator_types}"
     )
     console.success(
         f"Generated evaluator types: {click.style(relative_output_path, fg='cyan')}"

src/uipath/_cli/_utils/_constants.py

@@ -57,8 +57,6 @@
     | SPECIAL_EXTENSIONS
 )
 
-EVALS_DIRECTORY_NAME = "evals"
-
 
 def is_binary_file(file_extension: str) -> bool:
     """Determine if a file should be treated as binary."""

src/uipath/_cli/cli_add.py

@@ -6,8 +6,8 @@
 
 import click
 
+from .._utils.constants import EVALS_FOLDER
 from ._utils._console import ConsoleLogger
-from ._utils._constants import EVALS_DIRECTORY_NAME
 from ._utils._resources import Resources
 
 logger = logging.getLogger(__name__)
@@ -47,7 +47,7 @@ def generate_evaluator_template(evaluator_name: str) -> str:
 
 def create_evaluator(evaluator_name):
     cwd = Path.cwd()
-    custom_evaluators_dir = cwd / EVALS_DIRECTORY_NAME / "evaluators" / "custom"
+    custom_evaluators_dir = cwd / EVALS_FOLDER / "evaluators" / "custom"
 
     if not custom_evaluators_dir.exists():
         console.info(
@@ -69,7 +69,7 @@ def create_evaluator(evaluator_name):
     with open(file_path, "w") as f:
         f.write(template_content)
 
-    relative_path = f"evals/evaluators/custom/{filename}"
+    relative_path = f"{EVALS_FOLDER}/evaluators/custom/{filename}"
 
     console.success(f"Created new evaluator: {click.style(relative_path, fg='cyan')}")
     console.hint("Next steps:")

src/uipath/agent/utils/_utils.py

@@ -6,9 +6,6 @@
 from httpx import Response
 from pydantic import TypeAdapter
 
-from uipath._cli._utils._constants import (
-    EVALS_DIRECTORY_NAME,
-)
 from uipath._cli._utils._eval_set import EvalHelpers
 from uipath._cli._utils._project_files import pull_project
 from uipath._cli._utils._studio_project import (
@@ -18,6 +15,7 @@
     StudioSolutionsClient,
     resolve_path,
 )
+from uipath._utils.constants import EVALS_FOLDER
 from uipath.agent.models.agent import (
     AgentDefinition,
 )
@@ -99,7 +97,7 @@ def load_agent_definition(
 
     # Load evaluators from downloaded files
     evaluators = []
-    evaluators_dir = target_project_dir / EVALS_DIRECTORY_NAME / "evaluators"
+    evaluators_dir = target_project_dir / EVALS_FOLDER / "evaluators"
     if evaluators_dir.exists() and evaluators_dir.is_dir():
         for file_path in evaluators_dir.glob("*.json"):
             try:
@@ -114,7 +112,7 @@ def load_agent_definition(
 
     # Load evaluation sets from downloaded files
     evaluation_sets = []
-    eval_sets_dir = target_project_dir / EVALS_DIRECTORY_NAME / "eval-sets"
+    eval_sets_dir = target_project_dir / EVALS_FOLDER / "eval-sets"
     if eval_sets_dir.exists() and eval_sets_dir.is_dir():
         for file_path in eval_sets_dir.glob("*.json"):
             evaluation_set, _ = EvalHelpers.load_eval_set(str(file_path))