diff --git a/src/uipath/_cli/_evals/mocks/llm_mocker.py b/src/uipath/_cli/_evals/mocks/llm_mocker.py index aadfb45e9..2888a19b5 100644 --- a/src/uipath/_cli/_evals/mocks/llm_mocker.py +++ b/src/uipath/_cli/_evals/mocks/llm_mocker.py @@ -90,7 +90,11 @@ async def response( assert isinstance(self.context.strategy, LLMMockingStrategy) function_name = params.get("name") or func.__name__ - if function_name in [x.name for x in self.context.strategy.tools_to_simulate]: + normalized_function_name = function_name.replace("_", " ").lower() + if normalized_function_name in [ + x.name.replace("_", " ").lower() + for x in self.context.strategy.tools_to_simulate + ]: from uipath.platform import UiPath from uipath.platform.chat._llm_gateway_service import _cleanup_schema diff --git a/src/uipath/_cli/_evals/mocks/mocks.py b/src/uipath/_cli/_evals/mocks/mocks.py index ed8f1c3c6..793ddeb19 100644 --- a/src/uipath/_cli/_evals/mocks/mocks.py +++ b/src/uipath/_cli/_evals/mocks/mocks.py @@ -73,11 +73,12 @@ def clear_execution_context() -> None: def _normalize_tool_name(name: str) -> str: - """Normalize tool name by replacing underscores with spaces. + """Normalize tool name by replacing underscores with spaces and lowering case. Tool names may use spaces in configuration but underscores in execution. + Comparison is case-insensitive. """ - return name.replace("_", " ") + return name.replace("_", " ").lower() def is_tool_simulated(tool_name: str) -> bool: diff --git a/tests/cli/test_debug_simulation.py b/tests/cli/test_debug_simulation.py index 61498ee19..12f561480 100644 --- a/tests/cli/test_debug_simulation.py +++ b/tests/cli/test_debug_simulation.py @@ -524,3 +524,37 @@ def test_handles_tool_name_normalization(self, temp_dir: str): assert is_tool_simulated("Web Search") is True clear_execution_context() + + def test_handles_tool_name_case_insensitive(self, temp_dir: str): + """Test that tool name comparison is case-insensitive.""" + config = { + "enabled": True, + "instructions": "Test", + "toolsToSimulate": [ + {"name": "Web Reader"}, + {"name": "Web Search"}, + ], + } + simulation_path = Path(temp_dir) / "simulation.json" + with open(simulation_path, "w", encoding="utf-8") as f: + json.dump(config, f) + + with patch("uipath._cli.cli_debug.Path.cwd", return_value=Path(temp_dir)): + mocking_ctx = load_simulation_config() + assert mocking_ctx is not None + + from uipath._cli._evals._span_collection import ExecutionSpanCollector + from uipath._cli._evals.mocks.mocks import set_execution_context + + span_collector = ExecutionSpanCollector() + set_execution_context(mocking_ctx, span_collector, "test-id") + + # Case-insensitive matching should work + assert is_tool_simulated("web reader") is True + assert is_tool_simulated("WEB READER") is True + assert is_tool_simulated("Web reader") is True + assert is_tool_simulated("web search") is True + assert is_tool_simulated("WEB SEARCH") is True + assert is_tool_simulated("Non Existent Tool") is False + + clear_execution_context()