From 29ba31302b762003a197663755dd52fca666feb5 Mon Sep 17 00:00:00 2001
From: ThibaultLSDC <thibault.de.chezelles@gmail.com>
Date: Tue, 22 Apr 2025 09:35:06 -0400
Subject: [PATCH 1/2] adding a simple debug agent to manually test actions

---
 src/agentlab/agents/debug_agent.py | 84 ++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 src/agentlab/agents/debug_agent.py

diff --git a/src/agentlab/agents/debug_agent.py b/src/agentlab/agents/debug_agent.py
new file mode 100644
index 00000000..9d368f92
--- /dev/null
+++ b/src/agentlab/agents/debug_agent.py
@@ -0,0 +1,84 @@
+from copy import deepcopy
+from dataclasses import asdict, dataclass
+from functools import partial
+
+import bgym
+from browsergym.experiments.agent import Agent, AgentInfo
+from browsergym.utils.obs import flatten_axtree_to_str, flatten_dom_to_str, overlay_som, prune_html
+
+from agentlab.agents.agent_args import AgentArgs
+from agentlab.llm.chat_api import BaseModelArgs
+from agentlab.llm.llm_utils import ParseError, image_to_png_base64_url, parse_html_tags_raise, retry
+from agentlab.llm.tracking import cost_tracker_decorator
+
+
+@dataclass
+class DebugAgentArgs(AgentArgs):
+
+    def __post_init__(self):
+        try:  # some attributes might be temporarily args.CrossProd for hyperparameter generation
+            self.agent_name = f"debug".replace("/", "_")
+        except AttributeError:
+            pass
+        self.action_set_args = bgym.DEFAULT_BENCHMARKS[
+            "miniwob_tiny_test"
+        ]().high_level_action_set_args
+
+    def set_benchmark(self, benchmark: bgym.Benchmark, demo_mode):
+        self.action_set_args = benchmark.high_level_action_set_args
+
+    def make_agent(self):
+        return DebugAgent(self.action_set_args)
+
+
+class DebugAgent(Agent):
+    def __init__(
+        self,
+        action_set_args,
+    ):
+        self.action_set = action_set_args.make_action_set()
+
+    def obs_preprocessor(self, obs):
+        obs = deepcopy(obs)
+        obs["dom_txt"] = flatten_dom_to_str(
+            obs["dom_object"],
+            extra_properties=obs["extra_element_properties"],
+            with_visible=True,
+            with_clickable=True,
+            with_center_coords=True,
+            with_bounding_box_coords=True,
+            filter_visible_only=False,
+            filter_with_bid_only=False,
+            filter_som_only=False,
+        )
+        obs["axtree_txt"] = flatten_axtree_to_str(
+            obs["axtree_object"],
+            extra_properties=obs["extra_element_properties"],
+            with_visible=True,
+            with_clickable=True,
+            with_center_coords=True,
+            with_bounding_box_coords=True,
+            filter_visible_only=False,
+            filter_with_bid_only=False,
+            filter_som_only=False,
+        )
+        obs["pruned_html"] = prune_html(obs["dom_txt"])
+        obs["screenshot_som"] = overlay_som(
+            obs["screenshot"], extra_properties=obs["extra_element_properties"]
+        )
+        return obs
+
+    def get_action(self, obs):
+
+        # print(obs["pruned_html"])
+        print("\n")
+        action = input(obs["axtree_txt"] + "\n")
+        agent_info = AgentInfo(
+            think="nope",
+            chat_messages=[],
+            stats={},
+        )
+        return action, agent_info
+
+
+DEBUG_AGENT = DebugAgentArgs()

From 851f3d48ddbeb4c3d8d5220950552b4a27d8df8d Mon Sep 17 00:00:00 2001
From: Thibault LSDC <78021491+TLSDC@users.noreply.github.com>
Date: Tue, 22 Apr 2025 09:41:58 -0400
Subject: [PATCH 2/2] Added html parametrization for miniwob

---
 src/agentlab/agents/debug_agent.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/agentlab/agents/debug_agent.py b/src/agentlab/agents/debug_agent.py
index 9d368f92..b907b2b0 100644
--- a/src/agentlab/agents/debug_agent.py
+++ b/src/agentlab/agents/debug_agent.py
@@ -23,20 +23,25 @@ def __post_init__(self):
         self.action_set_args = bgym.DEFAULT_BENCHMARKS[
             "miniwob_tiny_test"
         ]().high_level_action_set_args
+        self.use_html = False
 
     def set_benchmark(self, benchmark: bgym.Benchmark, demo_mode):
+        if benchmark.name.startswith("miniwob"):
+            self.use_html = True
         self.action_set_args = benchmark.high_level_action_set_args
 
     def make_agent(self):
-        return DebugAgent(self.action_set_args)
+        return DebugAgent(self.action_set_args, use_html=self.use_html)
 
 
 class DebugAgent(Agent):
     def __init__(
         self,
         action_set_args,
+        use_html=False,
     ):
         self.action_set = action_set_args.make_action_set()
+        self.use_html = use_html
 
     def obs_preprocessor(self, obs):
         obs = deepcopy(obs)
@@ -72,7 +77,8 @@ def get_action(self, obs):
 
         # print(obs["pruned_html"])
         print("\n")
-        action = input(obs["axtree_txt"] + "\n")
+        observation = obs["pruned_html"] if self.use_html else obs["axtree_txt"]
+        action = input(observation + "\n")
         agent_info = AgentInfo(
             think="nope",
             chat_messages=[],