Skip to content

Improve xray #259

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
recursix merged 1 commit into from
Jul 4, 2025
Merged

Improve xray #259

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 68 additions & 18 deletions src/agentlab/analyze/agent_xray.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ class StepId:
@dataclass
class Info:
results_dir: Path = None # to root directory of all experiments
exp_list_dir: Path = None # the path of the currently selected experiment
study_dirs: Path = None # the path of the currently selected experiment
result_df: pd.DataFrame = None # the raw loaded df
agent_df: pd.DataFrame = None # the df filtered for selected agent
tasks_df: pd.DataFrame = None # the unique tasks for selected agent
Expand Down Expand Up @@ -178,6 +178,8 @@ def run_gradio(results_dir: Path):
agent_task_id = gr.State(value=None)
step_id = gr.State(value=None)

hidden_key_input = gr.Textbox(visible=False, elem_id="key_capture")

with gr.Accordion("Help", open=False):
gr.Markdown(
"""\
Expand Down Expand Up @@ -207,6 +209,7 @@ def run_gradio(results_dir: Path):
exp_dir_choice = gr.Dropdown(
choices=get_directory_contents(results_dir),
value=select_dir_instructions,
multiselect=True,
label="Experiment Directory",
show_label=False,
scale=6,
Expand Down Expand Up @@ -502,6 +505,32 @@ def run_gradio(results_dir: Path):

demo.load(fn=refresh_exp_dir_choices, inputs=exp_dir_choice, outputs=exp_dir_choice)

demo.load(
None,
None,
None,
js="""
function() {
document.addEventListener('keydown', function(e) {
if ((e.key === 'ArrowLeft' || e.key === 'ArrowRight') && (e.metaKey || e.ctrlKey)) {
e.preventDefault();
const hiddenInput = document.querySelector('#key_capture input, #key_capture textarea');
if (hiddenInput) {
let event = e.key === 'ArrowLeft' ? 'Cmd+Left' : 'Cmd+Right';
hiddenInput.value = event;
hiddenInput.dispatchEvent(new Event('input', {bubbles: true}));
}
}
});
}
""",
)
hidden_key_input.change(
handle_key_event,
inputs=[hidden_key_input, step_id],
outputs=[hidden_key_input, step_id],
)

demo.queue()

do_share = os.getenv("AGENTXRAY_SHARE_GRADIO", "false").lower() == "true"
Expand All @@ -511,6 +540,25 @@ def run_gradio(results_dir: Path):
demo.launch(server_port=port, share=do_share)


def handle_key_event(key_event, step_id: StepId):

if key_event:
global info

# print(f"Key event: {key_event}")
step = step_id.step
if key_event.startswith("Cmd+Left"):
step = max(0, step - 1)
elif key_event.startswith("Cmd+Right"):
step = min(len(info.exp_result.steps_info) - 2, step + 1)
else:
return gr.update()
# print(f"Updating step to {step} from key event {key_event}")
info.step = step
step_id = StepId(episode_id=step_id.episode_id, step=step)
return ("", step_id)


def tab_select(evt: gr.SelectData):
global info
info.active_tab = evt.value
Expand Down Expand Up @@ -546,8 +594,8 @@ def get_screenshot(
):
if step is None:
step = info.step
step_info = info.exp_result.steps_info[step]
try:
step_info = info.exp_result.steps_info[step]
is_som = som_or_not == "SOM Screenshots"
img = info.exp_result.get_screenshot(step, som=is_som)
if annotate:
Expand All @@ -557,7 +605,7 @@ def get_screenshot(
else:
action_colored = None
return img, action_colored
except FileNotFoundError:
except (FileNotFoundError, IndexError):
return None, None


Expand Down Expand Up @@ -839,6 +887,10 @@ def get_episode_info(info: Info):
try:
env_args = info.exp_result.exp_args.env_args
steps_info = info.exp_result.steps_info
if info.step >= len(steps_info):
info.step = len(steps_info) - 1
Comment on lines +890 to +891
Copy link

@korbit-ai korbit-ai bot Jul 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Empty Steps List Index Error category Error Handling

Tell me more
What is the issue?

The code attempts to handle index errors by setting step to len(steps_info) - 1, but this could still fail if steps_info is empty.

Why this matters

If steps_info is an empty list, len(steps_info) - 1 would be -1, causing an IndexError when trying to access steps_info[-1].

Suggested change ∙ Feature Preview
if len(steps_info) == 0:
    info.step = 0
elif info.step >= len(steps_info):
    info.step = len(steps_info) - 1
Provide feedback to improve future suggestions

Nice Catch Incorrect Not in Scope Not in coding standard Other

💬 Looking for more details? Reply to this comment to chat with Korbit.

if len(steps_info) == 0:
return "No steps were taken in this episode."
step_info = steps_info[info.step]
try:
goal = step_info.obs["goal_object"]
Expand Down Expand Up @@ -1040,31 +1092,29 @@ def update_global_stats():


def update_error_report():
report_files = list(info.exp_list_dir.glob("error_report*.md"))
if len(report_files) == 0:
return "No error report found"
report_files = sorted(report_files, key=os.path.getctime, reverse=True)
return report_files[0].read_text()
return inspect_results.error_report(info.result_df, max_stack_trace=3, use_log=True)


def new_exp_dir(exp_dir, progress=gr.Progress(), just_refresh=False):
if exp_dir == select_dir_instructions:
return None, None
def new_exp_dir(study_names: list, progress=gr.Progress(), just_refresh=False):
global info

exp_dir = exp_dir.split(" - ")[0]
# remove select_dir_instructions from study_names
if select_dir_instructions in study_names:
study_names.remove(select_dir_instructions)

if len(exp_dir) == 0:
info.exp_list_dir = None
if len(study_names) == 0:
return None, None

info.exp_list_dir = info.results_dir / exp_dir
info.result_df = inspect_results.load_result_df(info.exp_list_dir, progress_fn=progress.tqdm)
info.study_dirs = [info.results_dir / study_name.split(" - ")[0] for study_name in study_names]
info.result_df = inspect_results.load_result_df(info.study_dirs, progress_fn=progress.tqdm)
info.result_df = remove_args_from_col(info.result_df)

study_summary = inspect_results.summarize_study(info.result_df)
# save study_summary
study_summary.to_csv(info.exp_list_dir / "summary_df.csv", index=False)
agent_report = display_table(study_summary)

for study_dir in info.study_dirs:
study_summary.to_csv(study_dir / "summary_df.csv", index=False)
agent_report = display_table(study_summary)

info.agent_id_keys = agent_report.index.names
agent_report.reset_index(inplace=True)
Expand Down