From d00da1894411abd73185526ff961b13c362dc252 Mon Sep 17 00:00:00 2001 From: optimass Date: Mon, 13 Jan 2025 17:40:39 +0000 Subject: [PATCH 1/3] added `study.shuffle_exps()` feature --- src/agentlab/experiments/study.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/agentlab/experiments/study.py b/src/agentlab/experiments/study.py index 195aea09..74c2b49b 100644 --- a/src/agentlab/experiments/study.py +++ b/src/agentlab/experiments/study.py @@ -157,6 +157,12 @@ def get_results(self, suffix="", also_save=True): return result_df, summary_df, error_report + def shuffle_exps(self): + """Shuffle the experiments in the study.""" + import random + + random.shuffle(self.exp_args_list) + @dataclass class Study(AbstractStudy): From 9dfbbda3a4de4a9ee052784a3a7c9910ec5a171b Mon Sep 17 00:00:00 2001 From: optimass Date: Tue, 14 Jan 2025 00:03:31 +0000 Subject: [PATCH 2/3] details matter --- src/agentlab/experiments/study.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/agentlab/experiments/study.py b/src/agentlab/experiments/study.py index 74c2b49b..b08b11d0 100644 --- a/src/agentlab/experiments/study.py +++ b/src/agentlab/experiments/study.py @@ -20,6 +20,7 @@ from agentlab.experiments.launch_exp import find_incomplete, non_dummy_count, run_experiments from agentlab.experiments.multi_server import BaseServer, WebArenaInstanceVars from multiprocessing import Pool, Manager, Queue +import random logger = logging.getLogger(__name__) @@ -159,9 +160,7 @@ def get_results(self, suffix="", also_save=True): def shuffle_exps(self): """Shuffle the experiments in the study.""" - import random - - random.shuffle(self.exp_args_list) + self.exp_args_list = random.sample(self.exp_args_list, len(self.exp_args_list)) @dataclass @@ -407,6 +406,8 @@ def load(dir: Path) -> "Study": inspect_results.yield_all_exp_results(savedir_base=dir, progress_fn=None) ) benchmark_name = first_result.exp_args.env_args.task_name.split(".")[0] + # if "" + agent_args = first_result.exp_args.agent_args study = Study(agent_args=agent_args, benchmark=benchmark_name, dir=dir) else: From 2f79dcd43f44170e8db675e1ee6569ce9d7614e1 Mon Sep 17 00:00:00 2001 From: optimass Date: Tue, 14 Jan 2025 19:59:50 +0000 Subject: [PATCH 3/3] clean up --- src/agentlab/experiments/study.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/agentlab/experiments/study.py b/src/agentlab/experiments/study.py index b08b11d0..38283290 100644 --- a/src/agentlab/experiments/study.py +++ b/src/agentlab/experiments/study.py @@ -406,8 +406,6 @@ def load(dir: Path) -> "Study": inspect_results.yield_all_exp_results(savedir_base=dir, progress_fn=None) ) benchmark_name = first_result.exp_args.env_args.task_name.split(".")[0] - # if "" - agent_args = first_result.exp_args.agent_args study = Study(agent_args=agent_args, benchmark=benchmark_name, dir=dir) else: