From 140ac616f899b48bad3601f92f5598dff73d1483 Mon Sep 17 00:00:00 2001 From: Jianwen Song Date: Thu, 23 Oct 2025 11:25:29 +1100 Subject: [PATCH 1/4] [feat] evaluation script for adding non-scale cases --- scripts/metrics/compute_overall_map.py | 3 +- scripts/metrics/gen_mpeg_cttc_csv.py | 82 +++++++++++++++++++++++--- scripts/metrics/utils.py | 9 ++- 3 files changed, 84 insertions(+), 10 deletions(-) diff --git a/scripts/metrics/compute_overall_map.py b/scripts/metrics/compute_overall_map.py index 9d38617..ad862f2 100644 --- a/scripts/metrics/compute_overall_map.py +++ b/scripts/metrics/compute_overall_map.py @@ -54,7 +54,7 @@ from compressai_vision.evaluators.evaluators import BaseEvaluator -CLASSES = ["CLASS-AB", "CLASS-C", "CLASS-D"] +CLASSES = ["CLASS-AB", "CLASS-C", "CLASS-D", "CLASS-AB*"] SEQS_BY_CLASS = { CLASSES[0]: [ @@ -67,6 +67,7 @@ ], CLASSES[1]: ["BasketballDrill", "BQMall", "PartyScene", "RaceHorses_832x480"], CLASSES[2]: ["BasketballPass", "BQSquare", "BlowingBubbles", "RaceHorses"], + CLASSES[3]: ["Traffic", "BQTerrace"] } SEQUENCE_TO_OFFSET = { diff --git a/scripts/metrics/gen_mpeg_cttc_csv.py b/scripts/metrics/gen_mpeg_cttc_csv.py index 708d090..b711605 100644 --- a/scripts/metrics/gen_mpeg_cttc_csv.py +++ b/scripts/metrics/gen_mpeg_cttc_csv.py @@ -58,7 +58,7 @@ DATASETS = ["TVD", "SFU", "OIV6", "HIEVE", "PANDASET"] -def read_df_rec(path, seq_list, nb_operation_points, fn_regex=r"summary.csv"): +def read_df_rec(path, seq_list, nb_operation_points, fn_regex=r"summary.csv", prefix: str | None = None,): summary_csvs = [f for f in iglob(join(path, "**", fn_regex), recursive=True)] if nb_operation_points > 0: seq_names = [ @@ -70,10 +70,18 @@ def read_df_rec(path, seq_list, nb_operation_points, fn_regex=r"summary.csv"): len([f for f in summary_csvs if sequence in f]) == nb_operation_points ), f"Did not find {nb_operation_points} results for {sequence}" - return pd.concat( - (pd.read_csv(f) for f in summary_csvs), - ignore_index=True, - ) + dfs = [] + for f in summary_csvs: + df = pd.read_csv(f) + + seq_dir = Path(os.path.relpath(f, path)).parts[0] + if prefix and prefix in seq_dir: + df["Dataset"] = df["Dataset"].apply( + lambda x: f"{prefix}{x}" if not x.startswith(f"{prefix}") else x + ) + + dfs.append(df) + return pd.concat(dfs, ignore_index=True) def df_append(df1, df2): @@ -155,9 +163,10 @@ def generate_csv_classwise_video_map( nb_operation_points: int = 4, no_cactus: bool = False, skip_classwise: bool = False, + prefix: str | None = None, ): opts_metrics = {"AP": 0, "AP50": 1, "AP75": 2, "APS": 3, "APM": 4, "APL": 5} - results_df = read_df_rec(result_path, seq_list, nb_operation_points) + results_df = read_df_rec(result_path, seq_list, nb_operation_points, prefix=prefix) # sort sorterIndex = dict(zip(seq_list, range(len(seq_list)))) @@ -211,7 +220,7 @@ def generate_csv_classwise_video_map( output_df = df_append(output_df, class_wise_results_df) - return output_df + return output_df, results_df def generate_csv_classwise_video_mota( @@ -409,6 +418,12 @@ def generate_csv(result_path, seq_list, nb_operation_points): default=False, help="exclude Cactus sequence for FCM eval", ) + parser.add_argument( + "--add-non-scale", + action="store_true", + default=False, + help="Add non-scale option using ns_Traffic/ns_BQTerrace with original GT", + ) args = parser.parse_args() @@ -476,6 +491,12 @@ def generate_csv(result_path, seq_list, nb_operation_points): "BlowingBubbles_416x240_50", "RaceHorses_416x240_30", ] + + if args.mode == "FCM" and args.add_non_scale: + ns_seq_list = ["ns_Traffic_2560x1600_30", "ns_BQTerrace_1920x1080_60"] + seq_list.extend(ns_seq_list) + seq_prefix = "ns_" + if args.mode == "VCM" and not args.include_optional: seq_list.remove("Kimono_1920x1080_24") seq_list.remove("Cactus_1920x1080_50") @@ -483,7 +504,7 @@ def generate_csv(result_path, seq_list, nb_operation_points): if args.mode == "FCM" and args.no_cactus: seq_list.remove("Cactus_1920x1080_50") - output_df = generate_csv_classwise_video_map( + output_df, results_df = generate_csv_classwise_video_map( norm_result_path, args.dataset_path, classes, @@ -493,6 +514,7 @@ def generate_csv(result_path, seq_list, nb_operation_points): args.nb_operation_points, args.no_cactus, args.mode == "VCM", # skip classwise evaluation + prefix=seq_prefix if "seq_prefix" in locals() else None, # adding prefix to non-scale sequence ) if args.mode == "VCM": @@ -502,6 +524,50 @@ def generate_csv(result_path, seq_list, nb_operation_points): perf_name="end_accuracy", rate_name="bitrate (kbps)", ) + else: + # add CLASS-AB* using ns_* results for Traffic and BQTerrace + if args.add_non_scale: + class_ab_star = { + "CLASS-AB*": [ + "ns_Traffic", + "ns_BQTerrace", + ] + } + # Compute classwise mAP for AB* using ns_* eval results but original GT + class_wise_maps = [] + for q in range(args.nb_operation_points): + items = utils.search_items( + norm_result_path, + args.dataset_path, + q, + list(class_ab_star.values())[0], + BaseEvaluator.get_coco_eval_info_name, + by_name=True, + gt_folder=args.gt_folder, + gt_name_overrides={ + "ns_Traffic": "Traffic", + "ns_BQTerrace": "BQTerrace", + }, + ) + + assert ( + len(items) > 0 + ), "No evaluation information found for CLASS-AB* in provided result directories..." + + summary = compute_overall_mAP("CLASS-AB*", items) + maps = summary.values[0][{"AP": 0, "AP50": 1}[metric]] + class_wise_maps.append(maps) + + matched_seq_names = [] + for seq_info in items: + name, _, _ = get_seq_info(seq_info[utils.SEQ_INFO_KEY]) + matched_seq_names.append(name) + + class_wise_results_df = generate_classwise_df( + results_df, {"CLASS-AB*": matched_seq_names} + ) + class_wise_results_df["end_accuracy"] = class_wise_maps + output_df = df_append(output_df, class_wise_results_df) elif args.dataset_name == "OIV6": output_df = generate_csv( norm_result_path, ["MPEGOIV6"], args.nb_operation_points diff --git a/scripts/metrics/utils.py b/scripts/metrics/utils.py index 65fea7d..e6f62c3 100644 --- a/scripts/metrics/utils.py +++ b/scripts/metrics/utils.py @@ -40,6 +40,7 @@ import re from pathlib import Path +from typing import Optional, Dict __all__ = [ "get_seq_number", @@ -174,6 +175,7 @@ def search_items( by_name=False, pandaset_flag=False, gt_folder="annotations", + gt_name_overrides: Optional[Dict[str, str]] = None, ): _ret_list = [] for seq_name in seq_list: @@ -189,8 +191,13 @@ def search_items( seq_name, dataset_path, gt_folder ) else: + _gt_lookup_name = ( + gt_name_overrides.get(seq_name, seq_name) + if gt_name_overrides + else seq_name + ) seq_info_path, seq_gt_path = get_seq_info_path_by_seq_name( - seq_name, dataset_path, gt_folder + _gt_lookup_name, dataset_path, gt_folder ) else: # by number seq_num = get_seq_number(seq_name) From 0f3e099c8e60120860062b5dca9ce1395e89ec88 Mon Sep 17 00:00:00 2001 From: Jianwen Song Date: Thu, 23 Oct 2025 11:51:06 +1100 Subject: [PATCH 2/4] [fix] mse error for other inferences --- compressai_vision/evaluators/evaluators.py | 69 +++++++++++-------- compressai_vision/run/eval_split_inference.py | 8 ++- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/compressai_vision/evaluators/evaluators.py b/compressai_vision/evaluators/evaluators.py index 0ed5e3f..55ff967 100644 --- a/compressai_vision/evaluators/evaluators.py +++ b/compressai_vision/evaluators/evaluators.py @@ -176,7 +176,10 @@ def results(self, save_path: str = None): self.write_results(out) - overall_mse = None + # summary = {} + # for key, item_dict in out.items(): + # summary[f"{key}"] = item_dict["AP"] + if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -205,11 +208,9 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - # summary = {} - # for key, item_dict in out.items(): - # summary[f"{key}"] = item_dict["AP"] - - return out, overall_mse + return out, overall_mse + else: + return out @register_evaluator("OIC-EVAL") @@ -474,7 +475,11 @@ def results(self, save_path: str = None): self.write_results(out) - overall_mse = None + summary = {} + for key, value in out.items(): + name = "-".join(key.split("/")[1:]) + summary[name] = value + if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -503,12 +508,9 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - summary = {} - for key, value in out.items(): - name = "-".join(key.split("/")[1:]) - summary[name] = value - - return summary, overall_mse + return summary, overall_mse + else: + return summary @register_evaluator("SEMANTICSEG-EVAL") @@ -607,7 +609,6 @@ def results(self, save_path: str = None): self.write_results(class_mIoU) - overall_mse = None if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -636,7 +637,9 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - return class_mIoU, overall_mse + return class_mIoU, overall_mse + else: + return class_mIoU @register_evaluator("MOT-JDE-EVAL") @@ -788,7 +791,6 @@ def results(self, save_path: str = None): self.write_results(out) - overall_mse = None if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -817,7 +819,9 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - return out, overall_mse + return out, overall_mse + else: + return out @staticmethod def digest_summary(summary): @@ -1126,7 +1130,10 @@ def results(self, save_path: str = None): self.write_results(eval_results) - overall_mse = None + *listed_items, summary = eval_results + + self._logger.info("\n" + summary) + if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -1155,11 +1162,12 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - *listed_items, summary = eval_results - - self._logger.info("\n" + summary) - - return {"AP": listed_items[0] * 100, "AP50": listed_items[1] * 100}, overall_mse + return { + "AP": listed_items[0] * 100, + "AP50": listed_items[1] * 100, + }, overall_mse + else: + return {"AP": listed_items[0] * 100, "AP50": listed_items[1] * 100} def _convert_to_coco_format(self, outputs, info_imgs, ids): # reference : yolox > evaluators > coco_evaluator > convert_to_coco_format @@ -1363,7 +1371,11 @@ def results(self, save_path: str = None): self.write_results(eval_results) - overall_mse = None + # item_keys = list(eval_results.keys()) + item_vals = list(eval_results.values()) + + # self._logger.info("\n" + summary) + if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -1392,12 +1404,9 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - # item_keys = list(eval_results.keys()) - item_vals = list(eval_results.values()) - - # self._logger.info("\n" + summary) - - return {"AP": item_vals[0] * 100, "AP50": item_vals[1] * 100}, overall_mse + return {"AP": item_vals[0] * 100, "AP50": item_vals[1] * 100}, overall_mse + else: + return {"AP": item_vals[0] * 100, "AP50": item_vals[1] * 100} @register_evaluator("VISUAL-QUALITY-EVAL") diff --git a/compressai_vision/run/eval_split_inference.py b/compressai_vision/run/eval_split_inference.py index 3a795ea..5f86d24 100644 --- a/compressai_vision/run/eval_split_inference.py +++ b/compressai_vision/run/eval_split_inference.py @@ -267,7 +267,9 @@ def main(conf: DictConfig): "avg_bpp": avg_bpp, "end_accuracy": performance, **elap_times, - "inv_mse": None if mse is None else 1.0 / mse, + **( + {"inv_mse": 0 if mse == 0 else 1.0 / mse} if mse is not None else {} + ), } ) print(tabulate(result_df, headers="keys", tablefmt="psql")) @@ -283,7 +285,9 @@ def main(conf: DictConfig): "bitrate (kbps)": bitrate, "end_accuracy": performance, **elap_times, - "inv_mse": None if mse is None else 1.0 / mse, + **( + {"inv_mse": 0 if mse == 0 else 1.0 / mse} if mse is not None else {} + ), } ) print(tabulate(result_df, headers="keys", tablefmt="psql")) From c6b47a51d9c641ada5f0a0e75cd2c6290cf46eb6 Mon Sep 17 00:00:00 2001 From: Jianwen Song Date: Thu, 23 Oct 2025 11:57:23 +1100 Subject: [PATCH 3/4] [fix] argument in evaluator --- compressai_vision/evaluators/evaluators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compressai_vision/evaluators/evaluators.py b/compressai_vision/evaluators/evaluators.py index 55ff967..3069952 100644 --- a/compressai_vision/evaluators/evaluators.py +++ b/compressai_vision/evaluators/evaluators.py @@ -1453,7 +1453,7 @@ def write_results(self, path: str = None): with open(f"{path}/{self.output_file_name}.json", "w", encoding="utf-8") as f: json.dump(self._evaluations, f, ensure_ascii=False, indent=4) - def digest(self, gt, pred): + def digest(self, gt, pred, mse_results=None): ref = gt[0]["image"].unsqueeze(0).cpu() tst = pred.unsqueeze(0).cpu() From 73be367b30fdfe99aa0cb5b788a85b03bc718bac Mon Sep 17 00:00:00 2001 From: Jianwen Song Date: Thu, 23 Oct 2025 17:29:11 +1100 Subject: [PATCH 4/4] [fix] classwise results --- .../video_remote_inference.py | 2 +- scripts/metrics/compute_overall_map.py | 2 +- scripts/metrics/gen_mpeg_cttc_csv.py | 91 ++++++++----------- scripts/metrics/utils.py | 22 +++-- 4 files changed, 55 insertions(+), 62 deletions(-) diff --git a/compressai_vision/pipelines/remote_inference/video_remote_inference.py b/compressai_vision/pipelines/remote_inference/video_remote_inference.py index e051322..a74a742 100644 --- a/compressai_vision/pipelines/remote_inference/video_remote_inference.py +++ b/compressai_vision/pipelines/remote_inference/video_remote_inference.py @@ -186,7 +186,7 @@ def __call__( start = time_measure() dec_d = { "file_name": dec_seq["file_names"][e], - "file_origin": d[e]["file_name"], + "file_origin": d[0]["file_name"], } # dec_d = {"file_name": dec_seq[0]["file_names"][e]} pred = vision_model.forward(org_map_func(dec_d)) diff --git a/scripts/metrics/compute_overall_map.py b/scripts/metrics/compute_overall_map.py index ad862f2..e2d57ef 100644 --- a/scripts/metrics/compute_overall_map.py +++ b/scripts/metrics/compute_overall_map.py @@ -67,7 +67,7 @@ ], CLASSES[1]: ["BasketballDrill", "BQMall", "PartyScene", "RaceHorses_832x480"], CLASSES[2]: ["BasketballPass", "BQSquare", "BlowingBubbles", "RaceHorses"], - CLASSES[3]: ["Traffic", "BQTerrace"] + CLASSES[3]: ["Traffic", "BQTerrace"], } SEQUENCE_TO_OFFSET = { diff --git a/scripts/metrics/gen_mpeg_cttc_csv.py b/scripts/metrics/gen_mpeg_cttc_csv.py index b711605..88177fe 100644 --- a/scripts/metrics/gen_mpeg_cttc_csv.py +++ b/scripts/metrics/gen_mpeg_cttc_csv.py @@ -58,7 +58,13 @@ DATASETS = ["TVD", "SFU", "OIV6", "HIEVE", "PANDASET"] -def read_df_rec(path, seq_list, nb_operation_points, fn_regex=r"summary.csv", prefix: str | None = None,): +def read_df_rec( + path, + seq_list, + nb_operation_points, + fn_regex=r"summary.csv", + prefix: str | None = None, +): summary_csvs = [f for f in iglob(join(path, "**", fn_regex), recursive=True)] if nb_operation_points > 0: seq_names = [ @@ -163,10 +169,13 @@ def generate_csv_classwise_video_map( nb_operation_points: int = 4, no_cactus: bool = False, skip_classwise: bool = False, - prefix: str | None = None, + seq_prefix: str = None, + dataset_prefix: str = None, ): opts_metrics = {"AP": 0, "AP50": 1, "AP75": 2, "APS": 3, "APM": 4, "APL": 5} - results_df = read_df_rec(result_path, seq_list, nb_operation_points, prefix=prefix) + results_df = read_df_rec( + result_path, seq_list, nb_operation_points, prefix=seq_prefix + ) # sort sorterIndex = dict(zip(seq_list, range(len(seq_list)))) @@ -186,6 +195,13 @@ def generate_csv_classwise_video_map( classwise_name = list(seqs_by_class.keys())[0] classwise_seqs = list(seqs_by_class.values())[0] + cur_seq_prefix = ( + seq_prefix + if seq_prefix + and any(name.startswith(seq_prefix) for name in classwise_seqs) + else None + ) + class_wise_maps = [] for q in range(nb_operation_points): items = utils.search_items( @@ -196,6 +212,8 @@ def generate_csv_classwise_video_map( BaseEvaluator.get_coco_eval_info_name, by_name=True, gt_folder=gt_folder, + seq_prefix=cur_seq_prefix, + dataset_prefix=dataset_prefix, ) assert ( @@ -211,7 +229,11 @@ def generate_csv_classwise_video_map( matched_seq_names = [] for seq_info in items: name, _, _ = get_seq_info(seq_info[utils.SEQ_INFO_KEY]) - matched_seq_names.append(name) + matched_seq_names.append( + f"{seq_prefix}{name}" + if seq_prefix and seq_prefix in seq_info[utils.SEQ_NAME_KEY] + else name + ) class_wise_results_df = generate_classwise_df( results_df, {classwise_name: matched_seq_names} @@ -220,7 +242,7 @@ def generate_csv_classwise_video_map( output_df = df_append(output_df, class_wise_results_df) - return output_df, results_df + return output_df def generate_csv_classwise_video_mota( @@ -436,6 +458,7 @@ def generate_csv(result_path, seq_list, nb_operation_points): if args.dataset_name == "SFU": metric = args.metric + dataset_prefix = "sfu-hw-" class_ab = { "CLASS-AB": [ "Traffic", @@ -496,6 +519,13 @@ def generate_csv(result_path, seq_list, nb_operation_points): ns_seq_list = ["ns_Traffic_2560x1600_30", "ns_BQTerrace_1920x1080_60"] seq_list.extend(ns_seq_list) seq_prefix = "ns_" + class_ab_star = { + "CLASS-AB*": [ + "ns_Traffic", + "ns_BQTerrace", + ] + } + classes.append(class_ab_star) if args.mode == "VCM" and not args.include_optional: seq_list.remove("Kimono_1920x1080_24") @@ -504,7 +534,7 @@ def generate_csv(result_path, seq_list, nb_operation_points): if args.mode == "FCM" and args.no_cactus: seq_list.remove("Cactus_1920x1080_50") - output_df, results_df = generate_csv_classwise_video_map( + output_df = generate_csv_classwise_video_map( norm_result_path, args.dataset_path, classes, @@ -514,7 +544,10 @@ def generate_csv(result_path, seq_list, nb_operation_points): args.nb_operation_points, args.no_cactus, args.mode == "VCM", # skip classwise evaluation - prefix=seq_prefix if "seq_prefix" in locals() else None, # adding prefix to non-scale sequence + seq_prefix=seq_prefix + if "seq_prefix" in locals() + else None, # adding prefix to non-scale sequence + dataset_prefix=dataset_prefix if "dataset_prefix" in locals() else None, ) if args.mode == "VCM": @@ -524,50 +557,6 @@ def generate_csv(result_path, seq_list, nb_operation_points): perf_name="end_accuracy", rate_name="bitrate (kbps)", ) - else: - # add CLASS-AB* using ns_* results for Traffic and BQTerrace - if args.add_non_scale: - class_ab_star = { - "CLASS-AB*": [ - "ns_Traffic", - "ns_BQTerrace", - ] - } - # Compute classwise mAP for AB* using ns_* eval results but original GT - class_wise_maps = [] - for q in range(args.nb_operation_points): - items = utils.search_items( - norm_result_path, - args.dataset_path, - q, - list(class_ab_star.values())[0], - BaseEvaluator.get_coco_eval_info_name, - by_name=True, - gt_folder=args.gt_folder, - gt_name_overrides={ - "ns_Traffic": "Traffic", - "ns_BQTerrace": "BQTerrace", - }, - ) - - assert ( - len(items) > 0 - ), "No evaluation information found for CLASS-AB* in provided result directories..." - - summary = compute_overall_mAP("CLASS-AB*", items) - maps = summary.values[0][{"AP": 0, "AP50": 1}[metric]] - class_wise_maps.append(maps) - - matched_seq_names = [] - for seq_info in items: - name, _, _ = get_seq_info(seq_info[utils.SEQ_INFO_KEY]) - matched_seq_names.append(name) - - class_wise_results_df = generate_classwise_df( - results_df, {"CLASS-AB*": matched_seq_names} - ) - class_wise_results_df["end_accuracy"] = class_wise_maps - output_df = df_append(output_df, class_wise_results_df) elif args.dataset_name == "OIV6": output_df = generate_csv( norm_result_path, ["MPEGOIV6"], args.nb_operation_points diff --git a/scripts/metrics/utils.py b/scripts/metrics/utils.py index e6f62c3..8cc83ea 100644 --- a/scripts/metrics/utils.py +++ b/scripts/metrics/utils.py @@ -40,7 +40,7 @@ import re from pathlib import Path -from typing import Optional, Dict +from typing import Dict, Optional __all__ = [ "get_seq_number", @@ -86,8 +86,10 @@ def get_eval_info_path_by_seq_num(seq_num, _path, qidx: int, name_func: callable return eval_info_path, dname -def get_eval_info_path_by_seq_name(seq_name, _path, _qidx: int, name_func: callable): - result = get_folder_path_by_seq_name(seq_name, _path) +def get_eval_info_path_by_seq_name( + seq_name, _path, _qidx: int, name_func: callable, dataset_prefix=None +): + result = get_folder_path_by_seq_name(seq_name, _path, dataset_prefix) if result is None: return eval_folder, _dname = result @@ -156,9 +158,12 @@ def get_folder_path_by_seq_num(seq_num, _path): return None -def get_folder_path_by_seq_name(seq_name, _path): +def get_folder_path_by_seq_name(seq_name, _path, dataset_prefix=None): _folder_list = [f for f in Path(_path).iterdir() if f.is_dir()] + if dataset_prefix is not None: + seq_name = f"{dataset_prefix}{seq_name}" + for _name in _folder_list: if seq_name in _name.stem: return _name.resolve(), _name.stem @@ -175,13 +180,14 @@ def search_items( by_name=False, pandaset_flag=False, gt_folder="annotations", - gt_name_overrides: Optional[Dict[str, str]] = None, + seq_prefix=None, + dataset_prefix=None, ): _ret_list = [] for seq_name in seq_list: if by_name is True: result = get_eval_info_path_by_seq_name( - seq_name, result_path, rate_point, eval_func + seq_name, result_path, rate_point, eval_func, dataset_prefix ) if result is None: continue @@ -192,9 +198,7 @@ def search_items( ) else: _gt_lookup_name = ( - gt_name_overrides.get(seq_name, seq_name) - if gt_name_overrides - else seq_name + seq_name.split(seq_prefix)[-1] if seq_prefix else seq_name ) seq_info_path, seq_gt_path = get_seq_info_path_by_seq_name( _gt_lookup_name, dataset_path, gt_folder