diff --git a/compressai_vision/evaluators/evaluators.py b/compressai_vision/evaluators/evaluators.py index 0ed5e3f..3069952 100644 --- a/compressai_vision/evaluators/evaluators.py +++ b/compressai_vision/evaluators/evaluators.py @@ -176,7 +176,10 @@ def results(self, save_path: str = None): self.write_results(out) - overall_mse = None + # summary = {} + # for key, item_dict in out.items(): + # summary[f"{key}"] = item_dict["AP"] + if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -205,11 +208,9 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - # summary = {} - # for key, item_dict in out.items(): - # summary[f"{key}"] = item_dict["AP"] - - return out, overall_mse + return out, overall_mse + else: + return out @register_evaluator("OIC-EVAL") @@ -474,7 +475,11 @@ def results(self, save_path: str = None): self.write_results(out) - overall_mse = None + summary = {} + for key, value in out.items(): + name = "-".join(key.split("/")[1:]) + summary[name] = value + if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -503,12 +508,9 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - summary = {} - for key, value in out.items(): - name = "-".join(key.split("/")[1:]) - summary[name] = value - - return summary, overall_mse + return summary, overall_mse + else: + return summary @register_evaluator("SEMANTICSEG-EVAL") @@ -607,7 +609,6 @@ def results(self, save_path: str = None): self.write_results(class_mIoU) - overall_mse = None if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -636,7 +637,9 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - return class_mIoU, overall_mse + return class_mIoU, overall_mse + else: + return class_mIoU @register_evaluator("MOT-JDE-EVAL") @@ -788,7 +791,6 @@ def results(self, save_path: str = None): self.write_results(out) - overall_mse = None if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -817,7 +819,9 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - return out, overall_mse + return out, overall_mse + else: + return out @staticmethod def digest_summary(summary): @@ -1126,7 +1130,10 @@ def results(self, save_path: str = None): self.write_results(eval_results) - overall_mse = None + *listed_items, summary = eval_results + + self._logger.info("\n" + summary) + if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -1155,11 +1162,12 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - *listed_items, summary = eval_results - - self._logger.info("\n" + summary) - - return {"AP": listed_items[0] * 100, "AP50": listed_items[1] * 100}, overall_mse + return { + "AP": listed_items[0] * 100, + "AP50": listed_items[1] * 100, + }, overall_mse + else: + return {"AP": listed_items[0] * 100, "AP50": listed_items[1] * 100} def _convert_to_coco_format(self, outputs, info_imgs, ids): # reference : yolox > evaluators > coco_evaluator > convert_to_coco_format @@ -1363,7 +1371,11 @@ def results(self, save_path: str = None): self.write_results(eval_results) - overall_mse = None + # item_keys = list(eval_results.keys()) + item_vals = list(eval_results.values()) + + # self._logger.info("\n" + summary) + if self._mse_results: mse_results_dict = {"per_frame_mse": self._mse_results} overall_mse = 0.0 @@ -1392,12 +1404,9 @@ def results(self, save_path: str = None): ) as f: json.dump(mse_results_dict, f, ensure_ascii=False, indent=4) - # item_keys = list(eval_results.keys()) - item_vals = list(eval_results.values()) - - # self._logger.info("\n" + summary) - - return {"AP": item_vals[0] * 100, "AP50": item_vals[1] * 100}, overall_mse + return {"AP": item_vals[0] * 100, "AP50": item_vals[1] * 100}, overall_mse + else: + return {"AP": item_vals[0] * 100, "AP50": item_vals[1] * 100} @register_evaluator("VISUAL-QUALITY-EVAL") @@ -1444,7 +1453,7 @@ def write_results(self, path: str = None): with open(f"{path}/{self.output_file_name}.json", "w", encoding="utf-8") as f: json.dump(self._evaluations, f, ensure_ascii=False, indent=4) - def digest(self, gt, pred): + def digest(self, gt, pred, mse_results=None): ref = gt[0]["image"].unsqueeze(0).cpu() tst = pred.unsqueeze(0).cpu() diff --git a/compressai_vision/pipelines/remote_inference/video_remote_inference.py b/compressai_vision/pipelines/remote_inference/video_remote_inference.py index e051322..a74a742 100644 --- a/compressai_vision/pipelines/remote_inference/video_remote_inference.py +++ b/compressai_vision/pipelines/remote_inference/video_remote_inference.py @@ -186,7 +186,7 @@ def __call__( start = time_measure() dec_d = { "file_name": dec_seq["file_names"][e], - "file_origin": d[e]["file_name"], + "file_origin": d[0]["file_name"], } # dec_d = {"file_name": dec_seq[0]["file_names"][e]} pred = vision_model.forward(org_map_func(dec_d)) diff --git a/compressai_vision/run/eval_split_inference.py b/compressai_vision/run/eval_split_inference.py index 3a795ea..5f86d24 100644 --- a/compressai_vision/run/eval_split_inference.py +++ b/compressai_vision/run/eval_split_inference.py @@ -267,7 +267,9 @@ def main(conf: DictConfig): "avg_bpp": avg_bpp, "end_accuracy": performance, **elap_times, - "inv_mse": None if mse is None else 1.0 / mse, + **( + {"inv_mse": 0 if mse == 0 else 1.0 / mse} if mse is not None else {} + ), } ) print(tabulate(result_df, headers="keys", tablefmt="psql")) @@ -283,7 +285,9 @@ def main(conf: DictConfig): "bitrate (kbps)": bitrate, "end_accuracy": performance, **elap_times, - "inv_mse": None if mse is None else 1.0 / mse, + **( + {"inv_mse": 0 if mse == 0 else 1.0 / mse} if mse is not None else {} + ), } ) print(tabulate(result_df, headers="keys", tablefmt="psql")) diff --git a/scripts/metrics/compute_overall_map.py b/scripts/metrics/compute_overall_map.py index 9d38617..e2d57ef 100644 --- a/scripts/metrics/compute_overall_map.py +++ b/scripts/metrics/compute_overall_map.py @@ -54,7 +54,7 @@ from compressai_vision.evaluators.evaluators import BaseEvaluator -CLASSES = ["CLASS-AB", "CLASS-C", "CLASS-D"] +CLASSES = ["CLASS-AB", "CLASS-C", "CLASS-D", "CLASS-AB*"] SEQS_BY_CLASS = { CLASSES[0]: [ @@ -67,6 +67,7 @@ ], CLASSES[1]: ["BasketballDrill", "BQMall", "PartyScene", "RaceHorses_832x480"], CLASSES[2]: ["BasketballPass", "BQSquare", "BlowingBubbles", "RaceHorses"], + CLASSES[3]: ["Traffic", "BQTerrace"], } SEQUENCE_TO_OFFSET = { diff --git a/scripts/metrics/gen_mpeg_cttc_csv.py b/scripts/metrics/gen_mpeg_cttc_csv.py index 708d090..88177fe 100644 --- a/scripts/metrics/gen_mpeg_cttc_csv.py +++ b/scripts/metrics/gen_mpeg_cttc_csv.py @@ -58,7 +58,13 @@ DATASETS = ["TVD", "SFU", "OIV6", "HIEVE", "PANDASET"] -def read_df_rec(path, seq_list, nb_operation_points, fn_regex=r"summary.csv"): +def read_df_rec( + path, + seq_list, + nb_operation_points, + fn_regex=r"summary.csv", + prefix: str | None = None, +): summary_csvs = [f for f in iglob(join(path, "**", fn_regex), recursive=True)] if nb_operation_points > 0: seq_names = [ @@ -70,10 +76,18 @@ def read_df_rec(path, seq_list, nb_operation_points, fn_regex=r"summary.csv"): len([f for f in summary_csvs if sequence in f]) == nb_operation_points ), f"Did not find {nb_operation_points} results for {sequence}" - return pd.concat( - (pd.read_csv(f) for f in summary_csvs), - ignore_index=True, - ) + dfs = [] + for f in summary_csvs: + df = pd.read_csv(f) + + seq_dir = Path(os.path.relpath(f, path)).parts[0] + if prefix and prefix in seq_dir: + df["Dataset"] = df["Dataset"].apply( + lambda x: f"{prefix}{x}" if not x.startswith(f"{prefix}") else x + ) + + dfs.append(df) + return pd.concat(dfs, ignore_index=True) def df_append(df1, df2): @@ -155,9 +169,13 @@ def generate_csv_classwise_video_map( nb_operation_points: int = 4, no_cactus: bool = False, skip_classwise: bool = False, + seq_prefix: str = None, + dataset_prefix: str = None, ): opts_metrics = {"AP": 0, "AP50": 1, "AP75": 2, "APS": 3, "APM": 4, "APL": 5} - results_df = read_df_rec(result_path, seq_list, nb_operation_points) + results_df = read_df_rec( + result_path, seq_list, nb_operation_points, prefix=seq_prefix + ) # sort sorterIndex = dict(zip(seq_list, range(len(seq_list)))) @@ -177,6 +195,13 @@ def generate_csv_classwise_video_map( classwise_name = list(seqs_by_class.keys())[0] classwise_seqs = list(seqs_by_class.values())[0] + cur_seq_prefix = ( + seq_prefix + if seq_prefix + and any(name.startswith(seq_prefix) for name in classwise_seqs) + else None + ) + class_wise_maps = [] for q in range(nb_operation_points): items = utils.search_items( @@ -187,6 +212,8 @@ def generate_csv_classwise_video_map( BaseEvaluator.get_coco_eval_info_name, by_name=True, gt_folder=gt_folder, + seq_prefix=cur_seq_prefix, + dataset_prefix=dataset_prefix, ) assert ( @@ -202,7 +229,11 @@ def generate_csv_classwise_video_map( matched_seq_names = [] for seq_info in items: name, _, _ = get_seq_info(seq_info[utils.SEQ_INFO_KEY]) - matched_seq_names.append(name) + matched_seq_names.append( + f"{seq_prefix}{name}" + if seq_prefix and seq_prefix in seq_info[utils.SEQ_NAME_KEY] + else name + ) class_wise_results_df = generate_classwise_df( results_df, {classwise_name: matched_seq_names} @@ -409,6 +440,12 @@ def generate_csv(result_path, seq_list, nb_operation_points): default=False, help="exclude Cactus sequence for FCM eval", ) + parser.add_argument( + "--add-non-scale", + action="store_true", + default=False, + help="Add non-scale option using ns_Traffic/ns_BQTerrace with original GT", + ) args = parser.parse_args() @@ -421,6 +458,7 @@ def generate_csv(result_path, seq_list, nb_operation_points): if args.dataset_name == "SFU": metric = args.metric + dataset_prefix = "sfu-hw-" class_ab = { "CLASS-AB": [ "Traffic", @@ -476,6 +514,19 @@ def generate_csv(result_path, seq_list, nb_operation_points): "BlowingBubbles_416x240_50", "RaceHorses_416x240_30", ] + + if args.mode == "FCM" and args.add_non_scale: + ns_seq_list = ["ns_Traffic_2560x1600_30", "ns_BQTerrace_1920x1080_60"] + seq_list.extend(ns_seq_list) + seq_prefix = "ns_" + class_ab_star = { + "CLASS-AB*": [ + "ns_Traffic", + "ns_BQTerrace", + ] + } + classes.append(class_ab_star) + if args.mode == "VCM" and not args.include_optional: seq_list.remove("Kimono_1920x1080_24") seq_list.remove("Cactus_1920x1080_50") @@ -493,6 +544,10 @@ def generate_csv(result_path, seq_list, nb_operation_points): args.nb_operation_points, args.no_cactus, args.mode == "VCM", # skip classwise evaluation + seq_prefix=seq_prefix + if "seq_prefix" in locals() + else None, # adding prefix to non-scale sequence + dataset_prefix=dataset_prefix if "dataset_prefix" in locals() else None, ) if args.mode == "VCM": diff --git a/scripts/metrics/utils.py b/scripts/metrics/utils.py index 65fea7d..8cc83ea 100644 --- a/scripts/metrics/utils.py +++ b/scripts/metrics/utils.py @@ -40,6 +40,7 @@ import re from pathlib import Path +from typing import Dict, Optional __all__ = [ "get_seq_number", @@ -85,8 +86,10 @@ def get_eval_info_path_by_seq_num(seq_num, _path, qidx: int, name_func: callable return eval_info_path, dname -def get_eval_info_path_by_seq_name(seq_name, _path, _qidx: int, name_func: callable): - result = get_folder_path_by_seq_name(seq_name, _path) +def get_eval_info_path_by_seq_name( + seq_name, _path, _qidx: int, name_func: callable, dataset_prefix=None +): + result = get_folder_path_by_seq_name(seq_name, _path, dataset_prefix) if result is None: return eval_folder, _dname = result @@ -155,9 +158,12 @@ def get_folder_path_by_seq_num(seq_num, _path): return None -def get_folder_path_by_seq_name(seq_name, _path): +def get_folder_path_by_seq_name(seq_name, _path, dataset_prefix=None): _folder_list = [f for f in Path(_path).iterdir() if f.is_dir()] + if dataset_prefix is not None: + seq_name = f"{dataset_prefix}{seq_name}" + for _name in _folder_list: if seq_name in _name.stem: return _name.resolve(), _name.stem @@ -174,12 +180,14 @@ def search_items( by_name=False, pandaset_flag=False, gt_folder="annotations", + seq_prefix=None, + dataset_prefix=None, ): _ret_list = [] for seq_name in seq_list: if by_name is True: result = get_eval_info_path_by_seq_name( - seq_name, result_path, rate_point, eval_func + seq_name, result_path, rate_point, eval_func, dataset_prefix ) if result is None: continue @@ -189,8 +197,11 @@ def search_items( seq_name, dataset_path, gt_folder ) else: + _gt_lookup_name = ( + seq_name.split(seq_prefix)[-1] if seq_prefix else seq_name + ) seq_info_path, seq_gt_path = get_seq_info_path_by_seq_name( - seq_name, dataset_path, gt_folder + _gt_lookup_name, dataset_path, gt_folder ) else: # by number seq_num = get_seq_number(seq_name)