From c5604f6751df7625d34e1c36600068e5b73b2a75 Mon Sep 17 00:00:00 2001 From: Solveig Date: Thu, 6 Feb 2025 14:24:33 +0100 Subject: [PATCH 1/5] fixed error in F1.py --- utils/metrics/F1.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/utils/metrics/F1.py b/utils/metrics/F1.py index 1e0e795..1a0cee9 100644 --- a/utils/metrics/F1.py +++ b/utils/metrics/F1.py @@ -46,7 +46,7 @@ def __init__(self, num_classes): self.fp = torch.zeros(num_classes) self.fn = torch.zeros(num_classes) - def update(self, preds, target): + def forward(self, preds, target): """ Update the variables with predictions and true labels. @@ -66,17 +66,6 @@ def update(self, preds, target): self.fp[i] += torch.sum((preds == i) & (target != i)).float() self.fn[i] += torch.sum((preds != i) & (target == i)).float() - def compute(self): - """ - Compute the F1 score. - - Returns - ------- - torch.Tensor - The computed F1 score. - """ - - # Compute F1 score based on the specified averaging method f1_score = ( 2 * torch.sum(self.tp) From 4f8725c2ab283bd68614b5b687b49eb4e1e1ca0c Mon Sep 17 00:00:00 2001 From: Solveig Date: Thu, 6 Feb 2025 14:28:32 +0100 Subject: [PATCH 2/5] updated F1 test function --- tests/test_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index d6da0ab..97d651a 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -26,7 +26,7 @@ def test_f1score(): target = torch.tensor([0, 1, 0, 2]) - f1_metric.update(preds, target) + f1_metric(preds, target) assert f1_metric.tp.sum().item() > 0, "Expected some true positives." assert f1_metric.fp.sum().item() > 0, "Expected some false positives." assert f1_metric.fn.sum().item() > 0, "Expected some false negatives." From 26ff680a930c6363cfff528e531ae88b5068d598 Mon Sep 17 00:00:00 2001 From: Solveig Date: Mon, 10 Feb 2025 10:51:20 +0100 Subject: [PATCH 3/5] added micro/macro averaging option to F1 --- utils/metrics/F1.py | 100 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 82 insertions(+), 18 deletions(-) diff --git a/utils/metrics/F1.py b/utils/metrics/F1.py index 1a0cee9..0c7a5e2 100644 --- a/utils/metrics/F1.py +++ b/utils/metrics/F1.py @@ -4,29 +4,39 @@ class F1Score(nn.Module): """ - F1 Score implementation with direct averaging inside the compute method. + F1 Score implementation with support for both macro and micro averaging. + + This class computes the F1 score during training using either macro or micro averaging. + The F1 score is calculated based on the true positives (TP), false positives (FP), + and false negatives (FN) for each class. Parameters ---------- num_classes : int - Number of classes. + The number of classes in the classification task. + + macro_averaging : bool, optional, default=False + If True, computes the macro-averaged F1 score. If False, computes the micro-averaged F1 score. Attributes ---------- num_classes : int - The number of classes. + The number of classes in the classification task. tp : torch.Tensor - Tensor for True Positives (TP) for each class. + Tensor storing the count of True Positives (TP) for each class. fp : torch.Tensor - Tensor for False Positives (FP) for each class. + Tensor storing the count of False Positives (FP) for each class. fn : torch.Tensor - Tensor for False Negatives (FN) for each class. + Tensor storing the count of False Negatives (FN) for each class. + + macro_averaging : bool + A flag indicating whether to compute the macro-averaged F1 score or not. """ - def __init__(self, num_classes): + def __init__(self, num_classes, macro_averaging=False): """ Initializes the F1Score object, setting up the necessary state variables. @@ -35,28 +45,81 @@ def __init__(self, num_classes): num_classes : int The number of classes in the classification task. + macro_averaging : bool, optional, default=False + If True, computes the macro-averaged F1 score. If False, computes the micro-averaged F1 score. """ - super().__init__() self.num_classes = num_classes + self.macro_averaging = macro_averaging - # Initialize variables for True Positives (TP), False Positives (FP), and False Negatives (FN) + # Initialize variables for True Positives (TP), False Positives (FP), and False Negatives (FN) self.tp = torch.zeros(num_classes) self.fp = torch.zeros(num_classes) self.fn = torch.zeros(num_classes) + def _micro_F1(self): + """ + Compute the Micro F1 score by aggregating TP, FP, and FN across all classes. + + Micro F1 score is calculated globally by considering all predictions together, regardless of class. + + Returns + ------- + torch.Tensor + The micro-averaged F1 score. + """ + tp = torch.sum(self.tp) + fp = torch.sum(self.fp) + fn = torch.sum(self.fn) + + precision = tp / (tp + fp + 1e-8) # Avoid division by zero + recall = tp / (tp + fn + 1e-8) # Avoid division by zero + + f1 = 2 * precision * recall / (precision + recall + 1e-8) # Avoid division by zero + return f1 + + def _macro_F1(self): + """ + Compute the Macro F1 score by calculating the F1 score per class and averaging. + + Macro F1 score is calculated as the average of per-class F1 scores. This approach treats all classes equally, + regardless of their frequency. + + Returns + ------- + torch.Tensor + The macro-averaged F1 score. + """ + precision_per_class = self.tp / (self.tp + self.fp + 1e-8) # Avoid division by zero + recall_per_class = self.tp / (self.tp + self.fn + 1e-8) # Avoid division by zero + f1_per_class = 2 * precision_per_class * recall_per_class / ( + precision_per_class + recall_per_class + 1e-8) # Avoid division by zero + + # Take the average of F1 scores across all classes + f1_score = torch.mean(f1_per_class) + return f1_score + def forward(self, preds, target): """ - Update the variables with predictions and true labels. + Update the True Positives, False Positives, and False Negatives, and compute the F1 score. + + This method computes the F1 score based on the predictions and true labels. It can compute either the + macro-averaged or micro-averaged F1 score, depending on the `macro_averaging` flag. Parameters ---------- preds : torch.Tensor - Predicted logits (shape: [batch_size, num_classes]). + Predicted logits or class indices (shape: [batch_size, num_classes]). + These logits are typically the output of a softmax or sigmoid activation. target : torch.Tensor - True labels (shape: [batch_size]). + True labels (shape: [batch_size]), where each element is an integer representing the true class. + + Returns + ------- + torch.Tensor + The computed F1 score (either micro or macro, based on `macro_averaging`). """ preds = torch.argmax(preds, dim=1) @@ -66,10 +129,11 @@ def forward(self, preds, target): self.fp[i] += torch.sum((preds == i) & (target != i)).float() self.fn[i] += torch.sum((preds != i) & (target == i)).float() - f1_score = ( - 2 - * torch.sum(self.tp) - / (2 * torch.sum(self.tp) + torch.sum(self.fp) + torch.sum(self.fn)) - ) + if self.macro_averaging: + # Calculate Macro F1 score + f1_score = self._macro_F1() + else: + # Calculate Micro F1 score + f1_score = self._micro_F1() - return f1_score + return f1_score \ No newline at end of file From 18dfea2a176772198d24119f28f5540e4b7a138c Mon Sep 17 00:00:00 2001 From: Solveig Date: Tue, 11 Feb 2025 19:11:07 +0100 Subject: [PATCH 4/5] adjusted my dataloader to the new format, added test for my model --- tests/test_models.py | 17 ++++++++++++++++- utils/dataloaders/uspsh5_7_9.py | 16 +++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/test_models.py b/tests/test_models.py index efc5412..3a4d133 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,7 +1,7 @@ import pytest import torch -from utils.models import ChristianModel, JanModel +from utils.models import ChristianModel, JanModel, SolveigModel @pytest.mark.parametrize( @@ -33,3 +33,18 @@ def test_jan_model(image_shape, num_classes): assert y.shape == (n, num_classes), f"Shape: {y.shape}" + +@pytest.mark.parametrize( + "image_shape, num_classes", + [((3, 16, 16), 3), ((3, 16, 16), 7)], +) +def test_solveig_model(image_shape, num_classes): + n, c, h, w = 5, *image_shape + + model = SolveigModel(image_shape, num_classes) + + x = torch.randn(n, c, h, w) + y = model(x) + + assert y.shape == (n, num_classes), f"Shape: {y.shape}" + diff --git a/utils/dataloaders/uspsh5_7_9.py b/utils/dataloaders/uspsh5_7_9.py index 98cbd03..dc6d48b 100644 --- a/utils/dataloaders/uspsh5_7_9.py +++ b/utils/dataloaders/uspsh5_7_9.py @@ -4,6 +4,7 @@ from PIL import Image from torch.utils.data import Dataset from torchvision import transforms +from pathlib import Path class USPSH5_Digit_7_9_Dataset(Dataset): @@ -30,7 +31,7 @@ class USPSH5_Digit_7_9_Dataset(Dataset): A transform function to apply to the images. """ - def __init__(self, h5_path, mode, transform=None): + def __init__(self, data_path, train = False, transform=None): super().__init__() """ Initializes the USPS dataset by loading images and labels from the given `.h5` file. @@ -43,12 +44,13 @@ def __init__(self, h5_path, mode, transform=None): transform : callable, optional, default=None A transform function to apply on images. """ - + self.filename = "usps.h5" + path = data_path if isinstance(data_path, Path) else Path(data_path) + self.filepath = path / self.filename self.transform = transform - self.mode = mode - self.h5_path = h5_path + self.mode = "train" if train else "test" # Load the dataset from the HDF5 file - with h5py.File(self.h5_path, "r") as hf: + with h5py.File(self.filepath, "r") as hf: images = hf[self.mode]["data"][:] labels = hf[self.mode]["target"][:] @@ -105,8 +107,8 @@ def main(): # Load the dataset dataset = USPSH5_Digit_7_9_Dataset( - h5_path="C:/Users/Solveig/OneDrive/Dokumente/UiT PhD/Courses/Git/usps.h5", - mode="train", + data_path="C:/Users/Solveig/OneDrive/Dokumente/UiT PhD/Courses/Git", + train = False, transform=transform, ) data_loader = torch.utils.data.DataLoader(dataset, batch_size=2, shuffle=True) From 6e0c34590177dc205b062a7cdc3f963a1bf930db Mon Sep 17 00:00:00 2001 From: salomaestro Date: Thu, 13 Feb 2025 13:08:33 +0100 Subject: [PATCH 5/5] Ruff and Isort --- main.py | 4 ---- tests/test_models.py | 1 - utils/arg_parser.py | 2 -- utils/dataloaders/svhn.py | 2 -- utils/dataloaders/uspsh5_7_9.py | 7 ++++--- utils/metrics/F1.py | 21 +++++++++++++++------ 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/main.py b/main.py index f9a7c98..c6230eb 100644 --- a/main.py +++ b/main.py @@ -30,9 +30,7 @@ def main(): device = args.device - if "usps" in args.dataset.lower(): - transform = transforms.Compose( [ transforms.Resize((28, 28)), @@ -47,7 +45,6 @@ def main(): data_dir=args.datafolder, transform=transform, val_size=args.val_size, - ) train_metrics = MetricWrapper( @@ -129,7 +126,6 @@ def main(): project=args.run_name, tags=[args.modelname, args.dataset], config=args, - ) wandb.watch(model) diff --git a/tests/test_models.py b/tests/test_models.py index 7e78ab7..e94c805 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,7 +1,6 @@ import pytest import torch - from utils.models import ChristianModel, JanModel, MagnusModel, SolveigModel diff --git a/utils/arg_parser.py b/utils/arg_parser.py index 31cfcba..240226c 100644 --- a/utils/arg_parser.py +++ b/utils/arg_parser.py @@ -33,7 +33,6 @@ def get_args(): help="Whether model should be saved or not.", ) - # Data/Model specific values parser.add_argument( "--modelname", @@ -83,7 +82,6 @@ def get_args(): "--macro_averaging", action="store_true", help="If the flag is included, the metrics will be calculated using macro averaging.", - ) # Training specific values diff --git a/utils/dataloaders/svhn.py b/utils/dataloaders/svhn.py index 14e2edb..e48b517 100644 --- a/utils/dataloaders/svhn.py +++ b/utils/dataloaders/svhn.py @@ -1,6 +1,5 @@ import os - import h5py import numpy as np from PIL import Image @@ -95,7 +94,6 @@ def __getitem__(self, index): img = Image.fromarray(h5f["images"][index]) if self.nr_channels == 1: - img = img.convert("L") if self.transforms is not None: img = self.transforms(img) diff --git a/utils/dataloaders/uspsh5_7_9.py b/utils/dataloaders/uspsh5_7_9.py index dc6d48b..4d63255 100644 --- a/utils/dataloaders/uspsh5_7_9.py +++ b/utils/dataloaders/uspsh5_7_9.py @@ -1,10 +1,11 @@ +from pathlib import Path + import h5py import numpy as np import torch from PIL import Image from torch.utils.data import Dataset from torchvision import transforms -from pathlib import Path class USPSH5_Digit_7_9_Dataset(Dataset): @@ -31,7 +32,7 @@ class USPSH5_Digit_7_9_Dataset(Dataset): A transform function to apply to the images. """ - def __init__(self, data_path, train = False, transform=None): + def __init__(self, data_path, train=False, transform=None): super().__init__() """ Initializes the USPS dataset by loading images and labels from the given `.h5` file. @@ -108,7 +109,7 @@ def main(): # Load the dataset dataset = USPSH5_Digit_7_9_Dataset( data_path="C:/Users/Solveig/OneDrive/Dokumente/UiT PhD/Courses/Git", - train = False, + train=False, transform=transform, ) data_loader = torch.utils.data.DataLoader(dataset, batch_size=2, shuffle=True) diff --git a/utils/metrics/F1.py b/utils/metrics/F1.py index 509c4ba..0833389 100644 --- a/utils/metrics/F1.py +++ b/utils/metrics/F1.py @@ -76,7 +76,9 @@ def _micro_F1(self): precision = tp / (tp + fp + 1e-8) # Avoid division by zero recall = tp / (tp + fn + 1e-8) # Avoid division by zero - f1 = 2 * precision * recall / (precision + recall + 1e-8) # Avoid division by zero + f1 = ( + 2 * precision * recall / (precision + recall + 1e-8) + ) # Avoid division by zero return f1 def _macro_F1(self): @@ -91,10 +93,18 @@ def _macro_F1(self): torch.Tensor The macro-averaged F1 score. """ - precision_per_class = self.tp / (self.tp + self.fp + 1e-8) # Avoid division by zero - recall_per_class = self.tp / (self.tp + self.fn + 1e-8) # Avoid division by zero - f1_per_class = 2 * precision_per_class * recall_per_class / ( - precision_per_class + recall_per_class + 1e-8) # Avoid division by zero + precision_per_class = self.tp / ( + self.tp + self.fp + 1e-8 + ) # Avoid division by zero + recall_per_class = self.tp / ( + self.tp + self.fn + 1e-8 + ) # Avoid division by zero + f1_per_class = ( + 2 + * precision_per_class + * recall_per_class + / (precision_per_class + recall_per_class + 1e-8) + ) # Avoid division by zero # Take the average of F1 scores across all classes f1_score = torch.mean(f1_per_class) @@ -138,4 +148,3 @@ def forward(self, preds, target): f1_score = self._micro_F1() return f1_score -