From 728d4d84fae64d36cfde2d1eb2e6492d2aad2ec3 Mon Sep 17 00:00:00 2001 From: Tennessee Leeuwenburg Date: Wed, 26 Mar 2025 22:21:52 +1100 Subject: [PATCH 1/5] Capitalise the "N" in the normaliser class Start adding a little test coverage --- .../pyearthtools/data/transforms/normalisation/default.py | 4 ++-- .../pyearthtools/data/transforms/normalisation/normalise.py | 6 +++--- .../data/transforms/normalisation/unnormalise.py | 6 +++--- packages/data/src/pyearthtools/data/transforms/transform.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/data/src/pyearthtools/data/transforms/normalisation/default.py b/packages/data/src/pyearthtools/data/transforms/normalisation/default.py index 34019014..11c7c100 100644 --- a/packages/data/src/pyearthtools/data/transforms/normalisation/default.py +++ b/packages/data/src/pyearthtools/data/transforms/normalisation/default.py @@ -44,7 +44,7 @@ def open_file(file: str | tuple | dict): data = open_files(file) if isinstance(data, (xr.Dataset, xr.DataArray)): data = get_default_transforms()(data) - data = pyearthtools.data.transforms.coordinates.drop("time", ignore_missing=True)(data) + data = pyearthtools.data.transforms.coordinates.Drop("time", ignore_missing=True)(data) return data @@ -61,7 +61,7 @@ def under_func(*args, **kwargs): return under_func -class normaliser: +class Normaliser: def __init__( self, index: pyearthtools.data.AdvancedTimeIndex, diff --git a/packages/data/src/pyearthtools/data/transforms/normalisation/normalise.py b/packages/data/src/pyearthtools/data/transforms/normalisation/normalise.py index d0969dd9..91a7f448 100644 --- a/packages/data/src/pyearthtools/data/transforms/normalisation/normalise.py +++ b/packages/data/src/pyearthtools/data/transforms/normalisation/normalise.py @@ -23,18 +23,18 @@ xr.set_options(keep_attrs=True) -from pyearthtools.data.transforms.normalisation.default import normaliser, open_file +from pyearthtools.data.transforms.normalisation.default import Normaliser, open_file from pyearthtools.data.transforms.transform import FunctionTransform, Transform -class Normalise(normaliser): +class Normalise(Normaliser): """ Normalise incoming data. Either call this class, or get attribute for specific normalisation strategy """ - @functools.wraps(normaliser.__init__) + @functools.wraps(Normaliser.__init__) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/packages/data/src/pyearthtools/data/transforms/normalisation/unnormalise.py b/packages/data/src/pyearthtools/data/transforms/normalisation/unnormalise.py index af8b2047..38d4b262 100644 --- a/packages/data/src/pyearthtools/data/transforms/normalisation/unnormalise.py +++ b/packages/data/src/pyearthtools/data/transforms/normalisation/unnormalise.py @@ -21,16 +21,16 @@ import numpy as np import xarray as xr -from pyearthtools.data.transforms.normalisation.default import normaliser, open_file +from pyearthtools.data.transforms.normalisation.default import Normaliser, open_file from pyearthtools.data.transforms.transform import FunctionTransform, Transform xr.set_options(keep_attrs=True) -class Unnormalise(normaliser): +class Unnormalise(Normaliser): """Unnormalise Incoming Data""" - @functools.wraps(normaliser) + @functools.wraps(Normaliser) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/packages/data/src/pyearthtools/data/transforms/transform.py b/packages/data/src/pyearthtools/data/transforms/transform.py index a2348749..7d29f879 100644 --- a/packages/data/src/pyearthtools/data/transforms/transform.py +++ b/packages/data/src/pyearthtools/data/transforms/transform.py @@ -240,8 +240,8 @@ def apply(self, dataset: XR_TYPES | tuple[XR_TYPES] | list[XR_TYPES] | dict[str, def __call__(self, dataset: XR_TYPES | tuple[XR_TYPES] | list[XR_TYPES] | dict[str, XR_TYPES]) -> XR_TYPES | Any: # Do not try to transform empty datasets - if not dataset: - return dataset + if dataset is None: + return None for transform in self._transforms: dataset = transform(dataset) From e8d58c51f50909802a88e5abe3e9db62d2a3aa39 Mon Sep 17 00:00:00 2001 From: Tennessee Leeuwenburg Date: Wed, 26 Mar 2025 22:22:43 +1100 Subject: [PATCH 2/5] Add packages/data/tests/data/transform/normalisation/test_default.py --- .../transform/normalisation/test_default.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 packages/data/tests/data/transform/normalisation/test_default.py diff --git a/packages/data/tests/data/transform/normalisation/test_default.py b/packages/data/tests/data/transform/normalisation/test_default.py new file mode 100644 index 00000000..02bcf60a --- /dev/null +++ b/packages/data/tests/data/transform/normalisation/test_default.py @@ -0,0 +1,21 @@ +import pyearthtools.data.transforms.normalisation +from pyearthtools.data.transforms.normalisation import default +import xarray as xr +import numpy as np + +sample_da = xr.DataArray(coords={"latitude": [1,2,3,4], "longitude": [1,2,3]}, + data=np.ones((4,3))) + +def test_open_file(monkeypatch): + + monkeypatch.setattr(pyearthtools.data.transforms.normalisation.default, + 'open_files', + lambda x: sample_da) + + result = default.open_file("pretend_filename.nc") + assert result is not None + + +def test_Normaliser(): + n = default.Normaliser("fake index", "start", "end", "month") + n.check_init_args() From 6fca72c6c6deea646fcf87cb17248673d9787fd4 Mon Sep 17 00:00:00 2001 From: Tennessee Leeuwenburg Date: Wed, 26 Mar 2025 22:50:44 +1100 Subject: [PATCH 3/5] Fixes a bug where positional arguments were used but named arguments were required --- .../data/transforms/normalisation/default.py | 2 +- .../transform/normalisation/test_default.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/data/src/pyearthtools/data/transforms/normalisation/default.py b/packages/data/src/pyearthtools/data/transforms/normalisation/default.py index 11c7c100..b4df1784 100644 --- a/packages/data/src/pyearthtools/data/transforms/normalisation/default.py +++ b/packages/data/src/pyearthtools/data/transforms/normalisation/default.py @@ -214,7 +214,7 @@ def get_aggregation( # ) aggregated_data = get_and_print( - lambda: pyearthtools.data.transforms.aggregation.over(method, dims)( + lambda: pyearthtools.data.transforms.aggregation.over(method=method, dimension=dims)( self.index.series( **retrieval_args, transforms=transforms, diff --git a/packages/data/tests/data/transform/normalisation/test_default.py b/packages/data/tests/data/transform/normalisation/test_default.py index 02bcf60a..970d7c72 100644 --- a/packages/data/tests/data/transform/normalisation/test_default.py +++ b/packages/data/tests/data/transform/normalisation/test_default.py @@ -1,5 +1,7 @@ import pyearthtools.data.transforms.normalisation from pyearthtools.data.transforms.normalisation import default +from pyearthtools.data.time import Petdt +import pyearthtools.data.indexes import xarray as xr import numpy as np @@ -16,6 +18,17 @@ def test_open_file(monkeypatch): assert result is not None -def test_Normaliser(): - n = default.Normaliser("fake index", "start", "end", "month") +def test_Normaliser(monkeypatch): + + monkeypatch.setattr("pyearthtools.data.indexes.AdvancedTimeIndex.__abstractmethods__", set()) + + data_interval = "day" + ati = pyearthtools.data.indexes.AdvancedTimeIndex(data_interval) + monkeypatch.setattr(ati, "get", lambda x: sample_da) + start = Petdt("2023-02") + end = Petdt("2023-06") + + n = default.Normaliser(ati, start, end, "month") n.check_init_args() + + n.get_average("temperature") From 60d4d5575f91be4e05a3af7cffdb0c787858b58b Mon Sep 17 00:00:00 2001 From: Tennessee Leeuwenburg Date: Wed, 26 Mar 2025 22:57:00 +1100 Subject: [PATCH 4/5] Test ability of normaliser to get an average --- .../data/tests/data/transform/normalisation/test_default.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/data/tests/data/transform/normalisation/test_default.py b/packages/data/tests/data/transform/normalisation/test_default.py index 970d7c72..fa67c785 100644 --- a/packages/data/tests/data/transform/normalisation/test_default.py +++ b/packages/data/tests/data/transform/normalisation/test_default.py @@ -31,4 +31,6 @@ def test_Normaliser(monkeypatch): n = default.Normaliser(ati, start, end, "month") n.check_init_args() - n.get_average("temperature") + result = n.get_average("temperature") + assert result == 1 + From 2257a9afbec69c277b17391ab8344f80e674d3f9 Mon Sep 17 00:00:00 2001 From: Tennessee Leeuwenburg Date: Wed, 26 Mar 2025 23:28:22 +1100 Subject: [PATCH 5/5] Further coverage for test_default.py --- .../transform/normalisation/test_default.py | 56 ++++++++++++++++++- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/packages/data/tests/data/transform/normalisation/test_default.py b/packages/data/tests/data/transform/normalisation/test_default.py index fa67c785..b43a7651 100644 --- a/packages/data/tests/data/transform/normalisation/test_default.py +++ b/packages/data/tests/data/transform/normalisation/test_default.py @@ -4,9 +4,17 @@ import pyearthtools.data.indexes import xarray as xr import numpy as np +import pytest + +sample_da = xr.DataArray(coords={"latitude": [1,2,3,4], + "longitude": [1,2,3], + "time": ["2023-02"] + }, + data=np.ones((4,3,1))) + +sample_ds = xr.Dataset(coords={"latitude": [1,2,3,4], "longitude": [1,2,3], "time": ["2023-02"]}, + data_vars={"temperature": sample_da}) -sample_da = xr.DataArray(coords={"latitude": [1,2,3,4], "longitude": [1,2,3]}, - data=np.ones((4,3))) def test_open_file(monkeypatch): @@ -26,7 +34,7 @@ def test_Normaliser(monkeypatch): ati = pyearthtools.data.indexes.AdvancedTimeIndex(data_interval) monkeypatch.setattr(ati, "get", lambda x: sample_da) start = Petdt("2023-02") - end = Petdt("2023-06") + end = Petdt("2023-03") n = default.Normaliser(ati, start, end, "month") n.check_init_args() @@ -34,3 +42,45 @@ def test_Normaliser(monkeypatch): result = n.get_average("temperature") assert result == 1 + r_mean, r_std = n.get_deviation("temperature") + assert r_mean == 1 + assert r_std == 0 + + r_anomaly = n.get_anomaly("temperature") + assert r_anomaly is not None + + # FIXME: Need to update the whole test creation to be a time-aware dataset + # r_range = n.get_range("temperature") + # assert r_range["temperature"]["max"] == 1 + # assert r_range["temperature"]["min"] == 1 + + result = n.none + assert result is not None + +def test_Normaliser_errors(monkeypatch): + + monkeypatch.setattr("pyearthtools.data.indexes.AdvancedTimeIndex.__abstractmethods__", set()) + + data_interval = "day" + ati = pyearthtools.data.indexes.AdvancedTimeIndex(data_interval) + monkeypatch.setattr(ati, "get", lambda x: sample_da) + start = Petdt("2023-02") + end = Petdt("2023-03") + + n = default.Normaliser(ati, start, end, "month") + + with pytest.raises(NotImplementedError): + n.function() + + + not_implemented = [n.log, n.anomaly, n.deviation, n.deviation_spatial, n.range] + for ni in not_implemented: + with pytest.raises(NotImplementedError): + ni() + + + + + + +