diff --git a/.github/workflows/run-pre-commit.yml b/.github/workflows/run-pre-commit.yml index 845e7727..22036d5f 100644 --- a/.github/workflows/run-pre-commit.yml +++ b/.github/workflows/run-pre-commit.yml @@ -20,11 +20,12 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.x' - - name: Install dependencies + - name: Install dev dependencies run: | python -m pip install --upgrade pip pip install .[dev] - name: pre-commit checks run: | pre-commit run black --all-files - pre-commit run isort --all-files + # TODO enable once all cyclic imports have been fixed + # pre-commit run isort --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48ee0308..9a41190d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,3 +9,7 @@ repos: rev: 22.10.0 hooks: - id: black +- repo: https://github.com/pycqa/isort + rev: 6.0.1 + hooks: + - id: isort diff --git a/packages/data/src/pyearthtools/data/derived/insolation.py b/packages/data/src/pyearthtools/data/derived/insolation.py index f4ca1a55..96f8c23c 100644 --- a/packages/data/src/pyearthtools/data/derived/insolation.py +++ b/packages/data/src/pyearthtools/data/derived/insolation.py @@ -163,7 +163,6 @@ def derive(self, time: pd.Timestamp) -> xr.Dataset: coords={"time": time, "latitude": self._latitude[:, 0], "longitude": self._longitude[0, :]}, ) - insolation.time.encoding.update( {"dtype": "int32", "units": "hours since 1900-01-01 00:00:00.0", "calendar": "gregorian"} ) diff --git a/packages/data/src/pyearthtools/data/save/save_utils.py b/packages/data/src/pyearthtools/data/save/save_utils.py index 93bfde3c..7b7cbebb 100644 --- a/packages/data/src/pyearthtools/data/save/save_utils.py +++ b/packages/data/src/pyearthtools/data/save/save_utils.py @@ -253,6 +253,7 @@ def __exit__(self, *args): SLEEP_INTERVAL = 0.1 + class ManageFiles: """ Automatically manage the saving of files. diff --git a/packages/data/src/pyearthtools/data/transforms/coordinates.py b/packages/data/src/pyearthtools/data/transforms/coordinates.py index 2d89a485..a08541d6 100644 --- a/packages/data/src/pyearthtools/data/transforms/coordinates.py +++ b/packages/data/src/pyearthtools/data/transforms/coordinates.py @@ -151,9 +151,7 @@ class ReIndex(Transform): """Reindex Coordinates""" def __init__( - self, - coordinates: dict[str, Literal["reversed", "sorted"] | Iterable] | xr.Coordinates | None = None, - **coords + self, coordinates: dict[str, Literal["reversed", "sorted"] | Iterable] | xr.Coordinates | None = None, **coords ): """ Reindex coordinates @@ -323,7 +321,6 @@ def apply(self, dataset: xr.Dataset) -> xr.Dataset: return dataset - class Drop(Transform): """Drop items from Dataset""" @@ -370,10 +367,10 @@ def apply(self, dataset: xr.Dataset) -> xr.Dataset: def weak_cast_to_int(value): - ''' + """ Basically, turns integer floats to int types, otherwise does nothing. - ''' + """ try: if int(value) == value: value = int(value) @@ -602,7 +599,6 @@ def apply(self, dataset: xr.Dataset) -> xr.Dataset: # return dict(as_dataarray=self._as_dataarray, **self._coordinates) - class Pad(Transform): """ Pad data diff --git a/packages/data/tests/data/archive/test_extensions.py b/packages/data/tests/data/archive/test_extensions.py index f9ed0fd8..e2ba9ccb 100644 --- a/packages/data/tests/data/archive/test_extensions.py +++ b/packages/data/tests/data/archive/test_extensions.py @@ -5,26 +5,21 @@ def test_register_archive(): - - @extensions.register_archive("NewData") class NewData: - def __init__(self, args): - self.args = args - + def __init__(self, args): + self.args = args # If registering over the top of an existing name, confirm warning occurs with pytest.warns(pyearthtools.data.AccessorRegistrationWarning): + @extensions.register_archive("NewData") class NewData: - _pyearthtools_initialisation = {"class": {}} + _pyearthtools_initialisation = {"class": {}} def __init__(self, args): self.args = args def __call__(self, *args, **kwargs): pass - - - diff --git a/packages/data/tests/data/derived/test_insolation.py b/packages/data/tests/data/derived/test_insolation.py index a2dac1f9..ab53a1ed 100644 --- a/packages/data/tests/data/derived/test_insolation.py +++ b/packages/data/tests/data/derived/test_insolation.py @@ -1,10 +1,11 @@ from pyearthtools.data.derived import insolation from pyearthtools.data.time import Petdt + def test_array(): - a = insolation.array([1,2,3,4,5]) - assert all(a == [1,2,3,4,5]) + a = insolation.array([1, 2, 3, 4, 5]) + assert all(a == [1, 2, 3, 4, 5]) def test_Insolation(): @@ -12,9 +13,9 @@ def test_Insolation(): insol = insolation.Insolation( [-10], [40], - ) + ) - at_time = ['2021-01-01'] + at_time = ["2021-01-01"] # import pudb; pudb.set_trace() @@ -23,4 +24,3 @@ def test_Insolation(): # TODO: Check the insolation calculation # TODO: Check the result is the right type/contents - diff --git a/packages/data/tests/data/indexes/test_indexes.py b/packages/data/tests/data/indexes/test_indexes.py index 03709a8c..b2f9e8ca 100644 --- a/packages/data/tests/data/indexes/test_indexes.py +++ b/packages/data/tests/data/indexes/test_indexes.py @@ -1,69 +1,66 @@ - from pyearthtools.data import indexes import pyearthtools.data.archive from pyearthtools.data.time import Petdt import pytest import pathlib + def test_Index(monkeypatch): - monkeypatch.setattr("pyearthtools.data.indexes.Index.__abstractmethods__", set()) + monkeypatch.setattr("pyearthtools.data.indexes.Index.__abstractmethods__", set()) - idx = indexes.Index() + idx = indexes.Index() - with pytest.raises(NotImplementedError): - idx.get() + with pytest.raises(NotImplementedError): + idx.get() + + with pytest.raises(NotImplementedError): + idx() - with pytest.raises(NotImplementedError): - idx() def test_FileSystemIndex(monkeypatch): - monkeypatch.setattr("pyearthtools.data.indexes.Index.__abstractmethods__", set()) - fsi = indexes.FileSystemIndex() - with pytest.raises(NotImplementedError): - fsi.filesystem("anything") + monkeypatch.setattr("pyearthtools.data.indexes.Index.__abstractmethods__", set()) + fsi = indexes.FileSystemIndex() + with pytest.raises(NotImplementedError): + fsi.filesystem("anything") - # Confirm the test begins without a root directory set - monkeypatch.delattr(pyearthtools.data.archive, "ROOT_DIRECTORIES", raising=False) - fsi = indexes.FileSystemIndex() - with pytest.raises(KeyError): - fsi.ROOT_DIRECTORIES + # Confirm the test begins without a root directory set + monkeypatch.delattr(pyearthtools.data.archive, "ROOT_DIRECTORIES", raising=False) + fsi = indexes.FileSystemIndex() + with pytest.raises(KeyError): + fsi.ROOT_DIRECTORIES - # Confirm the FSI responds to the root directory variable correctly - monkeypatch.setattr(pyearthtools.data.archive, "ROOT_DIRECTORIES", "Hello", raising=False) - fsi = indexes.FileSystemIndex() - assert fsi.ROOT_DIRECTORIES == "Hello" + # Confirm the FSI responds to the root directory variable correctly + monkeypatch.setattr(pyearthtools.data.archive, "ROOT_DIRECTORIES", "Hello", raising=False) + fsi = indexes.FileSystemIndex() + assert fsi.ROOT_DIRECTORIES == "Hello" + monkeypatch.setattr(fsi, "filesystem", lambda x: __file__, raising=False) + assert fsi.search("needle") == __file__ - monkeypatch.setattr(fsi, 'filesystem', lambda x: __file__, raising=False) - assert fsi.search("needle") == __file__ + # Check the current Python file exists + assert fsi.exists(__file__) + assert fsi.exists(pathlib.Path(__file__)) + assert fsi.exists({"a": __file__}) + assert fsi.exists((__file__, __file__)) - # Check the current Python file exists - assert fsi.exists(__file__) - assert fsi.exists(pathlib.Path(__file__)) - assert fsi.exists({"a": __file__}) - assert fsi.exists((__file__, __file__)) + with pytest.raises(KeyError): + assert fsi.load(__file__) is not None + # TODO test actual netcdf file loading - with pytest.raises(KeyError): - assert fsi.load(__file__) is not None - # TODO test actual netcdf file loading + with pytest.raises(KeyError): + assert fsi.get(__file__) is not None + # TODO test actual netcdf file getting - with pytest.raises(KeyError): - assert fsi.get(__file__) is not None - # TODO test actual netcdf file getting def test_AdvancedTimeIndex(monkeypatch): - monkeypatch.setattr("pyearthtools.data.indexes.AdvancedTimeIndex.__abstractmethods__", set()) - - data_interval = "day" - ati = indexes.AdvancedTimeIndex(data_interval) - dt = Petdt('2023-02') - - monkeypatch.setattr(pyearthtools.data.indexes.Index, 'get', lambda x, y: x, raising=False) - ati.retrieve(dt, use_simple = True) - - + monkeypatch.setattr("pyearthtools.data.indexes.AdvancedTimeIndex.__abstractmethods__", set()) + data_interval = "day" + ati = indexes.AdvancedTimeIndex(data_interval) + dt = Petdt("2023-02") + monkeypatch.setattr(pyearthtools.data.indexes.Index, "get", lambda x, y: x, raising=False) + ati.retrieve(dt, use_simple=True) diff --git a/packages/data/tests/data/transform/test_coordinates.py b/packages/data/tests/data/transform/test_coordinates.py index 03e131ed..b9b4ec8d 100644 --- a/packages/data/tests/data/transform/test_coordinates.py +++ b/packages/data/tests/data/transform/test_coordinates.py @@ -16,11 +16,10 @@ da_unclear = xr.DataArray(coords={"longitude": lon_unclear}, dims=["longitude"]) ds_vertical = xr.Dataset( - coords = {"longitude": list(range(0, 4)), - "vertical": list(range(0, 3))}, - data_vars = {"temperature": (["longitude", "vertical"], - np.random.rand(4,3))} - ) + coords={"longitude": list(range(0, 4)), "vertical": list(range(0, 3))}, + data_vars={"temperature": (["longitude", "vertical"], np.random.rand(4, 3))}, +) + def test_get_longitude(): @@ -41,99 +40,98 @@ def test_get_longitude(): with pytest.raises(ValueError): result = coordinates.get_longitude(da_unclear, transform=False) + def test_StandardLongitude(): - conform = coordinates.StandardLongitude("0-360") - fixed = conform.apply(da180) - assert fixed is not None - unchanged = conform.apply(da360) - # TODO - shouldn't this be true? - # assert xr.testing.assert_equal(fixed, da360) + conform = coordinates.StandardLongitude("0-360") + fixed = conform.apply(da180) + assert fixed is not None + unchanged = conform.apply(da360) + # TODO - shouldn't this be true? + # assert xr.testing.assert_equal(fixed, da360) + + conform = coordinates.StandardLongitude("-180-180") + fixed = conform.apply(da360) + unchanged = conform.apply(da180) + assert fixed is not None + # TODO - shouldn't this be true? + # assert xr.testing.assert_equal(fixed, da180) - conform = coordinates.StandardLongitude("-180-180") - fixed = conform.apply(da360) - unchanged = conform.apply(da180) - assert fixed is not None - # TODO - shouldn't this be true? - # assert xr.testing.assert_equal(fixed, da180) def test_ReIndex(): - tf_reindex = coordinates.ReIndex({"longitude": "reversed"}) - tf_reindex.apply(da180) - # TODO: Assert the range of the reversed coordinate is 180 to -180 + tf_reindex = coordinates.ReIndex({"longitude": "reversed"}) + tf_reindex.apply(da180) + # TODO: Assert the range of the reversed coordinate is 180 to -180 + def test_Select(): - tf_select = coordinates.Select({"longitude": slice(10, 120)}) - result = tf_select.apply(da180) - assert result is not None - # TODO: Check the result against the requested slice + tf_select = coordinates.Select({"longitude": slice(10, 120)}) + result = tf_select.apply(da180) + assert result is not None + # TODO: Check the result against the requested slice def test_Drop(): - tf_drop = coordinates.Drop("vertical") - result = tf_drop.apply(ds_vertical) + tf_drop = coordinates.Drop("vertical") + result = tf_drop.apply(ds_vertical) + + # TODO: Assert that the dimension has been dropped - # TODO: Assert that the dimension has been dropped def test_Flatten(): + tf_flatten = coordinates.Flatten("vertical") + result = tf_flatten.apply(ds_vertical) - tf_flatten = coordinates.Flatten("vertical") - result = tf_flatten.apply(ds_vertical) + # TODO: Assert the flattened data looks correct - # TODO: Assert the flattened data looks correct def test_Expand(): - tf_expand = coordinates.Expand("vertical") - result = tf_expand.apply(ds_vertical) + tf_expand = coordinates.Expand("vertical") + result = tf_expand.apply(ds_vertical) - # TODO: Assert the expanded dataset has a vertical dimension + # TODO: Assert the expanded dataset has a vertical dimension def test_SelectFlatten(): - tf_selectflatten = coordinates.SelectFlatten( - {"longitude": slice(10, 120)} - ) + tf_selectflatten = coordinates.SelectFlatten({"longitude": slice(10, 120)}) - with pytest.raises(NotImplementedError): - result = tf_selectflatten.apply(ds_vertical) - # TODO fix the code or avoid the issue + with pytest.raises(NotImplementedError): + result = tf_selectflatten.apply(ds_vertical) + # TODO fix the code or avoid the issue + + # TODO: Check the values of the resulting dataset - # TODO: Check the values of the resulting dataset def test_Assign(): - tf_assign = coordinates.Assign( - {"longitude": list(range(0, 4)), - "vertical": list(range(3, 6)) - } - ) + tf_assign = coordinates.Assign({"longitude": list(range(0, 4)), "vertical": list(range(3, 6))}) + + result = tf_assign.apply(ds_vertical) - result = tf_assign.apply(ds_vertical) + # TODO: check the values of the vertical coords - # TODO: check the values of the vertical coords def test_Pad(): - tf_pad = coordinates.Pad({"longitude": list(range(0,4))}) + tf_pad = coordinates.Pad({"longitude": list(range(0, 4))}) - with pytest.raises(ValueError): - result = tf_pad.apply(ds_vertical) - # TODO: Fix the code or fix the test + with pytest.raises(ValueError): + result = tf_pad.apply(ds_vertical) + # TODO: Fix the code or fix the test - # TODO: check the values of the result - + # TODO: check the values of the result -def test_weak_cast_to_int(): - wcti = coordinates.weak_cast_to_int +def test_weak_cast_to_int(): - assert wcti(5.0) == 5 - assert type(wcti(5.0)) == int + wcti = coordinates.weak_cast_to_int - assert wcti("hello") == "hello" + assert wcti(5.0) == 5 + assert type(wcti(5.0)) == int + assert wcti("hello") == "hello" diff --git a/pyproject.toml b/pyproject.toml index 142ef949..a9ff2b32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,8 @@ all = [ "pyearthtools-pipeline[all] @ git+https://github.com/ACCESS-Community-Hub/PyEarthTools.git#subdirectory=packages/pipeline", "pyearthtools-training[all] @ git+https://github.com/ACCESS-Community-Hub/PyEarthTools.git#subdirectory=packages/training" ] -test = ["pytest", "pytest-cov", "pytest-xdist", "pudb", "pre-commit"] +test = ["pytest", "pytest-cov", "pytest-xdist", "pudb"] +dev = ["pre-commit"] docs = [ "sphinx", "myst-parser", diff --git a/requirements-dev.txt b/requirements-dev.txt index f38c25af..b49237bf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,4 +3,4 @@ -e packages/training[all] -e packages/pipeline[all] -e packages/tutorial --e .[test,docs] +-e .[test,dev,docs]