diff --git a/packages/data/.pre-commit-config.yaml b/packages/data/.pre-commit-config.yaml index f52b12ee..850429d6 100644 --- a/packages/data/.pre-commit-config.yaml +++ b/packages/data/.pre-commit-config.yaml @@ -1,32 +1,49 @@ ---- -# See https://pre-commit.com for more information -# See https://pre-commit.com/hooks.html for more hooks repos: - - repo: https://github.com/pre-commit/pre-commit-hooks +- repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.2.0 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: check-added-large-files - args: ['--maxkb=10485760'] - - id: check-byte-order-marker - - id: check-executables-have-shebangs - - repo: https://github.com/psf/black - rev: 22.3.0 + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/psf/black + rev: 22.10.0 hooks: - - id: black - args: ['--line-length', '120'] - # - repo: https://github.com/PyCQA/flake8 - # rev: 3.9.2 - # hooks: - # - id: flake8 - # args: ['--max-line-length', '120', "--extend-ignore", "E203"] - - repo: https://github.com/Quantco/pre-commit-mirrors-shellcheck - rev: 0.7.1 + - id: black +- repo: https://github.com/pycqa/isort + rev: 6.0.1 hooks: - - id: shellcheck-conda - - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.64.0 - hooks: - - id: terraform_validate + - id: isort + + +# --- +# # See https://pre-commit.com for more information +# # See https://pre-commit.com/hooks.html for more hooks +# repos: +# - repo: https://github.com/pre-commit/pre-commit-hooks +# rev: v3.2.0 +# hooks: +# - id: trailing-whitespace +# - id: end-of-file-fixer +# - id: check-yaml +# - id: check-added-large-files +# args: ['--maxkb=10485760'] +# - id: check-byte-order-marker +# - id: check-executables-have-shebangs +# - repo: https://github.com/psf/black +# rev: 22.3.0 +# hooks: +# - id: black +# # - repo: https://github.com/PyCQA/flake8 +# # rev: 3.9.2 +# # hooks: +# # - id: flake8 +# # args: ['--max-line-length', '120', "--extend-ignore", "E203"] +# - repo: https://github.com/Quantco/pre-commit-mirrors-shellcheck +# rev: 0.7.1 +# hooks: +# - id: shellcheck-conda +# - repo: https://github.com/antonbabenko/pre-commit-terraform +# rev: v1.64.0 +# hooks: +# - id: terraform_validate + diff --git a/packages/data/src/pyearthtools/data/__init__.py b/packages/data/src/pyearthtools/data/__init__.py index 10ab403d..64dbe237 100644 --- a/packages/data/src/pyearthtools/data/__init__.py +++ b/packages/data/src/pyearthtools/data/__init__.py @@ -78,65 +78,71 @@ __version__ = "0.1.0" -from pyearthtools.data import logger -from pyearthtools.data import config - -from pyearthtools.data.time import Petdt, TimeResolution, TimeDelta, TimeRange -from pyearthtools.data.time import Petdt as datetime +import warnings as __python_warnings -from pyearthtools.data.exceptions import DataNotFoundError, InvalidIndexError -from pyearthtools.data.warnings import ( - IndexWarning, - pyearthtoolsDataWarning, - AccessorRegistrationWarning, +import pyearthtools.utils +from pyearthtools.data import ( + archive, + config, + derived, + download, + indexes, + logger, + modifications, ) - - +from pyearthtools.data import operations +from pyearthtools.data import operations as op +from pyearthtools.data import ( + patterns, + save, + static, +) +from pyearthtools.data import transforms +from pyearthtools.data import transforms as transform +from pyearthtools.data import ( + utils, +) +from pyearthtools.data.archive.utils import auto_import from pyearthtools.data.collection import Collection, LabelledCollection - -# from pyearthtools.data.catalog import Catalog, CatalogEntry - +from pyearthtools.data.exceptions import DataNotFoundError, InvalidIndexError from pyearthtools.data.indexes import ( - Index, - DataIndex, - FileSystemIndex, - TimeIndex, - AdvancedTimeIndex, AdvancedTimeDataIndex, + AdvancedTimeIndex, + ArchiveIndex, BaseTimeIndex, + CachingForecastIndex, + CachingIndex, DataFileSystemIndex, - ArchiveIndex, + DataIndex, + FileSystemIndex, ForecastIndex, - StaticDataIndex, - CachingIndex, - CachingForecastIndex, + Index, IntakeIndex, IntakeIndexCache, + StaticDataIndex, + TimeIndex, + register_accessor, ) -from pyearthtools.data import indexes -from pyearthtools.data.indexes import register_accessor - -from pyearthtools.data import operations as op - -from pyearthtools.data import archive, operations, static, transforms, patterns, download, modifications, derived, utils -from pyearthtools.data import transforms as transform +from pyearthtools.data.load import load from pyearthtools.data.patterns import PatternIndex - +from pyearthtools.data.save import ManageFiles, ManageTemp +from pyearthtools.data.time import Petdt +from pyearthtools.data.time import Petdt as datetime +from pyearthtools.data.time import TimeDelta, TimeRange, TimeResolution +from pyearthtools.data.transforms.derive import evaluate from pyearthtools.data.transforms.transform import ( + FunctionTransform, Transform, TransformCollection, - FunctionTransform, ) -from pyearthtools.data.transforms.derive import evaluate -from pyearthtools.data import save -from pyearthtools.data.save import ManageFiles, ManageTemp - -from pyearthtools.data.load import load +from pyearthtools.data.warnings import ( + AccessorRegistrationWarning, + IndexWarning, + pyearthtoolsDataWarning, +) -import pyearthtools.utils -import warnings as __python_warnings +# from pyearthtools.data.catalog import Catalog, CatalogEntry -from pyearthtools.data.archive.utils import auto_import """Auto import archives if available""" diff --git a/packages/data/src/pyearthtools/data/archive/__init__.py b/packages/data/src/pyearthtools/data/archive/__init__.py index 2c3d4bac..8e071950 100644 --- a/packages/data/src/pyearthtools/data/archive/__init__.py +++ b/packages/data/src/pyearthtools/data/archive/__init__.py @@ -36,9 +36,7 @@ """ from pyearthtools.data.archive.extensions import register_archive - -from pyearthtools.data.archive.root import set_root, reset_root, config_root - +from pyearthtools.data.archive.root import config_root, reset_root, set_root ZARR_IMPORTED = True try: diff --git a/packages/data/src/pyearthtools/data/archive/extensions.py b/packages/data/src/pyearthtools/data/archive/extensions.py index 6e1c2efc..fb8cf7a0 100644 --- a/packages/data/src/pyearthtools/data/archive/extensions.py +++ b/packages/data/src/pyearthtools/data/archive/extensions.py @@ -39,10 +39,9 @@ from __future__ import annotations -from types import ModuleType -from typing import Callable, Any - import warnings +from types import ModuleType +from typing import Any, Callable import pyearthtools.data from pyearthtools.data import archive diff --git a/packages/data/src/pyearthtools/data/archive/root.py b/packages/data/src/pyearthtools/data/archive/root.py index 682fadb7..e8fd4e07 100644 --- a/packages/data/src/pyearthtools/data/archive/root.py +++ b/packages/data/src/pyearthtools/data/archive/root.py @@ -15,8 +15,8 @@ from __future__ import annotations -import warnings import logging +import warnings import pyearthtools.data from pyearthtools.data import archive diff --git a/packages/data/src/pyearthtools/data/archive/utils.py b/packages/data/src/pyearthtools/data/archive/utils.py index 30284418..5349c460 100644 --- a/packages/data/src/pyearthtools/data/archive/utils.py +++ b/packages/data/src/pyearthtools/data/archive/utils.py @@ -21,14 +21,13 @@ from __future__ import annotations -from pathlib import Path -import os -import re - import importlib import logging -from typing import Any +import os +import re import warnings +from pathlib import Path +from typing import Any from pyearthtools.data import archive from pyearthtools.data.indexes.utilities.fileload import open_static diff --git a/packages/data/src/pyearthtools/data/archive/zarr.py b/packages/data/src/pyearthtools/data/archive/zarr.py index 4b230111..80bd2c81 100644 --- a/packages/data/src/pyearthtools/data/archive/zarr.py +++ b/packages/data/src/pyearthtools/data/archive/zarr.py @@ -21,23 +21,20 @@ from __future__ import annotations +import logging +from os import PathLike from pathlib import Path from typing import Any, Literal -from os import PathLike -import xarray as xr import dask -import logging - import pyearthtools.data -from pyearthtools.data.time import Petdt -from pyearthtools.data.transforms import Transform, TransformCollection +import xarray as xr from pyearthtools.data.indexes.indexes import DataFileSystemIndex, TimeIndex - from pyearthtools.data.indexes.utilities.dimensions import identify_time_dimension -from pyearthtools.data.utils import parse_path - from pyearthtools.data.save import save +from pyearthtools.data.time import Petdt +from pyearthtools.data.transforms import Transform, TransformCollection +from pyearthtools.data.utils import parse_path LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/src/pyearthtools/data/catalog.py b/packages/data/src/pyearthtools/data/catalog.py index 484da692..b6aa8125 100644 --- a/packages/data/src/pyearthtools/data/catalog.py +++ b/packages/data/src/pyearthtools/data/catalog.py @@ -40,26 +40,22 @@ """ from __future__ import annotations -from collections import OrderedDict - -from typing import Optional -import yaml +import io import json -from pathlib import Path -from typing import Any, Callable, Optional import types import warnings +from collections import OrderedDict from functools import lru_cache -import io - -from pyearthtools.utils.parsing import function_name -from pyearthtools.utils.initialisation.imports import dynamic_import +from pathlib import Path +from typing import Any, Callable, Optional import pyearthtools.data +import yaml from pyearthtools.data.collection import Collection, LabelledCollection - from pyearthtools.utils.decorators import alias_arguments +from pyearthtools.utils.initialisation.imports import dynamic_import +from pyearthtools.utils.parsing import function_name UTILS_REPR = False try: @@ -195,7 +191,7 @@ def convert(**kwargs): Converts non serialisable to serialisable """ for key, value in kwargs.items(): - if isinstance(value, (pyearthtools.data.TimeDelta, pyearthtools.data.TimeResolution, Path)): + if isinstance(value, (pyearthtools.data.time.TimeDelta, pyearthtools.data.TimeResolution, Path)): kwargs[key] = str(value) return kwargs diff --git a/packages/data/src/pyearthtools/data/collection.py b/packages/data/src/pyearthtools/data/collection.py index 870b412c..9ca5936b 100644 --- a/packages/data/src/pyearthtools/data/collection.py +++ b/packages/data/src/pyearthtools/data/collection.py @@ -22,10 +22,11 @@ """ from __future__ import annotations -from _collections_abc import dict_keys -from typing import Any from collections import OrderedDict +from typing import Any + +from _collections_abc import dict_keys class Collection(tuple): diff --git a/packages/data/src/pyearthtools/data/commands.py b/packages/data/src/pyearthtools/data/commands.py index f4861e89..085731f4 100644 --- a/packages/data/src/pyearthtools/data/commands.py +++ b/packages/data/src/pyearthtools/data/commands.py @@ -18,6 +18,7 @@ """ from __future__ import annotations + from pathlib import Path import click @@ -98,8 +99,8 @@ def create_structure(top, blacklisted, save, verbose): top: Path Location to generate structure for """ - from pyearthtools.data.indexes.utilities.structure import structure import yaml + from pyearthtools.data.indexes.utilities.structure import structure structure_dict: dict[str, dict | list] = {} structure_d: dict[str, dict[str, Any] | list[str]] = structure(top, blacklisted=blacklisted, verbose=verbose) # type: ignore diff --git a/packages/data/src/pyearthtools/data/derived/__init__.py b/packages/data/src/pyearthtools/data/derived/__init__.py index 480b3796..40c559ff 100644 --- a/packages/data/src/pyearthtools/data/derived/__init__.py +++ b/packages/data/src/pyearthtools/data/derived/__init__.py @@ -18,8 +18,11 @@ """ -from pyearthtools.data.derived.derived import DerivedValue, TimeDerivedValue, AdvancedTimeDerivedValue +from pyearthtools.data.derived.derived import ( + AdvancedTimeDerivedValue, + DerivedValue, + TimeDerivedValue, +) from pyearthtools.data.derived.insolation import Insolation - __all__ = ["DerivedValue", "TimeDerivedValue", "AdvancedTimeDerivedValue", "Insolation"] diff --git a/packages/data/src/pyearthtools/data/derived/derived.py b/packages/data/src/pyearthtools/data/derived/derived.py index b3cbdc92..85383c48 100644 --- a/packages/data/src/pyearthtools/data/derived/derived.py +++ b/packages/data/src/pyearthtools/data/derived/derived.py @@ -19,15 +19,13 @@ from __future__ import annotations -from typing import Union import inspect -from abc import abstractmethod, ABCMeta +from abc import ABCMeta, abstractmethod +from typing import Union import xarray as xr - - +from pyearthtools.data.indexes import AdvancedTimeDataIndex, DataIndex, TimeDataIndex from pyearthtools.data.time import Petdt, TimeDelta, TimeRange -from pyearthtools.data.indexes import DataIndex, TimeDataIndex, AdvancedTimeDataIndex class DerivedValue(DataIndex, metaclass=ABCMeta): diff --git a/packages/data/src/pyearthtools/data/derived/insolation.py b/packages/data/src/pyearthtools/data/derived/insolation.py index 96f8c23c..7f3955fa 100644 --- a/packages/data/src/pyearthtools/data/derived/insolation.py +++ b/packages/data/src/pyearthtools/data/derived/insolation.py @@ -18,11 +18,9 @@ from __future__ import annotations - import numpy as np -import xarray as xr import pandas as pd - +import xarray as xr from pyearthtools.data.derived.derived import AdvancedTimeDerivedValue DASK_IMPORTED = True diff --git a/packages/data/src/pyearthtools/data/download/__init__.py b/packages/data/src/pyearthtools/data/download/__init__.py index d4cafa95..5d7c7467 100644 --- a/packages/data/src/pyearthtools/data/download/__init__.py +++ b/packages/data/src/pyearthtools/data/download/__init__.py @@ -27,6 +27,6 @@ """ -from pyearthtools.data.download.templates import DownloadIndex -from pyearthtools.data.download import cds, arco +from pyearthtools.data.download import arco, cds from pyearthtools.data.download import ecmwf_opendata as opendata +from pyearthtools.data.download.templates import DownloadIndex diff --git a/packages/data/src/pyearthtools/data/download/arco/ERA5.py b/packages/data/src/pyearthtools/data/download/arco/ERA5.py index bc40c534..db638540 100644 --- a/packages/data/src/pyearthtools/data/download/arco/ERA5.py +++ b/packages/data/src/pyearthtools/data/download/arco/ERA5.py @@ -14,23 +14,20 @@ from __future__ import annotations -import functools -import xarray as xr +import functools from os import PathLike import pyearthtools.data -from pyearthtools.data.time import Petdt - -from pyearthtools.data.indexes import AdvancedTimeDataIndex, decorators, CachingIndex -from pyearthtools.data.transforms.transform import Transform, TransformCollection - -from pyearthtools.data.patterns.expanded_date import ExpandedDateVariable - +import xarray as xr from pyearthtools.data.download.arco.variables.ERA5 import ( ERA5_LEVELS, ERA_NAME_CHANGE, ) +from pyearthtools.data.indexes import AdvancedTimeDataIndex, CachingIndex, decorators +from pyearthtools.data.patterns.expanded_date import ExpandedDateVariable +from pyearthtools.data.time import Petdt +from pyearthtools.data.transforms.transform import Transform, TransformCollection ROOT_ARCO_DS = None diff --git a/packages/data/src/pyearthtools/data/download/cds/ERA5.py b/packages/data/src/pyearthtools/data/download/cds/ERA5.py index 17ec91a0..71e3fbc9 100644 --- a/packages/data/src/pyearthtools/data/download/cds/ERA5.py +++ b/packages/data/src/pyearthtools/data/download/cds/ERA5.py @@ -18,25 +18,22 @@ """ from __future__ import annotations -from typing import Literal, Union, Optional -from pathlib import Path -import warnings +import warnings +from pathlib import Path +from typing import Literal, Optional, Union import pyearthtools.data -from pyearthtools.data import Petdt -from pyearthtools.data.warnings import pyearthtoolsDataWarning - -from pyearthtools.data.indexes import decorators -from pyearthtools.data.transforms import Transform, TransformCollection - -from pyearthtools.data.download.cds.cds import root_cds, as_list from pyearthtools.data.download.cds._ERA5 import ( ERA5_LEVELS, ERA_PRESSURE_NAME_CHANGE, ERA_SINGLE_NAME_CHANGE, ) - +from pyearthtools.data.download.cds.cds import as_list, root_cds +from pyearthtools.data.indexes import decorators +from pyearthtools.data.time import Petdt +from pyearthtools.data.transforms import Transform, TransformCollection +from pyearthtools.data.warnings import pyearthtoolsDataWarning cds_type = [ "ensemble_mean", diff --git a/packages/data/src/pyearthtools/data/download/cds/cds.py b/packages/data/src/pyearthtools/data/download/cds/cds.py index ca1c543e..d792542f 100644 --- a/packages/data/src/pyearthtools/data/download/cds/cds.py +++ b/packages/data/src/pyearthtools/data/download/cds/cds.py @@ -18,19 +18,20 @@ """ from __future__ import annotations -from abc import abstractmethod, ABCMeta -from pathlib import Path -from typing import Iterable, Any, Type -import xarray as xr + import logging import warnings +from abc import ABCMeta, abstractmethod +from pathlib import Path +from typing import Any, Iterable, Type import urllib3 - -from pyearthtools.data import DataNotFoundError, Petdt -from pyearthtools.data.download import DownloadIndex +import xarray as xr +from pyearthtools.data.download.templates import DownloadIndex +from pyearthtools.data.exceptions import DataNotFoundError from pyearthtools.data.indexes import utilities -from pyearthtools.data.patterns import TemporalExpandedDateVariable, PatternIndex +from pyearthtools.data.patterns import PatternIndex, TemporalExpandedDateVariable +from pyearthtools.data.time import Petdt from pyearthtools.data.transforms import Transform, TransformCollection LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/src/pyearthtools/data/download/ecmwf_opendata/__init__.py b/packages/data/src/pyearthtools/data/download/ecmwf_opendata/__init__.py index 5a85ac5c..67417e34 100644 --- a/packages/data/src/pyearthtools/data/download/ecmwf_opendata/__init__.py +++ b/packages/data/src/pyearthtools/data/download/ecmwf_opendata/__init__.py @@ -17,5 +17,5 @@ ECMWF OpenData """ -from pyearthtools.data.download.ecmwf_opendata.opendata import OpenData from pyearthtools.data.download.ecmwf_opendata.convenience import AIFS, IFS +from pyearthtools.data.download.ecmwf_opendata.opendata import OpenData diff --git a/packages/data/src/pyearthtools/data/download/ecmwf_opendata/convenience.py b/packages/data/src/pyearthtools/data/download/ecmwf_opendata/convenience.py index f5fd7312..74d34475 100644 --- a/packages/data/src/pyearthtools/data/download/ecmwf_opendata/convenience.py +++ b/packages/data/src/pyearthtools/data/download/ecmwf_opendata/convenience.py @@ -16,6 +16,7 @@ """Convenience classes for OpenData""" import functools + from pyearthtools.data.download.ecmwf_opendata.opendata import OpenData diff --git a/packages/data/src/pyearthtools/data/download/ecmwf_opendata/opendata.py b/packages/data/src/pyearthtools/data/download/ecmwf_opendata/opendata.py index 2f48172b..9bdd16a8 100644 --- a/packages/data/src/pyearthtools/data/download/ecmwf_opendata/opendata.py +++ b/packages/data/src/pyearthtools/data/download/ecmwf_opendata/opendata.py @@ -15,22 +15,20 @@ from __future__ import annotations -from typing import Literal, Any import warnings -import xarray as xr +from typing import Any, Literal import pyearthtools.data +import xarray as xr +from pyearthtools.data.download.ecmwf_opendata import opendata_variables +from pyearthtools.data.download.templates import DownloadIndex from pyearthtools.data.indexes import ( + VARIABLE_DEFAULT, + VariableDefault, alias_arguments, check_arguments, - VariableDefault, - VARIABLE_DEFAULT, ) -from pyearthtools.data.download.ecmwf_opendata import opendata_variables -from pyearthtools.data.download import DownloadIndex - - VALID_MODELS = Literal["aifs", "ifs"] VALID_STREAM = Literal["oper", "enfo", "waef", "wave"] VALID_TYPES = Literal[ @@ -44,7 +42,7 @@ RENAME_DICT = {"t2m": "2t", "u10": "10u", "v10": "10v"} -def rounder(time: pyearthtools.data.Petdt, round_value: int = 6) -> int: +def rounder(time: pyearthtools.data.time.Petdt, round_value: int = 6) -> int: return (time.hour // round_value) * round_value @@ -195,7 +193,7 @@ def latest(self) -> xr.Dataset: """Get the latest data from `ecwmf-opendata`""" return self(self._get_latest()) - def download(self, querytime: str | pyearthtools.data.Petdt) -> xr.Dataset: + def download(self, querytime: str | pyearthtools.data.time.Petdt) -> xr.Dataset: """Download data from `ecwmf-opendata`""" if self._client is None: raise ImportError(f"`ecwmwf.opendata` was not imported, cannot download new data.") @@ -203,8 +201,8 @@ def download(self, querytime: str | pyearthtools.data.Petdt) -> xr.Dataset: if querytime == "l1atest": querytime = self._get_latest() - date = pyearthtools.data.Petdt(querytime).at_resolution("day") - time = rounder(pyearthtools.data.Petdt(querytime)) + date = pyearthtools.data.time.Petdt(querytime).at_resolution("day") + time = rounder(pyearthtools.data.time.Petdt(querytime)) request = dict(self._request_base) path = self.get_tempdir() diff --git a/packages/data/src/pyearthtools/data/download/templates.py b/packages/data/src/pyearthtools/data/download/templates.py index 7c86daec..5d2c63be 100644 --- a/packages/data/src/pyearthtools/data/download/templates.py +++ b/packages/data/src/pyearthtools/data/download/templates.py @@ -18,20 +18,19 @@ """ from __future__ import annotations -from abc import abstractmethod, ABCMeta -from typing import Type -import warnings -from tempfile import TemporaryDirectory +import functools +import warnings +from abc import ABCMeta, abstractmethod from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Type import urllib3 -import functools - -from pyearthtools.data.indexes import CachingIndex -from pyearthtools.data import IndexWarning, DataNotFoundError - +from pyearthtools.data.exceptions import DataNotFoundError +from pyearthtools.data.indexes.cacheIndex import CachingIndex from pyearthtools.data.patterns import PatternIndex +from pyearthtools.data.warnings import IndexWarning @functools.lru_cache(None) diff --git a/packages/data/src/pyearthtools/data/exceptions.py b/packages/data/src/pyearthtools/data/exceptions.py index f039d483..2325bba8 100644 --- a/packages/data/src/pyearthtools/data/exceptions.py +++ b/packages/data/src/pyearthtools/data/exceptions.py @@ -18,9 +18,9 @@ """ from __future__ import annotations -from typing import Any, Callable import warnings +from typing import Any, Callable class InvalidIndexError(KeyError): diff --git a/packages/data/src/pyearthtools/data/indexes/__init__.py b/packages/data/src/pyearthtools/data/indexes/__init__.py index cc57228e..4becdc2a 100644 --- a/packages/data/src/pyearthtools/data/indexes/__init__.py +++ b/packages/data/src/pyearthtools/data/indexes/__init__.py @@ -114,42 +114,40 @@ class StaticDataIndex{ """ +from pyearthtools.data.indexes import decorators, utilities +from pyearthtools.data.indexes.cacheIndex import ( + CachingForecastIndex, + CachingIndex, + FileSystemCacheIndex, + FunctionalCacheIndex, + FunctionalMemCacheIndex, + MemCache, +) +from pyearthtools.data.indexes.decorators import alias_arguments, check_arguments +from pyearthtools.data.indexes.extensions import register_accessor +from pyearthtools.data.indexes.fake import FakeIndex from pyearthtools.data.indexes.indexes import ( - Index, - DataIndex, - FileSystemIndex, - TimeIndex, - TimeDataIndex, - AdvancedTimeIndex, AdvancedTimeDataIndex, + AdvancedTimeIndex, + ArchiveIndex, BaseTimeIndex, DataFileSystemIndex, - ArchiveIndex, + DataIndex, + FileSystemIndex, ForecastIndex, + Index, StaticDataIndex, + TimeDataIndex, + TimeIndex, ) -from pyearthtools.data.indexes.cacheIndex import ( - FileSystemCacheIndex, - CachingIndex, - CachingForecastIndex, - FunctionalCacheIndex, - MemCache, - FunctionalMemCacheIndex, -) -from pyearthtools.data.indexes import utilities, decorators -from pyearthtools.data.indexes.extensions import register_accessor - -from pyearthtools.data.indexes.utilities.spellcheck import VariableDefault, VARIABLE_DEFAULT -from pyearthtools.data.indexes.utilities.structure import structure - -from pyearthtools.data.indexes.decorators import alias_arguments, check_arguments - from pyearthtools.data.indexes.intake import IntakeIndex, IntakeIndexCache from pyearthtools.data.indexes.templates import Structured - -from pyearthtools.data.indexes.fake import FakeIndex - from pyearthtools.data.indexes.utilities.folder_size import ByteSize +from pyearthtools.data.indexes.utilities.spellcheck import ( + VARIABLE_DEFAULT, + VariableDefault, +) +from pyearthtools.data.indexes.utilities.structure import structure __all__ = [ "Index", diff --git a/packages/data/src/pyearthtools/data/indexes/cacheIndex.py b/packages/data/src/pyearthtools/data/indexes/cacheIndex.py index bb6a12f3..ce5f71a7 100644 --- a/packages/data/src/pyearthtools/data/indexes/cacheIndex.py +++ b/packages/data/src/pyearthtools/data/indexes/cacheIndex.py @@ -15,37 +15,38 @@ from __future__ import annotations +import logging import sys +import time +import warnings from abc import abstractmethod from functools import cached_property -from pathlib import Path -from typing import Any, Literal, Callable -import warnings -import logging from hashlib import sha512 -import time from multiprocessing import Process - -import xarray as xr +from pathlib import Path +from typing import Any, Callable, Literal import pyearthtools.data - -from pyearthtools.data import patterns, TimeDelta, DataNotFoundError -from pyearthtools.data.transforms import Transform, TransformCollection -from pyearthtools.data.patterns.default import PatternIndex -from pyearthtools.data.warnings import pyearthtoolsDataWarning - -from pyearthtools.data.indexes import ( +import xarray as xr +from pyearthtools.data import patterns +from pyearthtools.data.exceptions import DataNotFoundError +from pyearthtools.data.indexes.indexes import ( ArchiveIndex, + DataIndex, FileSystemIndex, ForecastIndex, - TimeIndex, - DataIndex, Index, + TimeIndex, +) +from pyearthtools.data.indexes.utilities.delete_files import ( + delete_older_than, + delete_path, ) -from pyearthtools.data.indexes.utilities.delete_files import delete_older_than, delete_path from pyearthtools.data.indexes.utilities.folder_size import ByteSize, FolderSize - +from pyearthtools.data.patterns.default import PatternIndex +from pyearthtools.data.time import TimeDelta +from pyearthtools.data.transforms import Transform, TransformCollection +from pyearthtools.data.warnings import pyearthtoolsDataWarning from pyearthtools.utils.context import ChangeValue LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/src/pyearthtools/data/indexes/combine.py b/packages/data/src/pyearthtools/data/indexes/combine.py index ed21ac54..443c90da 100644 --- a/packages/data/src/pyearthtools/data/indexes/combine.py +++ b/packages/data/src/pyearthtools/data/indexes/combine.py @@ -16,10 +16,8 @@ from __future__ import annotations import xarray as xr - - -from pyearthtools.data.indexes import Index, AdvancedTimeDataIndex from pyearthtools.data import Petdt +from pyearthtools.data.indexes import AdvancedTimeDataIndex, Index from pyearthtools.data.operations import SpatialInterpolation, TemporalInterpolation from pyearthtools.data.transforms.transform import Transform, TransformCollection diff --git a/packages/data/src/pyearthtools/data/indexes/decorators.py b/packages/data/src/pyearthtools/data/indexes/decorators.py index 80c08ab9..7a2331bb 100644 --- a/packages/data/src/pyearthtools/data/indexes/decorators.py +++ b/packages/data/src/pyearthtools/data/indexes/decorators.py @@ -20,16 +20,14 @@ from __future__ import annotations import functools -import warnings - import inspect +import warnings from pathlib import Path from typing import Any -from pyearthtools.data.indexes.utilities import spellcheck, open_static -from pyearthtools.utils.decorators import alias_arguments - +from pyearthtools.data.indexes.utilities import open_static, spellcheck from pyearthtools.data.modifications import variable_modifications +from pyearthtools.utils.decorators import alias_arguments __all__ = [ "alias_arguments", diff --git a/packages/data/src/pyearthtools/data/indexes/fake.py b/packages/data/src/pyearthtools/data/indexes/fake.py index 7fe3c0ad..ca03f8b1 100644 --- a/packages/data/src/pyearthtools/data/indexes/fake.py +++ b/packages/data/src/pyearthtools/data/indexes/fake.py @@ -15,13 +15,11 @@ from __future__ import annotations -import xarray as xr import numpy as np - -from pyearthtools.data.time import Petdt -from pyearthtools.data.indexes.indexes import AdvancedTimeDataIndex - +import xarray as xr from pyearthtools.data.indexes.decorators import variable_modifications +from pyearthtools.data.indexes.indexes import AdvancedTimeDataIndex +from pyearthtools.data.time import Petdt class FakeIndex(AdvancedTimeDataIndex): diff --git a/packages/data/src/pyearthtools/data/indexes/indexes.py b/packages/data/src/pyearthtools/data/indexes/indexes.py index 615a9514..543230df 100644 --- a/packages/data/src/pyearthtools/data/indexes/indexes.py +++ b/packages/data/src/pyearthtools/data/indexes/indexes.py @@ -28,39 +28,32 @@ from __future__ import annotations -import warnings import datetime import functools import logging -from abc import abstractmethod, ABCMeta - +import warnings +from abc import ABCMeta, abstractmethod from pathlib import Path from typing import Any, Callable, Iterable, Literal, Optional -import xarray as xr - import pyearthtools.data import pyearthtools.utils - +import xarray as xr +from pyearthtools.data import operations +from pyearthtools.data.exceptions import DataNotFoundError +from pyearthtools.data.indexes.utilities import dimensions, open_files +from pyearthtools.data.indexes.utilities.mixins import ( + CallRedirectMixin, + CatalogMixin, +) +from pyearthtools.data.operations import forecast_op, index_operations, index_routines from pyearthtools.data.time import Petdt, TimeDelta, TimeRange - from pyearthtools.data.transforms.transform import ( + FunctionTransform, Transform, TransformCollection, - FunctionTransform, ) - from pyearthtools.data.warnings import IndexWarning -from pyearthtools.data.exceptions import DataNotFoundError -from pyearthtools.data.operations import index_routines, index_operations, forecast_op -from pyearthtools.data import operations - -from pyearthtools.data.indexes.utilities.mixins import ( - CallRedirectMixin, - CatalogMixin, -) -from pyearthtools.data.indexes.utilities import open_files, dimensions - from pyearthtools.utils.context import ChangeValue LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/src/pyearthtools/data/indexes/intake.py b/packages/data/src/pyearthtools/data/indexes/intake.py index 7f7d5638..d149a77d 100644 --- a/packages/data/src/pyearthtools/data/indexes/intake.py +++ b/packages/data/src/pyearthtools/data/indexes/intake.py @@ -19,19 +19,17 @@ from __future__ import annotations +import functools +import logging from pathlib import Path from typing import Any, Callable import xarray as xr -import logging -import functools - -from pyearthtools.data.indexes.indexes import DataIndex from pyearthtools.data.indexes.cacheIndex import FileSystemCacheIndex -from pyearthtools.data.transforms import Transform, TransformCollection -from pyearthtools.data.patterns.argument import flattened_combinations - +from pyearthtools.data.indexes.indexes import DataIndex from pyearthtools.data.indexes.utilities.delete_files import delete_path +from pyearthtools.data.patterns.argument import flattened_combinations +from pyearthtools.data.transforms import Transform, TransformCollection LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/src/pyearthtools/data/indexes/templates/structured.py b/packages/data/src/pyearthtools/data/indexes/templates/structured.py index ac6920d3..db7b17be 100644 --- a/packages/data/src/pyearthtools/data/indexes/templates/structured.py +++ b/packages/data/src/pyearthtools/data/indexes/templates/structured.py @@ -18,11 +18,15 @@ """ from __future__ import annotations + from pathlib import Path -from pyearthtools.data import Petdt from pyearthtools.data.indexes import ArchiveIndex, decorators -from pyearthtools.data.indexes.utilities.spellcheck import VARIABLE_DEFAULT, VariableDefault +from pyearthtools.data.indexes.utilities.spellcheck import ( + VARIABLE_DEFAULT, + VariableDefault, +) +from pyearthtools.data.time import Petdt from pyearthtools.data.transforms import Transform, TransformCollection diff --git a/packages/data/src/pyearthtools/data/indexes/utilities/__init__.py b/packages/data/src/pyearthtools/data/indexes/utilities/__init__.py index 8207006a..25eb68e5 100644 --- a/packages/data/src/pyearthtools/data/indexes/utilities/__init__.py +++ b/packages/data/src/pyearthtools/data/indexes/utilities/__init__.py @@ -17,6 +17,6 @@ Various utilites for `pyearthtools.data.indexes` """ -from pyearthtools.data.indexes.utilities import mixins, delete_files +from pyearthtools.data.indexes.utilities import delete_files, mixins from pyearthtools.data.indexes.utilities.fileload import open_files, open_static from pyearthtools.data.indexes.utilities.spellcheck import check_prompt diff --git a/packages/data/src/pyearthtools/data/indexes/utilities/delete_files.py b/packages/data/src/pyearthtools/data/indexes/utilities/delete_files.py index b240a9a9..b6ce9b76 100644 --- a/packages/data/src/pyearthtools/data/indexes/utilities/delete_files.py +++ b/packages/data/src/pyearthtools/data/indexes/utilities/delete_files.py @@ -19,14 +19,14 @@ from __future__ import annotations +import logging import os -from pathlib import Path import shutil import time +from pathlib import Path from typing import Literal, Sequence -import logging -from pyearthtools.data import TimeDelta +from pyearthtools.data.time import TimeDelta from pyearthtools.utils.context import Catch LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/src/pyearthtools/data/indexes/utilities/dimensions.py b/packages/data/src/pyearthtools/data/indexes/utilities/dimensions.py index 4ec0e6f3..ce794549 100644 --- a/packages/data/src/pyearthtools/data/indexes/utilities/dimensions.py +++ b/packages/data/src/pyearthtools/data/indexes/utilities/dimensions.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import annotations + from typing import Hashable import xarray as xr diff --git a/packages/data/src/pyearthtools/data/indexes/utilities/fileload.py b/packages/data/src/pyearthtools/data/indexes/utilities/fileload.py index bc395416..7268c0bd 100644 --- a/packages/data/src/pyearthtools/data/indexes/utilities/fileload.py +++ b/packages/data/src/pyearthtools/data/indexes/utilities/fileload.py @@ -14,21 +14,19 @@ from __future__ import annotations + +import copy import json import os +from importlib.resources import as_file, files +from pathlib import Path from typing import Any, Callable -import xarray as xr import numpy as np import pandas as pd - -from pathlib import Path -from importlib.resources import files, as_file -import yaml -import copy - - import pyearthtools.utils +import xarray as xr +import yaml pyearthtools_CATALOG_EXTENSIONS = [".cat", ".catalog"] BLACKLISTED_EXTENSIONS = [] diff --git a/packages/data/src/pyearthtools/data/indexes/utilities/folder_size.py b/packages/data/src/pyearthtools/data/indexes/utilities/folder_size.py index 267793cd..266cdaae 100644 --- a/packages/data/src/pyearthtools/data/indexes/utilities/folder_size.py +++ b/packages/data/src/pyearthtools/data/indexes/utilities/folder_size.py @@ -21,14 +21,15 @@ # from __future__ import annotations -import os -from typing import Literal, Sequence -from pathlib import Path import functools -import yaml +import os import re import time +from pathlib import Path +from typing import Literal, Sequence + +import yaml @functools.total_ordering diff --git a/packages/data/src/pyearthtools/data/indexes/utilities/mixins/__init__.py b/packages/data/src/pyearthtools/data/indexes/utilities/mixins/__init__.py index f742ffc7..56dc4055 100644 --- a/packages/data/src/pyearthtools/data/indexes/utilities/mixins/__init__.py +++ b/packages/data/src/pyearthtools/data/indexes/utilities/mixins/__init__.py @@ -18,6 +18,6 @@ """ -from pyearthtools.data.indexes.utilities.mixins.index_repr import reprMixin from pyearthtools.data.indexes.utilities.mixins.call_redirect import CallRedirectMixin from pyearthtools.data.indexes.utilities.mixins.catalogs import CatalogMixin +from pyearthtools.data.indexes.utilities.mixins.index_repr import reprMixin diff --git a/packages/data/src/pyearthtools/data/indexes/utilities/mixins/catalogs.py b/packages/data/src/pyearthtools/data/indexes/utilities/mixins/catalogs.py index 97802b62..437da18e 100644 --- a/packages/data/src/pyearthtools/data/indexes/utilities/mixins/catalogs.py +++ b/packages/data/src/pyearthtools/data/indexes/utilities/mixins/catalogs.py @@ -16,12 +16,10 @@ from __future__ import annotations import warnings - from pathlib import Path -from pyearthtools.utils import initialisation from pyearthtools.data.utils import parse_path - +from pyearthtools.utils import initialisation SUFFIX = "edi" diff --git a/packages/data/src/pyearthtools/data/indexes/utilities/mixins/index_repr.py b/packages/data/src/pyearthtools/data/indexes/utilities/mixins/index_repr.py index bae1eb82..27f14878 100644 --- a/packages/data/src/pyearthtools/data/indexes/utilities/mixins/index_repr.py +++ b/packages/data/src/pyearthtools/data/indexes/utilities/mixins/index_repr.py @@ -21,7 +21,6 @@ except ImportError: UTILS_REPR = False -import pyearthtools.data import pyearthtools.data.catalog diff --git a/packages/data/src/pyearthtools/data/indexes/utilities/spellcheck.py b/packages/data/src/pyearthtools/data/indexes/utilities/spellcheck.py index cd5e8b83..925ef880 100644 --- a/packages/data/src/pyearthtools/data/indexes/utilities/spellcheck.py +++ b/packages/data/src/pyearthtools/data/indexes/utilities/spellcheck.py @@ -18,6 +18,7 @@ """ from __future__ import annotations + from typing import Any, Type from pyearthtools.data.exceptions import InvalidIndexError diff --git a/packages/data/src/pyearthtools/data/indexes/utilities/structure.py b/packages/data/src/pyearthtools/data/indexes/utilities/structure.py index 4c14ef5e..6ca6d726 100644 --- a/packages/data/src/pyearthtools/data/indexes/utilities/structure.py +++ b/packages/data/src/pyearthtools/data/indexes/utilities/structure.py @@ -15,8 +15,8 @@ from __future__ import annotations -from pathlib import Path import os +from pathlib import Path from typing import Any import tqdm.auto as tqdm diff --git a/packages/data/src/pyearthtools/data/load.py b/packages/data/src/pyearthtools/data/load.py index 42f572ef..de070117 100644 --- a/packages/data/src/pyearthtools/data/load.py +++ b/packages/data/src/pyearthtools/data/load.py @@ -15,17 +15,14 @@ """Saving and Loading of `Indexes`""" -from typing import Union - -from pathlib import Path import os - -import yaml - -from pyearthtools.utils import initialisation +from pathlib import Path +from typing import Union import pyearthtools.data +import yaml from pyearthtools.data.utils import parse_path +from pyearthtools.utils import initialisation CONFIG_KEY = "--CONFIG--" diff --git a/packages/data/src/pyearthtools/data/modifications/__init__.py b/packages/data/src/pyearthtools/data/modifications/__init__.py index a8798b67..e4f9c929 100644 --- a/packages/data/src/pyearthtools/data/modifications/__init__.py +++ b/packages/data/src/pyearthtools/data/modifications/__init__.py @@ -108,12 +108,9 @@ class AggregationGeneral(Aggregation): """ +from pyearthtools.data.modifications import aggregations, constants, reductions +from pyearthtools.data.modifications.decorator import variable_modifications from pyearthtools.data.modifications.modification import Modification - from pyearthtools.data.modifications.register import register_modification -from pyearthtools.data.modifications.decorator import variable_modifications - -from pyearthtools.data.modifications import aggregations, reductions, constants - __all__ = ["register_modification", "variable_modifications", "Modification"] diff --git a/packages/data/src/pyearthtools/data/modifications/aggregations.py b/packages/data/src/pyearthtools/data/modifications/aggregations.py index b1e7f12e..761c6130 100644 --- a/packages/data/src/pyearthtools/data/modifications/aggregations.py +++ b/packages/data/src/pyearthtools/data/modifications/aggregations.py @@ -20,12 +20,12 @@ from __future__ import annotations from typing import Literal -import xarray as xr -from pyearthtools.data.time import Petdt, TimeDelta, TimeResolution, TimeRange +import xarray as xr from pyearthtools.data.indexes.utilities.dimensions import identify_time_dimension - -from pyearthtools.data.modifications import Modification, register_modification +from pyearthtools.data.modifications.modification import Modification +from pyearthtools.data.modifications.register import register_modification +from pyearthtools.data.time import Petdt, TimeDelta, TimeRange, TimeResolution class Aggregation(Modification): diff --git a/packages/data/src/pyearthtools/data/modifications/constants.py b/packages/data/src/pyearthtools/data/modifications/constants.py index 6e6699a7..46719df1 100644 --- a/packages/data/src/pyearthtools/data/modifications/constants.py +++ b/packages/data/src/pyearthtools/data/modifications/constants.py @@ -20,12 +20,12 @@ from __future__ import annotations from typing import Optional, Union -import xarray as xr -from pyearthtools.data.time import Petdt, TimeRange +import xarray as xr from pyearthtools.data.indexes.utilities.dimensions import identify_time_dimension - -from pyearthtools.data.modifications import Modification, register_modification +from pyearthtools.data.modifications.modification import Modification +from pyearthtools.data.modifications.register import register_modification +from pyearthtools.data.time import Petdt, TimeRange @register_modification("constant") diff --git a/packages/data/src/pyearthtools/data/modifications/decorator.py b/packages/data/src/pyearthtools/data/modifications/decorator.py index c5032a88..aac3859b 100644 --- a/packages/data/src/pyearthtools/data/modifications/decorator.py +++ b/packages/data/src/pyearthtools/data/modifications/decorator.py @@ -19,21 +19,17 @@ from __future__ import annotations +import functools import inspect -import re import json +import re +from typing import Any, Callable, Optional, Type, Union -import functools -from typing import Callable, Any, Optional, Type, Union import xarray as xr - -from pyearthtools.data.indexes import TimeDataIndex - -from pyearthtools.data.transforms.transform import Transform +from pyearthtools.data.indexes.indexes import TimeDataIndex from pyearthtools.data.modifications.modification import Modification - from pyearthtools.data.modifications.register import MODIFICATION_DICT - +from pyearthtools.data.transforms.transform import Transform __all__ = ["variable_modifications", "Modification"] diff --git a/packages/data/src/pyearthtools/data/modifications/modification.py b/packages/data/src/pyearthtools/data/modifications/modification.py index 6b82b277..db2cb0f1 100644 --- a/packages/data/src/pyearthtools/data/modifications/modification.py +++ b/packages/data/src/pyearthtools/data/modifications/modification.py @@ -20,14 +20,13 @@ from __future__ import annotations from abc import ABCMeta, abstractmethod - from typing import Any, Union -import xarray as xr import pyearthtools.data -from pyearthtools.data.indexes import TimeDataIndex +import pyearthtools.data.indexes.indexes +import xarray as xr +from pyearthtools.data.indexes.indexes import TimeDataIndex from pyearthtools.data.indexes.utilities.dimensions import identify_time_dimension - from pyearthtools.data.transforms.transform import TransformCollection diff --git a/packages/data/src/pyearthtools/data/modifications/reductions.py b/packages/data/src/pyearthtools/data/modifications/reductions.py index ffe8986e..ca8fbd8d 100644 --- a/packages/data/src/pyearthtools/data/modifications/reductions.py +++ b/packages/data/src/pyearthtools/data/modifications/reductions.py @@ -20,12 +20,12 @@ from __future__ import annotations from typing import Union -import xarray as xr -from pyearthtools.data.time import Petdt, TimeDelta, TimeResolution +import xarray as xr from pyearthtools.data.indexes.utilities.dimensions import identify_time_dimension - -from pyearthtools.data.modifications import Modification, register_modification +from pyearthtools.data.modifications.modification import Modification +from pyearthtools.data.modifications.register import register_modification +from pyearthtools.data.time import Petdt, TimeDelta, TimeResolution class Reduction(Modification): diff --git a/packages/data/src/pyearthtools/data/modifications/register.py b/packages/data/src/pyearthtools/data/modifications/register.py index 40344469..6b0d6cca 100644 --- a/packages/data/src/pyearthtools/data/modifications/register.py +++ b/packages/data/src/pyearthtools/data/modifications/register.py @@ -19,8 +19,8 @@ from __future__ import annotations -from typing import Callable, Any, Type import warnings +from typing import Any, Callable, Type import pyearthtools.data diff --git a/packages/data/src/pyearthtools/data/operations/__init__.py b/packages/data/src/pyearthtools/data/operations/__init__.py index 853b1f5b..5f316012 100644 --- a/packages/data/src/pyearthtools/data/operations/__init__.py +++ b/packages/data/src/pyearthtools/data/operations/__init__.py @@ -32,13 +32,13 @@ """ from pyearthtools.data.operations import interpolation +from pyearthtools.data.operations.aggregation import aggregation +from pyearthtools.data.operations.binning import binning from pyearthtools.data.operations.interpolation import ( + FullInterpolation, SpatialInterpolation, TemporalInterpolation, - FullInterpolation, ) from pyearthtools.data.operations.percentile import percentile -from pyearthtools.data.operations.aggregation import aggregation -from pyearthtools.data.operations.binning import binning # from pyearthtools.data.operations.index_routines import safe_series, series diff --git a/packages/data/src/pyearthtools/data/operations/aggregation.py b/packages/data/src/pyearthtools/data/operations/aggregation.py index d2b9156b..6ff4189e 100644 --- a/packages/data/src/pyearthtools/data/operations/aggregation.py +++ b/packages/data/src/pyearthtools/data/operations/aggregation.py @@ -16,8 +16,8 @@ from __future__ import annotations from typing import Callable -import xarray as xr +import xarray as xr from pyearthtools.data.transforms import aggregation as aggr_trans diff --git a/packages/data/src/pyearthtools/data/operations/binning.py b/packages/data/src/pyearthtools/data/operations/binning.py index f6ddcf59..94c61218 100644 --- a/packages/data/src/pyearthtools/data/operations/binning.py +++ b/packages/data/src/pyearthtools/data/operations/binning.py @@ -19,11 +19,10 @@ from __future__ import annotations -import xarray as xr from typing import Literal -from pyearthtools.data import TimeDelta - +import xarray as xr +from pyearthtools.data.time import TimeDelta BINNING_SETUP = { # Base Binning setup "seasonal": [ diff --git a/packages/data/src/pyearthtools/data/operations/forecast_op.py b/packages/data/src/pyearthtools/data/operations/forecast_op.py index ac8573e6..3f559d24 100644 --- a/packages/data/src/pyearthtools/data/operations/forecast_op.py +++ b/packages/data/src/pyearthtools/data/operations/forecast_op.py @@ -15,18 +15,18 @@ from __future__ import annotations - import xarray as xr - - -from pyearthtools.data.exceptions import DataNotFoundError, InvalidIndexError, InvalidDataError +from pyearthtools.data.exceptions import ( + DataNotFoundError, + InvalidDataError, + InvalidIndexError, +) +from pyearthtools.data.operations import index_routines +from pyearthtools.data.operations.utils import identify_time_dimension from pyearthtools.data.time import Petdt, TimeDelta, TimeRange from pyearthtools.data.transforms.transform import Transform, TransformCollection from pyearthtools.data.warnings import IndexWarning -from pyearthtools.data.operations import index_routines -from pyearthtools.data.operations.utils import identify_time_dimension - def forecast_series( DataFunction: "Index", diff --git a/packages/data/src/pyearthtools/data/operations/index_operations.py b/packages/data/src/pyearthtools/data/operations/index_operations.py index 0da55244..a5663af6 100644 --- a/packages/data/src/pyearthtools/data/operations/index_operations.py +++ b/packages/data/src/pyearthtools/data/operations/index_operations.py @@ -19,25 +19,23 @@ from __future__ import annotations -import datetime import builtins +import datetime import os from pathlib import Path from typing import Callable, Union import numpy as np -from tqdm.auto import tqdm, trange -import xarray as xr - import pyearthtools.data - -from pyearthtools.data.time import Petdt, TimeDelta +import xarray as xr from pyearthtools.data.exceptions import ( DataNotFoundError, InvalidIndexError, run_and_catch_exception, ) +from pyearthtools.data.time import Petdt, TimeDelta from pyearthtools.data.transforms import Transform, TransformCollection +from tqdm.auto import tqdm, trange def split_ds(dataset: xr.Dataset, divisions: int = 1, dim: str = "time") -> list[xr.Dataset]: diff --git a/packages/data/src/pyearthtools/data/operations/index_routines.py b/packages/data/src/pyearthtools/data/operations/index_routines.py index 9fa9e578..a8b890ff 100644 --- a/packages/data/src/pyearthtools/data/operations/index_routines.py +++ b/packages/data/src/pyearthtools/data/operations/index_routines.py @@ -17,25 +17,24 @@ import functools import logging -from typing import Iterable - import warnings +from typing import Iterable import numpy as np import pandas as pd -import xarray as xr - import pyearthtools - import pyearthtools.data import pyearthtools.utils - -from pyearthtools.data.exceptions import DataNotFoundError, InvalidIndexError, InvalidDataError -from pyearthtools.data.time import Petdt, TimeDelta, time_delta_resolution, TimeRange +import xarray as xr +from pyearthtools.data.exceptions import ( + DataNotFoundError, + InvalidDataError, + InvalidIndexError, +) +from pyearthtools.data.operations.utils import identify_time_dimension +from pyearthtools.data.time import Petdt, TimeDelta, TimeRange, time_delta_resolution from pyearthtools.data.transforms.transform import Transform, TransformCollection from pyearthtools.data.warnings import IndexWarning -from pyearthtools.data.operations.utils import identify_time_dimension - LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/src/pyearthtools/data/operations/interpolation.py b/packages/data/src/pyearthtools/data/operations/interpolation.py index 75ee92dc..b61ec68d 100644 --- a/packages/data/src/pyearthtools/data/operations/interpolation.py +++ b/packages/data/src/pyearthtools/data/operations/interpolation.py @@ -22,10 +22,8 @@ from typing import Callable import numpy as np -import xarray as xr - - import pyearthtools.data +import xarray as xr from pyearthtools.data.transforms import interpolation as interp diff --git a/packages/data/src/pyearthtools/data/operations/utils.py b/packages/data/src/pyearthtools/data/operations/utils.py index df985024..65ae54f4 100644 --- a/packages/data/src/pyearthtools/data/operations/utils.py +++ b/packages/data/src/pyearthtools/data/operations/utils.py @@ -1,4 +1,5 @@ from __future__ import annotations + from typing import Hashable import xarray as xr diff --git a/packages/data/src/pyearthtools/data/patterns/__init__.py b/packages/data/src/pyearthtools/data/patterns/__init__.py index cd4c26fc..c50ce098 100644 --- a/packages/data/src/pyearthtools/data/patterns/__init__.py +++ b/packages/data/src/pyearthtools/data/patterns/__init__.py @@ -52,41 +52,38 @@ """ from pyearthtools.data.patterns import utils - -from pyearthtools.data.patterns.default import ( - PatternIndex, - PatternTimeIndex, - PatternForecastIndex, - PatternVariableAware, -) - from pyearthtools.data.patterns.argument import ( Argument, ArgumentExpansion, - ArgumentExpansionVariable, ArgumentExpansionFactory, + ArgumentExpansionVariable, +) +from pyearthtools.data.patterns.default import ( + PatternForecastIndex, + PatternIndex, + PatternTimeIndex, + PatternVariableAware, ) from pyearthtools.data.patterns.direct import ( Direct, - TemporalDirect, - ForecastDirect, + DirectFactory, DirectVariable, + ForecastDirect, ForecastDirectVariable, + TemporalDirect, TemporalDirectVariable, - DirectFactory, ) from pyearthtools.data.patterns.expanded_date import ( ExpandedDate, - TemporalExpandedDate, - ForecastExpandedDate, + ExpandedDateFactory, ExpandedDateVariable, + ForecastExpandedDate, ForecastExpandedDateVariable, + TemporalExpandedDate, TemporalExpandedDateVariable, - ExpandedDateFactory, ) -from pyearthtools.data.patterns.static import Static from pyearthtools.data.patterns.parser import ParsingPattern - +from pyearthtools.data.patterns.static import Static ZARR_IMPORTED = True try: diff --git a/packages/data/src/pyearthtools/data/patterns/argument.py b/packages/data/src/pyearthtools/data/patterns/argument.py index ed537923..689dbd8b 100644 --- a/packages/data/src/pyearthtools/data/patterns/argument.py +++ b/packages/data/src/pyearthtools/data/patterns/argument.py @@ -18,18 +18,22 @@ from pathlib import Path from typing import Any, Iterable -from pyearthtools.data.patterns import PatternIndex, PatternVariableAware +import pyearthtools.utils from pyearthtools.data.indexes import decorators from pyearthtools.data.indexes.utilities import spellcheck - -import pyearthtools.utils +from pyearthtools.data.patterns.default import PatternIndex, PatternVariableAware from pyearthtools.utils.decorators import classproperty """ Generate FilePath Structure based upon expansion of arguments """ -DEFAULT_EXTENSION = pyearthtools.utils.config.get("data.patterns.default_extension") +import pyearthtools.data.logger # Initialise data logger + +try: + DEFAULT_EXTENSION = pyearthtools.utils.config.get("data.patterns.default_extension") +except: + DEFAULT_EXTENSION = ".nc" def flattened_combinations(iterable: Iterable[Any | Iterable[Any]], r: int = 1) -> list[Any | list[Any]]: diff --git a/packages/data/src/pyearthtools/data/patterns/default.py b/packages/data/src/pyearthtools/data/patterns/default.py index 008bad1f..4df15686 100644 --- a/packages/data/src/pyearthtools/data/patterns/default.py +++ b/packages/data/src/pyearthtools/data/patterns/default.py @@ -14,26 +14,25 @@ from __future__ import annotations -from abc import abstractmethod import os +import warnings +from abc import abstractmethod from pathlib import Path from typing import Any, Callable -import warnings import xarray as xr - from pyearthtools.data import patterns from pyearthtools.data.exceptions import DataNotFoundError -from pyearthtools.data.warnings import pyearthtoolsDataWarning -from pyearthtools.data.indexes import ( +from pyearthtools.data.indexes import decorators +from pyearthtools.data.indexes.indexes import ( + AdvancedTimeIndex, DataIndex, FileSystemIndex, - AdvancedTimeIndex, ForecastIndex, - decorators, ) from pyearthtools.data.save import save +from pyearthtools.data.warnings import pyearthtoolsDataWarning class PatternIndex(DataIndex, FileSystemIndex): diff --git a/packages/data/src/pyearthtools/data/patterns/direct.py b/packages/data/src/pyearthtools/data/patterns/direct.py index 2426cdc5..625b636f 100644 --- a/packages/data/src/pyearthtools/data/patterns/direct.py +++ b/packages/data/src/pyearthtools/data/patterns/direct.py @@ -26,21 +26,27 @@ from pathlib import Path from typing import Any -from pyearthtools.data.time import Petdt, TimeResolution +import pyearthtools.utils +from pyearthtools.data.indexes import decorators +from pyearthtools.data.indexes.indexes import TimeIndex from pyearthtools.data.patterns import ( + PatternForecastIndex, PatternIndex, PatternTimeIndex, PatternVariableAware, - PatternForecastIndex, ) -from pyearthtools.data.indexes import TimeIndex, decorators - -import pyearthtools.utils +from pyearthtools.data.time import Petdt, TimeResolution from pyearthtools.utils.decorators import classproperty DIRECTORY_PATTERN = "{ROOT_DIR}/{FILE}" FILE_PATTERN = "{prefix}{time}{extension}" -DEFAULT_EXTENSION = pyearthtools.utils.config.get("data.patterns.default_extension") + +import pyearthtools.data.logger + +try: + DEFAULT_EXTENSION = pyearthtools.utils.config.get("data.patterns.default_extension") +except: + DEFAULT_EXTENSION = ".nc" def nonNone(*args): diff --git a/packages/data/src/pyearthtools/data/patterns/expanded_date.py b/packages/data/src/pyearthtools/data/patterns/expanded_date.py index 952a8dee..d38d04cf 100644 --- a/packages/data/src/pyearthtools/data/patterns/expanded_date.py +++ b/packages/data/src/pyearthtools/data/patterns/expanded_date.py @@ -25,22 +25,26 @@ from pathlib import Path from typing import Any -from pyearthtools.data import Petdt, TimeResolution +import pyearthtools.utils +from pyearthtools.data.indexes import decorators from pyearthtools.data.patterns.default import ( + PatternForecastIndex, PatternIndex, PatternTimeIndex, - PatternForecastIndex, PatternVariableAware, ) -from pyearthtools.data.indexes import decorators - -import pyearthtools.utils +from pyearthtools.data.time import Petdt, TimeResolution from pyearthtools.utils.decorators import classproperty - DIRECTORY_PATTERN = "{ROOT_DIR}/{FILE_DATE}/{FILE}" FILE_PATTERN = "{prefix}{time}{extension}" -DEFAULT_EXTENSION = pyearthtools.utils.config.get("data.patterns.default_extension") + +import pyearthtools.data.logger + +try: + DEFAULT_EXTENSION = pyearthtools.utils.config.get("data.patterns.default_extension") +except: + DEFAULT_EXTENSION = ".nc" def parse_time_str(time, directory: bool = False, delimiter: str | tuple | list = "") -> str: diff --git a/packages/data/src/pyearthtools/data/patterns/parser.py b/packages/data/src/pyearthtools/data/patterns/parser.py index f1077579..f620ac91 100644 --- a/packages/data/src/pyearthtools/data/patterns/parser.py +++ b/packages/data/src/pyearthtools/data/patterns/parser.py @@ -20,21 +20,18 @@ from __future__ import annotations -from collections import OrderedDict import itertools -from typing import Any, Callable +from collections import OrderedDict from collections.abc import Iterable - from pathlib import Path +from string import Formatter +from typing import Any, Callable import pandas as pd import xarray as xr - -from string import Formatter - +from pyearthtools.data.exceptions import DataNotFoundError from pyearthtools.data.patterns import PatternIndex from pyearthtools.data.transforms import Transform, TransformCollection -from pyearthtools.data.exceptions import DataNotFoundError def update_value(old_val: Any | list[Any] | tuple[Any, ...], new_vals: Any) -> list[Any]: diff --git a/packages/data/src/pyearthtools/data/patterns/static.py b/packages/data/src/pyearthtools/data/patterns/static.py index bd410595..da291d68 100644 --- a/packages/data/src/pyearthtools/data/patterns/static.py +++ b/packages/data/src/pyearthtools/data/patterns/static.py @@ -19,7 +19,6 @@ from typing import Any import pyearthtools.data - from pyearthtools.data.patterns import PatternIndex from pyearthtools.data.transforms import Transform, TransformCollection diff --git a/packages/data/src/pyearthtools/data/patterns/utils.py b/packages/data/src/pyearthtools/data/patterns/utils.py index 780c767a..43efb1a2 100644 --- a/packages/data/src/pyearthtools/data/patterns/utils.py +++ b/packages/data/src/pyearthtools/data/patterns/utils.py @@ -15,10 +15,9 @@ from __future__ import annotations -import tempfile -import re import os - +import re +import tempfile from pathlib import Path diff --git a/packages/data/src/pyearthtools/data/patterns/zarr.py b/packages/data/src/pyearthtools/data/patterns/zarr.py index 71e8cf63..4ae3d0fe 100644 --- a/packages/data/src/pyearthtools/data/patterns/zarr.py +++ b/packages/data/src/pyearthtools/data/patterns/zarr.py @@ -15,9 +15,8 @@ import os -from pyearthtools.data.archive import zarr - from pyearthtools.data import patterns +from pyearthtools.data.archive.zarr import ZarrIndex from pyearthtools.data.patterns.default import PatternIndex diff --git a/packages/data/src/pyearthtools/data/save/array.py b/packages/data/src/pyearthtools/data/save/array.py index 9b7373c6..ce8a1d39 100644 --- a/packages/data/src/pyearthtools/data/save/array.py +++ b/packages/data/src/pyearthtools/data/save/array.py @@ -19,8 +19,7 @@ from typing import Any import numpy as np - -from pyearthtools.data.indexes import FileSystemIndex +from pyearthtools.data.indexes.indexes import FileSystemIndex from pyearthtools.data.save.save_utils import ManageFiles VALID_EXTENSIONS = [".npy", ".numpy"] diff --git a/packages/data/src/pyearthtools/data/save/dask.py b/packages/data/src/pyearthtools/data/save/dask.py index aff4849b..43f12804 100644 --- a/packages/data/src/pyearthtools/data/save/dask.py +++ b/packages/data/src/pyearthtools/data/save/dask.py @@ -20,9 +20,8 @@ from pathlib import Path from typing import Any -from pyearthtools.data.save.array import save as numpy_save - import dask.array as da +from pyearthtools.data.save.array import save as numpy_save def save( diff --git a/packages/data/src/pyearthtools/data/save/dataset.py b/packages/data/src/pyearthtools/data/save/dataset.py index 5203935e..11f8ac4c 100644 --- a/packages/data/src/pyearthtools/data/save/dataset.py +++ b/packages/data/src/pyearthtools/data/save/dataset.py @@ -15,14 +15,13 @@ from __future__ import annotations +import warnings from pathlib import Path from typing import Any -import warnings - -import xarray as xr import pyearthtools.utils -from pyearthtools.data.indexes import FileSystemIndex +import xarray as xr +from pyearthtools.data.indexes.indexes import FileSystemIndex from pyearthtools.data.save.save_utils import ManageFiles VALID_EXTENSIONS = [".nc", ".netcdf"] diff --git a/packages/data/src/pyearthtools/data/save/dispatcher.py b/packages/data/src/pyearthtools/data/save/dispatcher.py index 4e1819bb..aa2e86d9 100644 --- a/packages/data/src/pyearthtools/data/save/dispatcher.py +++ b/packages/data/src/pyearthtools/data/save/dispatcher.py @@ -14,23 +14,21 @@ from __future__ import annotations -from typing import Any -import xarray as xr -from xarray import plot as xplt +from typing import Any import numpy as np - +import xarray as xr from matplotlib.figure import Figure - -from pyearthtools.data.indexes import FileSystemIndex -from pyearthtools.data.save import dataset, json, array, plot +from pyearthtools.data.indexes.indexes import FileSystemIndex +from pyearthtools.data.save import array, dataset, json, plot +from xarray import plot as xplt DASK_IMPORTED = True try: import dask - from dask.delayed import Delayed import dask.array as da + from dask.delayed import Delayed from pyearthtools.data.save import dask as dask_save except (ImportError, ModuleNotFoundError) as e: DASK_IMPORTED = False diff --git a/packages/data/src/pyearthtools/data/save/json.py b/packages/data/src/pyearthtools/data/save/json.py index d07e3616..e55585ca 100644 --- a/packages/data/src/pyearthtools/data/save/json.py +++ b/packages/data/src/pyearthtools/data/save/json.py @@ -19,7 +19,7 @@ from pathlib import Path from typing import Any -from pyearthtools.data.indexes import FileSystemIndex +from pyearthtools.data.indexes.indexes import FileSystemIndex def save( diff --git a/packages/data/src/pyearthtools/data/save/plot.py b/packages/data/src/pyearthtools/data/save/plot.py index 6b8910e3..4583dcce 100644 --- a/packages/data/src/pyearthtools/data/save/plot.py +++ b/packages/data/src/pyearthtools/data/save/plot.py @@ -17,7 +17,7 @@ from pathlib import Path -from pyearthtools.data.indexes import FileSystemIndex +from pyearthtools.data.indexes.indexes import FileSystemIndex def save( diff --git a/packages/data/src/pyearthtools/data/save/save_utils.py b/packages/data/src/pyearthtools/data/save/save_utils.py index 7b7cbebb..aa0a16f6 100644 --- a/packages/data/src/pyearthtools/data/save/save_utils.py +++ b/packages/data/src/pyearthtools/data/save/save_utils.py @@ -15,17 +15,15 @@ from __future__ import annotations -from pathlib import Path -import uuid -import os - import logging +import os +import uuid import warnings +from pathlib import Path +from typing import Union from filelock import FileLock, Timeout -from typing import Union - VALID_PATH_TYPES = Union[str, Path, list[str], list[Path], list[Union[str, Path]]] LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/src/pyearthtools/data/static/_geographic/__init__.py b/packages/data/src/pyearthtools/data/static/_geographic/__init__.py index c9804a64..676363d6 100644 --- a/packages/data/src/pyearthtools/data/static/_geographic/__init__.py +++ b/packages/data/src/pyearthtools/data/static/_geographic/__init__.py @@ -26,5 +26,5 @@ DOWNLOAD_DATA: bool = True DATA_BASEDIRECTORY: Path = Path(__file__).parent.resolve().absolute() -from pyearthtools.data.static._geographic.retrieval import get # noqa: E402 F401 from pyearthtools.data.static._geographic import retrieval +from pyearthtools.data.static._geographic.retrieval import get # noqa: E402 F401 diff --git a/packages/data/src/pyearthtools/data/static/_geographic/retrieval.py b/packages/data/src/pyearthtools/data/static/_geographic/retrieval.py index 8a1db1e2..8bc378c7 100644 --- a/packages/data/src/pyearthtools/data/static/_geographic/retrieval.py +++ b/packages/data/src/pyearthtools/data/static/_geographic/retrieval.py @@ -27,7 +27,6 @@ from typing import Literal import yaml - from pyearthtools.data.exceptions import DataNotFoundError, InvalidIndexError from pyearthtools.data.static._geographic import DATA_BASEDIRECTORY, DOWNLOAD_DATA diff --git a/packages/data/src/pyearthtools/data/time.py b/packages/data/src/pyearthtools/data/time.py index 5659002b..13615237 100644 --- a/packages/data/src/pyearthtools/data/time.py +++ b/packages/data/src/pyearthtools/data/time.py @@ -26,14 +26,12 @@ from __future__ import annotations import datetime - import functools -from typing import Any, Generator, Literal, overload, Union +from typing import Any, Generator, Literal, Union, overload import numpy as np import pandas as pd import yaml - from pyearthtools.utils import initialisation VALID_RESOLUTIONS = Literal["year", "month", "day", "hour", "minute", "min", "second", "nanosecond"] diff --git a/packages/data/src/pyearthtools/data/transforms/__init__.py b/packages/data/src/pyearthtools/data/transforms/__init__.py index 1dd2424f..45910916 100644 --- a/packages/data/src/pyearthtools/data/transforms/__init__.py +++ b/packages/data/src/pyearthtools/data/transforms/__init__.py @@ -48,27 +48,26 @@ def apply(self, dataset): """ -from pyearthtools.data.transforms.transform import ( - Transform, - TransformCollection, - FunctionTransform, -) - from pyearthtools.data.transforms import ( aggregation, + attributes, coordinates, dimensions, + interpolation, + mask, normalisation, - utils, - variables, - attributes, optimisation, - values, - interpolation, region, - mask, + utils, + values, + variables, ) from pyearthtools.data.transforms.default import get_default_transforms # from pyearthtools.data.transforms.mask import MaskTransform as mask -from pyearthtools.data.transforms.derive import derive, Derive +from pyearthtools.data.transforms.derive import Derive, derive +from pyearthtools.data.transforms.transform import ( + FunctionTransform, + Transform, + TransformCollection, +) diff --git a/packages/data/src/pyearthtools/data/transforms/aggregation.py b/packages/data/src/pyearthtools/data/transforms/aggregation.py index 2ce2bcc2..05e3833a 100644 --- a/packages/data/src/pyearthtools/data/transforms/aggregation.py +++ b/packages/data/src/pyearthtools/data/transforms/aggregation.py @@ -18,8 +18,8 @@ from typing import Callable, Optional, Union import xarray as xr - -from pyearthtools.data.transforms import Transform, aggregation +from pyearthtools.data.transforms import aggregation +from pyearthtools.data.transforms.transform import Transform from pyearthtools.utils.initialisation.imports import dynamic_import known_methods = ["mean", "max", "min", "sum", "std"] diff --git a/packages/data/src/pyearthtools/data/transforms/attributes.py b/packages/data/src/pyearthtools/data/transforms/attributes.py index 0ee76227..cc9329ce 100644 --- a/packages/data/src/pyearthtools/data/transforms/attributes.py +++ b/packages/data/src/pyearthtools/data/transforms/attributes.py @@ -18,11 +18,11 @@ """ from __future__ import annotations + from typing import Any, Literal import xarray as xr - -from pyearthtools.data.transforms import Transform +from pyearthtools.data.transforms.transform import Transform from pyearthtools.utils.decorators import BackwardsCompatibility diff --git a/packages/data/src/pyearthtools/data/transforms/coordinates.py b/packages/data/src/pyearthtools/data/transforms/coordinates.py index a08541d6..9031c8b4 100644 --- a/packages/data/src/pyearthtools/data/transforms/coordinates.py +++ b/packages/data/src/pyearthtools/data/transforms/coordinates.py @@ -15,20 +15,16 @@ from __future__ import annotations -import warnings -from typing import Any, Hashable, Literal, Iterable import logging +import warnings +from typing import Any, Hashable, Iterable, Literal -import xarray as xr import numpy as np - - import pyearthtools.data - +import xarray as xr +from pyearthtools.data.exceptions import DataNotFoundError from pyearthtools.data.transforms.transform import Transform, TransformCollection from pyearthtools.data.warnings import pyearthtoolsDataWarning -from pyearthtools.data.exceptions import DataNotFoundError - from pyearthtools.utils.decorators import BackwardsCompatibility DASK_IMPORTED = False diff --git a/packages/data/src/pyearthtools/data/transforms/derive.py b/packages/data/src/pyearthtools/data/transforms/derive.py index e23a150c..655bb34e 100644 --- a/packages/data/src/pyearthtools/data/transforms/derive.py +++ b/packages/data/src/pyearthtools/data/transforms/derive.py @@ -18,17 +18,16 @@ """ from __future__ import annotations -from typing import Any -import xarray as xr -import numpy as np -import operator +import logging import math +import operator import re -import logging - +from typing import Any -from pyearthtools.data.transforms import Transform +import numpy as np +import xarray as xr +from pyearthtools.data.transforms.transform import Transform from pyearthtools.utils.decorators import BackwardsCompatibility LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/src/pyearthtools/data/transforms/dimensions.py b/packages/data/src/pyearthtools/data/transforms/dimensions.py index b6a88a3c..f84661fd 100644 --- a/packages/data/src/pyearthtools/data/transforms/dimensions.py +++ b/packages/data/src/pyearthtools/data/transforms/dimensions.py @@ -14,10 +14,10 @@ from __future__ import annotations -from typing import Any, Optional, Literal, TypeVar -import xarray as xr +from typing import Any, Literal, Optional, TypeVar +import xarray as xr from pyearthtools.data.transforms.transform import Transform from pyearthtools.utils.decorators import BackwardsCompatibility diff --git a/packages/data/src/pyearthtools/data/transforms/interpolation.py b/packages/data/src/pyearthtools/data/transforms/interpolation.py index 8622970b..457f47b0 100644 --- a/packages/data/src/pyearthtools/data/transforms/interpolation.py +++ b/packages/data/src/pyearthtools/data/transforms/interpolation.py @@ -14,18 +14,16 @@ from __future__ import annotations + from typing import Optional -import xarray as xr -from xarray.core.types import InterpOptions import numpy as np - - import pyearthtools.data +import xarray as xr from pyearthtools.data.transforms.transform import Transform from pyearthtools.data.transforms.utils import parse_dataset - from pyearthtools.utils.decorators import BackwardsCompatibility +from xarray.core.types import InterpOptions xESMF_IMPORTED = True try: diff --git a/packages/data/src/pyearthtools/data/transforms/mask.py b/packages/data/src/pyearthtools/data/transforms/mask.py index b1b847a7..afad791e 100644 --- a/packages/data/src/pyearthtools/data/transforms/mask.py +++ b/packages/data/src/pyearthtools/data/transforms/mask.py @@ -14,18 +14,15 @@ from __future__ import annotations -import operator - -from typing import Any, Literal, overload +import operator from pathlib import Path +from typing import Any, Literal, overload import numpy as np import xarray as xr - from pyearthtools.data.transforms.transform import Transform from pyearthtools.data.transforms.utils import parse_dataset - from pyearthtools.utils.decorators import BackwardsCompatibility OPERATIONS = ["==", "!=", ">", "<", ">=", "<="] diff --git a/packages/data/src/pyearthtools/data/transforms/normalisation/default.py b/packages/data/src/pyearthtools/data/transforms/normalisation/default.py index b4df1784..70112439 100644 --- a/packages/data/src/pyearthtools/data/transforms/normalisation/default.py +++ b/packages/data/src/pyearthtools/data/transforms/normalisation/default.py @@ -15,26 +15,23 @@ from __future__ import annotations +import logging +import tempfile +import warnings from abc import abstractmethod from pathlib import Path from typing import Callable, Hashable -import warnings -import tempfile -import logging - -import xarray as xr import pyearthtools.data - -from pyearthtools.data import Petdt +import xarray as xr +from pyearthtools.data.indexes.utilities.fileload import open_files +from pyearthtools.data.time import Petdt from pyearthtools.data.transforms.normalisation._utils import format_class_name from pyearthtools.data.transforms.transform import ( FunctionTransform, Transform, get_default_transforms, ) -from pyearthtools.data.indexes.utilities.fileload import open_files - from pyearthtools.utils.initialisation.imports import dynamic_import LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/src/pyearthtools/data/transforms/normalisation/unnormalise.py b/packages/data/src/pyearthtools/data/transforms/normalisation/unnormalise.py index 38d4b262..7ea2df9a 100644 --- a/packages/data/src/pyearthtools/data/transforms/normalisation/unnormalise.py +++ b/packages/data/src/pyearthtools/data/transforms/normalisation/unnormalise.py @@ -20,7 +20,6 @@ import numpy as np import xarray as xr - from pyearthtools.data.transforms.normalisation.default import Normaliser, open_file from pyearthtools.data.transforms.transform import FunctionTransform, Transform diff --git a/packages/data/src/pyearthtools/data/transforms/optimisation.py b/packages/data/src/pyearthtools/data/transforms/optimisation.py index ed7601dc..d89c1664 100644 --- a/packages/data/src/pyearthtools/data/transforms/optimisation.py +++ b/packages/data/src/pyearthtools/data/transforms/optimisation.py @@ -14,12 +14,11 @@ from __future__ import annotations -from typing import Literal, Any +from typing import Any, Literal import xarray as xr from pyearthtools.data.transforms.transform import Transform - from pyearthtools.utils.decorators import BackwardsCompatibility diff --git a/packages/data/src/pyearthtools/data/transforms/region.py b/packages/data/src/pyearthtools/data/transforms/region.py index c5588831..e4c7aa69 100644 --- a/packages/data/src/pyearthtools/data/transforms/region.py +++ b/packages/data/src/pyearthtools/data/transforms/region.py @@ -30,12 +30,10 @@ except ImportError: GEOPANDAS_IMPORTED = False -from pyearthtools.data.transforms import Transform +from pyearthtools.data.transforms.transform import Transform from pyearthtools.data.transforms.utils import parse_dataset - from pyearthtools.utils.decorators import BackwardsCompatibility - RegionLookupFILE = Path(__file__).parent / "RegionLookup.yaml" diff --git a/packages/data/src/pyearthtools/data/transforms/transform.py b/packages/data/src/pyearthtools/data/transforms/transform.py index 7d29f879..60ad5d73 100644 --- a/packages/data/src/pyearthtools/data/transforms/transform.py +++ b/packages/data/src/pyearthtools/data/transforms/transform.py @@ -15,20 +15,17 @@ from __future__ import annotations +import warnings from abc import ABCMeta, abstractmethod from types import FunctionType -from typing import Any, Callable, Union, TypeVar +from typing import Any, Callable, TypeVar, Union -import warnings +import pyearthtools.data.transforms +import pyearthtools.utils +import xarray as xr import yaml from pyearthtools.data.collection import Collection - -import xarray as xr - -import pyearthtools.data.transforms from pyearthtools.data.transforms.default import get_default_transforms - -import pyearthtools.utils from pyearthtools.utils import initialisation XR_TYPES = TypeVar("XR_TYPES", xr.DataArray, xr.Dataset, Union[xr.DataArray, xr.Dataset]) diff --git a/packages/data/src/pyearthtools/data/transforms/utils.py b/packages/data/src/pyearthtools/data/transforms/utils.py index a44267f1..14f92a42 100644 --- a/packages/data/src/pyearthtools/data/transforms/utils.py +++ b/packages/data/src/pyearthtools/data/transforms/utils.py @@ -19,10 +19,10 @@ from __future__ import annotations -import xarray as xr -from typing import Any from pathlib import Path +from typing import Any +import xarray as xr # DEFAULT_TRANSFORM_LOCATIONS = [ # "__main__.", diff --git a/packages/data/src/pyearthtools/data/transforms/values.py b/packages/data/src/pyearthtools/data/transforms/values.py index 4383f9b9..680df90d 100644 --- a/packages/data/src/pyearthtools/data/transforms/values.py +++ b/packages/data/src/pyearthtools/data/transforms/values.py @@ -18,11 +18,11 @@ """ from __future__ import annotations -from typing import Literal -import xarray as xr +from typing import Literal import pyearthtools.data +import xarray as xr from pyearthtools.data.transforms.transform import Transform from pyearthtools.utils.decorators import BackwardsCompatibility diff --git a/packages/data/src/pyearthtools/data/transforms/variables.py b/packages/data/src/pyearthtools/data/transforms/variables.py index e08c0b37..c09e47c7 100644 --- a/packages/data/src/pyearthtools/data/transforms/variables.py +++ b/packages/data/src/pyearthtools/data/transforms/variables.py @@ -15,11 +15,9 @@ from __future__ import annotations -import xarray as xr - import pyearthtools.data.transforms.attributes as attr +import xarray as xr from pyearthtools.data.transforms.transform import Transform - from pyearthtools.utils.decorators import BackwardsCompatibility # Backwards compatability diff --git a/packages/data/src/pyearthtools/data/utils.py b/packages/data/src/pyearthtools/data/utils.py index 0aff232b..a88b0c0c 100644 --- a/packages/data/src/pyearthtools/data/utils.py +++ b/packages/data/src/pyearthtools/data/utils.py @@ -15,13 +15,11 @@ from __future__ import annotations -from typing import Any - -import re +import logging import os - +import re from pathlib import Path -import logging +from typing import Any LOG = logging.getLogger("pyearthtools.data") diff --git a/packages/data/tests/data/archive/test_extensions.py b/packages/data/tests/data/archive/test_extensions.py index e2ba9ccb..70a1e61f 100644 --- a/packages/data/tests/data/archive/test_extensions.py +++ b/packages/data/tests/data/archive/test_extensions.py @@ -1,7 +1,6 @@ +import pyearthtools.data import pytest - from pyearthtools.data.archive import extensions -import pyearthtools.data def test_register_archive(): diff --git a/packages/data/tests/data/archive/test_zarr.py b/packages/data/tests/data/archive/test_zarr.py index 1c56b6ce..dfeb5a04 100644 --- a/packages/data/tests/data/archive/test_zarr.py +++ b/packages/data/tests/data/archive/test_zarr.py @@ -13,10 +13,9 @@ # limitations under the License. -import xarray as xr - import pyearthtools.data import pytest +import xarray as xr @pytest.mark.xfail diff --git a/packages/data/tests/data/indexes/test_cacheIndex.py b/packages/data/tests/data/indexes/test_cacheIndex.py index caa057bd..ad00c32c 100644 --- a/packages/data/tests/data/indexes/test_cacheIndex.py +++ b/packages/data/tests/data/indexes/test_cacheIndex.py @@ -1,5 +1,4 @@ import xarray as xr - from pyearthtools.data.indexes import cacheIndex diff --git a/packages/data/tests/data/indexes/test_decorators.py b/packages/data/tests/data/indexes/test_decorators.py index d80e518e..8ed78078 100644 --- a/packages/data/tests/data/indexes/test_decorators.py +++ b/packages/data/tests/data/indexes/test_decorators.py @@ -14,7 +14,6 @@ import pytest - from pyearthtools.data.indexes import decorators diff --git a/packages/data/tests/data/indexes/test_fake.py b/packages/data/tests/data/indexes/test_fake.py new file mode 100644 index 00000000..e69de29b diff --git a/packages/data/tests/data/indexes/test_indexes.py b/packages/data/tests/data/indexes/test_indexes.py index b2f9e8ca..94fc6381 100644 --- a/packages/data/tests/data/indexes/test_indexes.py +++ b/packages/data/tests/data/indexes/test_indexes.py @@ -1,8 +1,9 @@ -from pyearthtools.data import indexes +import pathlib + import pyearthtools.data.archive -from pyearthtools.data.time import Petdt import pytest -import pathlib +from pyearthtools.data import indexes +from pyearthtools.data.time import Petdt def test_Index(monkeypatch): diff --git a/packages/data/tests/data/indexes/test_modifications.py b/packages/data/tests/data/indexes/test_modifications.py index 3130191c..42bc914e 100644 --- a/packages/data/tests/data/indexes/test_modifications.py +++ b/packages/data/tests/data/indexes/test_modifications.py @@ -14,7 +14,6 @@ import pytest - from pyearthtools.data.indexes import FakeIndex diff --git a/packages/data/tests/data/indexes/utilities/mixins/test_index_repr.py b/packages/data/tests/data/indexes/utilities/mixins/test_index_repr.py index 86e0cd57..147c2538 100644 --- a/packages/data/tests/data/indexes/utilities/mixins/test_index_repr.py +++ b/packages/data/tests/data/indexes/utilities/mixins/test_index_repr.py @@ -14,9 +14,8 @@ from collections import namedtuple -from pyearthtools.data.indexes.utilities.mixins import index_repr - from pyearthtools.data.catalog import Catalog +from pyearthtools.data.indexes.utilities.mixins import index_repr class MixableTestClass(index_repr.reprMixin): diff --git a/packages/data/tests/data/operations/test_interpolation.py b/packages/data/tests/data/operations/test_interpolation.py index c24fd293..521d1291 100644 --- a/packages/data/tests/data/operations/test_interpolation.py +++ b/packages/data/tests/data/operations/test_interpolation.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pyearthtools.data.operations import interpolation import xarray as xr +from pyearthtools.data.operations import interpolation # Provides a basic forecast data structure in three dimensions SIMPLE_DA1 = xr.DataArray( diff --git a/packages/data/tests/data/test_catalog.py b/packages/data/tests/data/test_catalog.py index 5acb6eb6..3f766c20 100644 --- a/packages/data/tests/data/test_catalog.py +++ b/packages/data/tests/data/test_catalog.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pyearthtools -from pyearthtools.data import catalog +import io from collections import namedtuple + +import pyearthtools import pytest -import io +from pyearthtools.data import catalog def test_get_name(): diff --git a/packages/data/tests/data/time/test_time.py b/packages/data/tests/data/time/test_time.py index 0d5cb5a6..7a602259 100644 --- a/packages/data/tests/data/time/test_time.py +++ b/packages/data/tests/data/time/test_time.py @@ -14,7 +14,6 @@ import pytest - from pyearthtools.data.time import Petdt, TimeDelta, TimeRange, TimeResolution diff --git a/packages/data/tests/data/transform/normalisation/test_default.py b/packages/data/tests/data/transform/normalisation/test_default.py index 410c29da..0ec4d42b 100644 --- a/packages/data/tests/data/transform/normalisation/test_default.py +++ b/packages/data/tests/data/transform/normalisation/test_default.py @@ -1,10 +1,10 @@ -import pyearthtools.data.transforms.normalisation -from pyearthtools.data.transforms.normalisation import default -from pyearthtools.data.time import Petdt -import pyearthtools.data.indexes -import xarray as xr import numpy as np +import pyearthtools.data.indexes +import pyearthtools.data.transforms.normalisation import pytest +import xarray as xr +from pyearthtools.data.time import Petdt +from pyearthtools.data.transforms.normalisation import default sample_da = xr.DataArray( coords={"latitude": [1, 2, 3, 4], "longitude": [1, 2, 3], "time": ["2023-02"]}, data=np.ones((4, 3, 1)) diff --git a/packages/data/tests/data/transform/test_aggregation.py b/packages/data/tests/data/transform/test_aggregation.py index ee3fb99e..b1f7a462 100644 --- a/packages/data/tests/data/transform/test_aggregation.py +++ b/packages/data/tests/data/transform/test_aggregation.py @@ -13,10 +13,8 @@ # limitations under the License. import xarray as xr - from pyearthtools.data.transforms import aggregation - SIMPLE_DA1 = xr.DataArray( [ [ diff --git a/packages/data/tests/data/transform/test_coordinates.py b/packages/data/tests/data/transform/test_coordinates.py index 310bc690..520363b0 100644 --- a/packages/data/tests/data/transform/test_coordinates.py +++ b/packages/data/tests/data/transform/test_coordinates.py @@ -1,8 +1,7 @@ -from pyearthtools.data.transforms import coordinates -import xarray as xr import numpy as np import pytest - +import xarray as xr +from pyearthtools.data.transforms import coordinates # Test data, re-used across tests lon180 = list(range(-180, 180)) diff --git a/packages/data/tests/data/transform/test_derive.py b/packages/data/tests/data/transform/test_derive.py index b03cec5c..2ed1d853 100644 --- a/packages/data/tests/data/transform/test_derive.py +++ b/packages/data/tests/data/transform/test_derive.py @@ -13,9 +13,9 @@ # limitations under the License. -import pytest import math +import pytest from pyearthtools.data.transforms.derive import EquationException, evaluate diff --git a/packages/data/tests/data/transform/test_mask.py b/packages/data/tests/data/transform/test_mask.py index 6f7eac15..89bda51a 100644 --- a/packages/data/tests/data/transform/test_mask.py +++ b/packages/data/tests/data/transform/test_mask.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pyearthtools.data.transforms import mask import numpy as np import xarray as xr +from pyearthtools.data.transforms import mask SIMPLE_DA1 = xr.DataArray( [ diff --git a/packages/pipeline/src/pyearthtools/pipeline/__init__.py b/packages/pipeline/src/pyearthtools/pipeline/__init__.py index c0c301ae..ecc6f59a 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/__init__.py +++ b/packages/pipeline/src/pyearthtools/pipeline/__init__.py @@ -44,42 +44,36 @@ __version__ = "0.1.0" import pyearthtools.pipeline.logger - -from pyearthtools.pipeline.save import save, load -from pyearthtools.pipeline.controller import Pipeline, PipelineIndex - -from pyearthtools.pipeline.operation import Operation - from pyearthtools.pipeline import ( branching, + config, exceptions, filters, iterators, - samplers, - operations, modifications, + operations, + samplers, ) - -from pyearthtools.pipeline.marker import Marker, Empty - -from pyearthtools.pipeline.modifications import Cache, SequenceRetrieval, TemporalRetrieval - -from pyearthtools.pipeline.samplers import Sampler - -from pyearthtools.pipeline.iterators import Iterator - -from pyearthtools.pipeline.parallel import get_parallel - +from pyearthtools.pipeline.controller import Pipeline, PipelineIndex from pyearthtools.pipeline.exceptions import ( PipelineException, PipelineFilterException, PipelineRuntimeError, PipelineTypeError, ) +from pyearthtools.pipeline.iterators import Iterator +from pyearthtools.pipeline.marker import Empty, Marker +from pyearthtools.pipeline.modifications import ( + Cache, + SequenceRetrieval, + TemporalRetrieval, +) +from pyearthtools.pipeline.operation import Operation +from pyearthtools.pipeline.parallel import get_parallel +from pyearthtools.pipeline.samplers import Sampler +from pyearthtools.pipeline.save import load, save from pyearthtools.pipeline.warnings import PipelineWarning -from pyearthtools.pipeline import config - __all__ = [ "Sampler", "Iterator", diff --git a/packages/pipeline/src/pyearthtools/pipeline/branching/__init__.py b/packages/pipeline/src/pyearthtools/pipeline/branching/__init__.py index 04c56156..bf01ffbe 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/branching/__init__.py +++ b/packages/pipeline/src/pyearthtools/pipeline/branching/__init__.py @@ -13,12 +13,11 @@ # limitations under the License. +from pyearthtools.pipeline.branching import join, split, unify from pyearthtools.pipeline.branching.branching import PipelineBranchPoint -from pyearthtools.pipeline.branching.unify import Unifier from pyearthtools.pipeline.branching.join import Joiner from pyearthtools.pipeline.branching.split import Spliter from pyearthtools.pipeline.branching.stop import StopUndo - -from pyearthtools.pipeline.branching import unify, join, split +from pyearthtools.pipeline.branching.unify import Unifier __all__ = ["PipelineBranchPoint", "Unifier", "Joiner", "Spliter"] diff --git a/packages/pipeline/src/pyearthtools/pipeline/branching/branching.py b/packages/pipeline/src/pyearthtools/pipeline/branching/branching.py index ea6a1af3..bbcdfc6f 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/branching/branching.py +++ b/packages/pipeline/src/pyearthtools/pipeline/branching/branching.py @@ -15,22 +15,19 @@ from __future__ import annotations -from typing import Any, Literal, Optional, Union import warnings +from typing import Any, Literal, Optional, Union import graphviz - from pyearthtools.data.indexes import Index from pyearthtools.data.transforms import Transform, TransformCollection - -from pyearthtools.pipeline.step import PipelineStep -from pyearthtools.pipeline.controller import PipelineIndex, Pipeline, _Pipeline -from pyearthtools.pipeline.operation import Operation - from pyearthtools.pipeline import parallel -from pyearthtools.pipeline.warnings import PipelineWarning -from pyearthtools.pipeline.validation import filter_steps +from pyearthtools.pipeline.controller import Pipeline, PipelineIndex, _Pipeline from pyearthtools.pipeline.exceptions import PipelineRuntimeError +from pyearthtools.pipeline.operation import Operation +from pyearthtools.pipeline.step import PipelineStep +from pyearthtools.pipeline.validation import filter_steps +from pyearthtools.pipeline.warnings import PipelineWarning def get_key_from_steps(key: str, steps: tuple[Any, ...]): diff --git a/packages/pipeline/src/pyearthtools/pipeline/branching/join.py b/packages/pipeline/src/pyearthtools/pipeline/branching/join.py index 0348a650..3a83ceeb 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/branching/join.py +++ b/packages/pipeline/src/pyearthtools/pipeline/branching/join.py @@ -14,9 +14,9 @@ from __future__ import annotations -from abc import abstractmethod -from typing import Any, Literal, Type, Optional, Union +from abc import abstractmethod +from typing import Any, Literal, Optional, Type, Union from pyearthtools.pipeline.operation import Operation diff --git a/packages/pipeline/src/pyearthtools/pipeline/branching/split.py b/packages/pipeline/src/pyearthtools/pipeline/branching/split.py index 7dc1e6d0..468f7052 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/branching/split.py +++ b/packages/pipeline/src/pyearthtools/pipeline/branching/split.py @@ -14,9 +14,9 @@ from __future__ import annotations -from abc import abstractmethod -from typing import Any, Literal, Type, TypeVar, Optional, Union +from abc import abstractmethod +from typing import Any, Literal, Optional, Type, TypeVar, Union from pyearthtools.pipeline.operation import Operation diff --git a/packages/pipeline/src/pyearthtools/pipeline/branching/unify.py b/packages/pipeline/src/pyearthtools/pipeline/branching/unify.py index 63e08b6c..c3ad9750 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/branching/unify.py +++ b/packages/pipeline/src/pyearthtools/pipeline/branching/unify.py @@ -14,13 +14,12 @@ from __future__ import annotations + from abc import ABCMeta, abstractmethod from typing import Any, Union - -from pyearthtools.pipeline.operation import Operation from pyearthtools.pipeline.exceptions import PipelineUnificationException - +from pyearthtools.pipeline.operation import Operation __all__ = ["Unifier", "Equality"] diff --git a/packages/pipeline/src/pyearthtools/pipeline/controller.py b/packages/pipeline/src/pyearthtools/pipeline/controller.py index 568f3b9c..7cd412be 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/controller.py +++ b/packages/pipeline/src/pyearthtools/pipeline/controller.py @@ -18,30 +18,28 @@ from __future__ import annotations -from abc import ABCMeta, abstractmethod - -from typing import Any, ContextManager, Literal, Union, Optional, Type, overload -from pathlib import Path +import builtins import functools import logging +from abc import ABCMeta, abstractmethod +from pathlib import Path +from typing import Any, ContextManager, Literal, Optional, Type, Union, overload -import builtins import graphviz - +import pyearthtools.pipeline import pyearthtools.utils - from pyearthtools.data.indexes import Index from pyearthtools.data.transforms import Transform, TransformCollection - -import pyearthtools.pipeline +from pyearthtools.pipeline import filters, iterators, samplers +from pyearthtools.pipeline.exceptions import ( + ExceptionIgnoreContext, + PipelineFilterException, +) +from pyearthtools.pipeline.graph import Graphed, format_graph_node +from pyearthtools.pipeline.operation import Operation from pyearthtools.pipeline.recording import PipelineRecordingMixin -from pyearthtools.pipeline import samplers, iterators, filters from pyearthtools.pipeline.step import PipelineStep -from pyearthtools.pipeline.operation import Operation -from pyearthtools.pipeline.exceptions import PipelineFilterException, ExceptionIgnoreContext from pyearthtools.pipeline.validation import filter_steps -from pyearthtools.pipeline.graph import Graphed, format_graph_node - PIPELINE_TYPES = Union[Index, PipelineStep, Transform, TransformCollection] VALID_PIPELINE_TYPES = Union[PIPELINE_TYPES, tuple[PIPELINE_TYPES, ...], tuple[tuple, ...]] @@ -773,7 +771,7 @@ def save(self, path: Optional[Union[str, Path]] = None, only_steps: bool = False def _ipython_display_(self): """Override for repr of `Pipeline`, shows initialisation arguments and graph""" - from IPython.core.display import display, HTML + from IPython.core.display import HTML, display display(HTML(self._repr_html_())) diff --git a/packages/pipeline/src/pyearthtools/pipeline/exceptions.py b/packages/pipeline/src/pyearthtools/pipeline/exceptions.py index c4e96034..9bbddd34 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/exceptions.py +++ b/packages/pipeline/src/pyearthtools/pipeline/exceptions.py @@ -13,8 +13,8 @@ # limitations under the License. -from typing import Any, Optional, Type import warnings +from typing import Any, Optional, Type import pyearthtools.utils from pyearthtools.pipeline.warnings import PipelineWarning diff --git a/packages/pipeline/src/pyearthtools/pipeline/filters.py b/packages/pipeline/src/pyearthtools/pipeline/filters.py index d1f3574a..7c9b2fa4 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/filters.py +++ b/packages/pipeline/src/pyearthtools/pipeline/filters.py @@ -13,15 +13,13 @@ # limitations under the License. -from abc import abstractmethod, ABCMeta -from typing import Optional, Type, Union - import warnings +from abc import ABCMeta, abstractmethod +from typing import Optional, Type, Union import pyearthtools.utils - -from pyearthtools.pipeline.step import PipelineStep from pyearthtools.pipeline.exceptions import PipelineFilterException +from pyearthtools.pipeline.step import PipelineStep from pyearthtools.pipeline.warnings import PipelineWarning __all__ = ["Filter", "FilterCheck", "FilterWarningContext", "TypeFilter"] diff --git a/packages/pipeline/src/pyearthtools/pipeline/graph.py b/packages/pipeline/src/pyearthtools/pipeline/graph.py index bc7377aa..9e0009e9 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/graph.py +++ b/packages/pipeline/src/pyearthtools/pipeline/graph.py @@ -17,9 +17,8 @@ from typing import Any, Optional import graphviz - -from pyearthtools.data import Index import pyearthtools.pipeline +from pyearthtools.data import Index def format_graph_node(obj, parent: Optional[list[str]]) -> dict[str, Any]: diff --git a/packages/pipeline/src/pyearthtools/pipeline/iterators.py b/packages/pipeline/src/pyearthtools/pipeline/iterators.py index 607c8de9..bf4d8b5a 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/iterators.py +++ b/packages/pipeline/src/pyearthtools/pipeline/iterators.py @@ -20,16 +20,14 @@ """ from __future__ import annotations -from functools import cached_property -from abc import ABCMeta, abstractmethod -from typing import Any, Callable, Generator, Hashable, Iterable, Optional, Union +from abc import ABCMeta, abstractmethod +from functools import cached_property from pathlib import Path +from typing import Any, Callable, Generator, Hashable, Iterable, Optional, Union import numpy as np - -import pyearthtools.data - +import pyearthtools.data.time from pyearthtools.pipeline.recording import PipelineRecordingMixin @@ -178,22 +176,22 @@ def __init__(self, start: str, end: str, interval): Args: start (str): Start time. Must be understandable by - `pyearthtools.data.Petdt`. + `pyearthtools.data.time.Petdt`. end (str): End time. Must be understandable by - `pyearthtools.data.Petdt`. + `pyearthtools.data.time.Petdt`. interval (Any): Interval between times. Must be understandable by - `pyearthtools.data.TimeDelta`. + `pyearthtools.data.time.TimeDelta`. """ super().__init__() self.record_initialisation() import pyearthtools.data - self._timerange = pyearthtools.data.TimeRange(start, end, interval) + self._timerange = pyearthtools.data.time.TimeRange(start, end, interval) - def __iter__(self) -> Generator[pyearthtools.data.Petdt, None, None]: + def __iter__(self) -> Generator[pyearthtools.data.time.Petdt, None, None]: for i in self._timerange: yield i @@ -214,12 +212,12 @@ def __init__(self, start: str, interval: Any, num: int): Start time interval (Any): Interval between times. Must be understandable by - `pyearthtools.data.TimeDelta`. + `pyearthtools.data.time.TimeDelta`. num (int): Number of total samples to iterate over. """ - end = pyearthtools.data.Petdt(start) + (pyearthtools.data.TimeDelta(interval) * num) + end = pyearthtools.data.time.Petdt(start) + (pyearthtools.data.time.TimeDelta(interval) * num) super().__init__(start, str(end), interval) diff --git a/packages/pipeline/src/pyearthtools/pipeline/marker.py b/packages/pipeline/src/pyearthtools/pipeline/marker.py index da312584..5e7ae9d5 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/marker.py +++ b/packages/pipeline/src/pyearthtools/pipeline/marker.py @@ -14,7 +14,6 @@ import xarray as xr - from pyearthtools.pipeline.graph import Graphed from pyearthtools.pipeline.operation import PipelineStep diff --git a/packages/pipeline/src/pyearthtools/pipeline/modifications/__init__.py b/packages/pipeline/src/pyearthtools/pipeline/modifications/__init__.py index 22ef8896..e45a8e66 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/modifications/__init__.py +++ b/packages/pipeline/src/pyearthtools/pipeline/modifications/__init__.py @@ -13,17 +13,15 @@ # limitations under the License. +from pyearthtools.pipeline.modifications import idx_modification +from pyearthtools.pipeline.modifications.cache import Cache, MemCache, StaticCache from pyearthtools.pipeline.modifications.idx_modification import ( IdxModifier, IdxOverride, - TimeIdxModifier, SequenceRetrieval, TemporalRetrieval, + TimeIdxModifier, ) -from pyearthtools.pipeline.modifications.cache import Cache, StaticCache, MemCache - - -from pyearthtools.pipeline.modifications import idx_modification __all__ = [ "Cache", diff --git a/packages/pipeline/src/pyearthtools/pipeline/modifications/cache.py b/packages/pipeline/src/pyearthtools/pipeline/modifications/cache.py index 3dc0c53b..8b229aa0 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/modifications/cache.py +++ b/packages/pipeline/src/pyearthtools/pipeline/modifications/cache.py @@ -15,19 +15,17 @@ from __future__ import annotations -from typing import Any, Optional, Union, Literal +import shutil import warnings - -from pathlib import Path from hashlib import sha512 -import shutil +from pathlib import Path +from typing import Any, Literal, Optional, Union import pyearthtools.data from pyearthtools.data.patterns import PatternIndex - from pyearthtools.pipeline.controller import PipelineIndex -from pyearthtools.pipeline.warnings import PipelineWarning from pyearthtools.pipeline.exceptions import PipelineRuntimeError +from pyearthtools.pipeline.warnings import PipelineWarning CACHE_HASH_NAME = ".cache_hash" PIPELINE_SAVE_NAME = "pipeline.yaml" diff --git a/packages/pipeline/src/pyearthtools/pipeline/modifications/idx_modification.py b/packages/pipeline/src/pyearthtools/pipeline/modifications/idx_modification.py index 401b11ab..e6b664ba 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/modifications/idx_modification.py +++ b/packages/pipeline/src/pyearthtools/pipeline/modifications/idx_modification.py @@ -17,24 +17,21 @@ from __future__ import annotations +import warnings from dataclasses import dataclass from typing import Any, Callable, Iterable, Optional, Type, Union -import warnings -import xarray as xr import numpy as np - import pyearthtools.data - - +import xarray as xr from pyearthtools.pipeline.controller import PipelineIndex from pyearthtools.pipeline.parallel import ParallelEnabledMixin from pyearthtools.pipeline.warnings import PipelineWarning DASK_IMPORTED = True try: - from dask.delayed import Delayed, delayed import dask.array as da + from dask.delayed import Delayed, delayed except (ImportError, ModuleNotFoundError) as _: DASK_IMPORTED = False @@ -219,7 +216,7 @@ def __getitem__(self, idx: Any): class TimeIdxModifier(IdxModifier): - """`IdxModifier` which converts all `modification`'s to `pyearthtools.data.TimeDelta`""" + """`IdxModifier` which converts all `modification`'s to `pyearthtools.data.time.TimeDelta`""" def __init__( self, @@ -228,7 +225,7 @@ def __init__( **kwargs, ): """ - Modify `idx` but convert all `modification`'s to `pyearthtools.data.TimeDelta` + Modify `idx` but convert all `modification`'s to `pyearthtools.data.time.TimeDelta` Args: modification (Union[Any, tuple[Union[Any, tuple[Any, ...]], ...]]): @@ -252,7 +249,7 @@ def map_to_time(mod): """Map elements to `TimeDelta`""" if can_map(mod): return tuple(map(map_to_time, mod)) - return pyearthtools.data.TimeDelta(mod) + return pyearthtools.data.time.TimeDelta(mod) if extra_mods: modification = ( @@ -429,7 +426,7 @@ def __getitem__(self, idx: Any): class TemporalRetrieval(SequenceRetrieval): """ Retrieve a sequence of samples from `SequenceRetrieval`, - but force all indexes to be an `pyearthtools.data.Petdt`. + but force all indexes to be an `pyearthtools.data.time.Petdt`. Examples: >>> TemporalRetrieval(-6)['2000-01-01T12'] @@ -450,15 +447,15 @@ def __init__( def map_to_tuple(mod): if isinstance(mod, tuple): return tuple(map(map_to_tuple, mod)) - return pyearthtools.data.TimeDelta((mod, delta_unit)) + return pyearthtools.data.time.TimeDelta((mod, delta_unit)) if delta_unit is not None: self._modification = map_to_tuple(self._modification) def __getitem__(self, idx: Any): - if not isinstance(idx, pyearthtools.data.Petdt): - if not pyearthtools.data.Petdt.is_time(idx): - raise TypeError(f"Cannot convert {idx!r} to `pyearthtools.data.Petdt`.") - idx = pyearthtools.data.Petdt(idx) + if not isinstance(idx, pyearthtools.data.time.Petdt): + if not pyearthtools.data.time.Petdt.is_time(idx): + raise TypeError(f"Cannot convert {idx!r} to `pyearthtools.data.time.Petdt`.") + idx = pyearthtools.data.time.Petdt(idx) return super().__getitem__(idx) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operation.py b/packages/pipeline/src/pyearthtools/pipeline/operation.py index 908ad3c8..c036370c 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operation.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operation.py @@ -18,11 +18,9 @@ from typing import Literal, Optional, Type, Union import numpy as np - -from pyearthtools.pipeline.step import PipelineStep - -from pyearthtools.pipeline.decorators import potentialabstractmethod, PotentialABC from pyearthtools.pipeline import parallel +from pyearthtools.pipeline.decorators import PotentialABC, potentialabstractmethod +from pyearthtools.pipeline.step import PipelineStep __all__ = ["Operation"] diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/__init__.py b/packages/pipeline/src/pyearthtools/pipeline/operations/__init__.py index 63d1e3d2..da122fef 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/__init__.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/__init__.py @@ -26,9 +26,8 @@ import warnings -from pyearthtools.pipeline.operations import xarray, numpy +from pyearthtools.pipeline.operations import numpy, transform, xarray from pyearthtools.pipeline.operations.transforms import Transforms -from pyearthtools.pipeline.operations import transform try: from pyearthtools.pipeline.operations import dask diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/__init__.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/__init__.py index 7ac0c908..d94a9c80 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/__init__.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/__init__.py @@ -31,19 +31,22 @@ """ -from pyearthtools.pipeline.operations.dask.join import Stack, Concatenate, VStack, HStack - -from pyearthtools.pipeline.operations.dask.compute import Compute - from pyearthtools.pipeline.operations.dask import ( augment, + conversion, filters, normalisation, reshape, select, split, values, - conversion, +) +from pyearthtools.pipeline.operations.dask.compute import Compute +from pyearthtools.pipeline.operations.dask.join import ( + Concatenate, + HStack, + Stack, + VStack, ) __all__ = [ diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/augment.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/augment.py index 64ca1fa7..5fdac440 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/augment.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/augment.py @@ -20,7 +20,6 @@ import dask.array as da import numpy as np - from pyearthtools.pipeline.operations.dask.dask import DaskOperation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/compute.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/compute.py index 14fa8482..4d698757 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/compute.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/compute.py @@ -18,9 +18,7 @@ from typing import TypeVar import dask.array as da - from dask.delayed import Delayed - from pyearthtools.pipeline.operations.dask.dask import DaskOperation T = TypeVar("T", da.Array, Delayed) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/conversion.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/conversion.py index d44a6fa8..a0f497fb 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/conversion.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/conversion.py @@ -16,13 +16,11 @@ # type: ignore[reportPrivateImportUsage] -from typing import Any, Optional, Union, Hashable +from typing import Any, Hashable, Optional, Union import dask.array as da -import xarray as xr - import pyearthtools.data - +import xarray as xr from pyearthtools.pipeline.operations.dask.dask import DaskOperation ATTRIBUTES_IGNORE = ["license", "summary"] diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/dask.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/dask.py index 31924649..d85d00f0 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/dask.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/dask.py @@ -19,13 +19,11 @@ Dask specific operation """ -from typing import Type, Union, Optional - import functools -import numpy as np +from typing import Optional, Type, Union +import numpy as np import pyearthtools.utils - from pyearthtools.pipeline.operation import Operation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/filters.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/filters.py index 562977b6..b7e1adc3 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/filters.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/filters.py @@ -20,7 +20,6 @@ import dask.array as da import numpy as np - from pyearthtools.pipeline.filters import Filter, PipelineFilterException diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/join.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/join.py index 0045efde..ec20a49c 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/join.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/join.py @@ -15,10 +15,9 @@ # type: ignore[reportPrivateImportUsage] -from typing import Optional, Any +from typing import Any, Optional import dask.array as da - from pyearthtools.pipeline.branching.join import Joiner from pyearthtools.pipeline.operations.dask.dask import DaskOperation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/normalisation.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/normalisation.py index 63e44d85..f81cc079 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/normalisation.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/normalisation.py @@ -19,16 +19,11 @@ from pathlib import Path from typing import Union - -import numpy as np import dask.array as da - - +import numpy as np from pyearthtools.data.utils import parse_path - -from pyearthtools.utils.decorators import BackwardsCompatibility from pyearthtools.pipeline.operations.dask.dask import DaskOperation - +from pyearthtools.utils.decorators import BackwardsCompatibility FILE = Union[str, Path] diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/reshape.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/reshape.py index 58165f1e..6df2caa9 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/reshape.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/reshape.py @@ -15,15 +15,13 @@ # type: ignore[reportPrivateImportUsage] -from typing import Union, Optional, Any - import math -import einops +from typing import Any, Optional, Union -import numpy as np import dask.array as da +import einops +import numpy as np from dask.delayed import delayed - from pyearthtools.pipeline.operations.dask.dask import DaskOperation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/select.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/select.py index d9fdddfc..f3a21a49 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/select.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/select.py @@ -18,7 +18,6 @@ from typing import Optional import dask.array as da - from pyearthtools.pipeline.operations.dask.dask import DaskOperation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/split.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/split.py index 4069c10c..4664d9e3 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/split.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/split.py @@ -18,7 +18,6 @@ from typing import Optional import dask.array as da - from pyearthtools.pipeline.branching.split import Spliter from pyearthtools.pipeline.operations.dask.dask import DaskOperation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/values.py b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/values.py index 3197ad8a..152535bb 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/dask/values.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/dask/values.py @@ -15,13 +15,11 @@ # type: ignore[reportPrivateImportUsage] -from typing import Literal, Union, Optional +from typing import Literal, Optional, Union import dask.array as da import numpy as np - import pyearthtools.data - from pyearthtools.pipeline.operations.dask.dask import DaskOperation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/__init__.py b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/__init__.py index 0f06e0cb..104e06b0 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/__init__.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/__init__.py @@ -27,17 +27,21 @@ | split | Split numpy arrays into tuples | `OnAxis`, `OnSlice`, `VSplit`, `HSplit` | | values | Modify values of arrays | `FillNan`, `MaskValue`, `ForceNormalised` | """ -from pyearthtools.pipeline.operations.numpy.join import Stack, Concatenate, VStack, HStack - from pyearthtools.pipeline.operations.numpy import ( augment, + conversion, filters, normalisation, reshape, select, split, values, - conversion, +) +from pyearthtools.pipeline.operations.numpy.join import ( + Concatenate, + HStack, + Stack, + VStack, ) __all__ = [ diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/augment.py b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/augment.py index 0c120f04..c56e4d2b 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/augment.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/augment.py @@ -14,7 +14,6 @@ import numpy as np - from pyearthtools.pipeline.operation import Operation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/conversion.py b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/conversion.py index b4962074..da15f3c5 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/conversion.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/conversion.py @@ -13,13 +13,11 @@ # limitations under the License. -from typing import Any, Optional, Union, Hashable +from typing import Any, Hashable, Optional, Union import numpy as np -import xarray as xr - import pyearthtools.data - +import xarray as xr from pyearthtools.pipeline.operation import Operation ATTRIBUTES_IGNORE = ["license", "summary"] diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/join.py b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/join.py index 72e519a8..ae9327ce 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/join.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/join.py @@ -13,10 +13,9 @@ # limitations under the License. -from typing import Optional, Any +from typing import Any, Optional import numpy as np - from pyearthtools.pipeline.branching.join import Joiner diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/normalisation.py b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/normalisation.py index 7cb13a7d..98b46eb8 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/normalisation.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/normalisation.py @@ -18,12 +18,9 @@ from typing import Union import numpy as np - from pyearthtools.data.utils import parse_path - -from pyearthtools.utils.decorators import BackwardsCompatibility from pyearthtools.pipeline.operation import Operation - +from pyearthtools.utils.decorators import BackwardsCompatibility FILE = Union[str, Path] diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/reshape.py b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/reshape.py index 794b388e..5cd74101 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/reshape.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/reshape.py @@ -13,12 +13,11 @@ # limitations under the License. -from typing import Union, Optional, Any - import math +from typing import Any, Optional, Union + import einops import numpy as np - from pyearthtools.pipeline.operation import Operation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/select.py b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/select.py index 3303dbfb..fc6bdf80 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/select.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/select.py @@ -16,7 +16,6 @@ from typing import Any, Optional import numpy as np - from pyearthtools.pipeline.operation import Operation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/split.py b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/split.py index 275bd92e..d1f42530 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/split.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/split.py @@ -16,8 +16,6 @@ from typing import Optional import numpy as np - - from pyearthtools.pipeline.branching.split import Spliter diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/values.py b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/values.py index 3356017e..3925f0e4 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/values.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/numpy/values.py @@ -13,12 +13,10 @@ # limitations under the License. -from typing import Literal, Union, Optional +from typing import Literal, Optional, Union import numpy as np - import pyearthtools.data - from pyearthtools.pipeline.operation import Operation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/transform/__init__.py b/packages/pipeline/src/pyearthtools/pipeline/operations/transform/__init__.py index 0649d288..ddb0d2c8 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/transform/__init__.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/transform/__init__.py @@ -15,5 +15,5 @@ """Pipeline specific transforms""" -from pyearthtools.pipeline.operations.transform.add_variables import TimeOfYear from pyearthtools.pipeline.operations.transform.add_coordinates import AddCoordinates +from pyearthtools.pipeline.operations.transform.add_variables import TimeOfYear diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/transform/add_coordinates.py b/packages/pipeline/src/pyearthtools/pipeline/operations/transform/add_coordinates.py index 704dce1d..ce1f0b95 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/transform/add_coordinates.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/transform/add_coordinates.py @@ -17,9 +17,8 @@ from typing import Any, Union import numpy as np -import xarray as xr - import pyearthtools.data +import xarray as xr from pyearthtools.data import Transform LOG = logging.getLogger("pyearthtools.pipeline") diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/transforms.py b/packages/pipeline/src/pyearthtools/pipeline/operations/transforms.py index 2180df45..615f1202 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/transforms.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/transforms.py @@ -13,11 +13,10 @@ # limitations under the License. -from typing import Any, TypeVar, Union, Optional - -import xarray as xr +from typing import Any, Optional, TypeVar, Union import pyearthtools.data +import xarray as xr from pyearthtools.pipeline.operation import Operation T = TypeVar("T", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/__init__.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/__init__.py index 6ba4a0f4..05c07378 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/__init__.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/__init__.py @@ -33,22 +33,21 @@ | remapping | Reproject data | `HEALPix` | """ -from pyearthtools.pipeline.operations.xarray.compute import Compute -from pyearthtools.pipeline.operations.xarray.join import Merge, Concatenate -from pyearthtools.pipeline.operations.xarray.sort import Sort -from pyearthtools.pipeline.operations.xarray.chunk import Chunk - from pyearthtools.pipeline.operations.xarray import ( conversion, filters, + metadata, + normalisation, + remapping, reshape, select, split, values, - metadata, - normalisation, - remapping, ) +from pyearthtools.pipeline.operations.xarray.chunk import Chunk +from pyearthtools.pipeline.operations.xarray.compute import Compute +from pyearthtools.pipeline.operations.xarray.join import Concatenate, Merge +from pyearthtools.pipeline.operations.xarray.sort import Sort __all__ = [ "Compute", diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/chunk.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/chunk.py index cee85f8d..ef8ca6fd 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/chunk.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/chunk.py @@ -13,10 +13,9 @@ # limitations under the License. -from typing import TypeVar, Optional, Literal +from typing import Literal, Optional, TypeVar import xarray as xr - from pyearthtools.pipeline.operation import Operation T = TypeVar("T", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/compute.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/compute.py index d55b2c77..13eb24a0 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/compute.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/compute.py @@ -16,7 +16,6 @@ from typing import TypeVar import xarray as xr - from pyearthtools.pipeline.operation import Operation T = TypeVar("T", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/conversion.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/conversion.py index 936489f6..e6784eab 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/conversion.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/conversion.py @@ -16,15 +16,13 @@ # type: ignore[reportPrivateImportUsage] -from typing import Optional, Union, TypeVar from pathlib import Path +from typing import Optional, TypeVar, Union import numpy as np import xarray as xr - -from pyearthtools.utils.data import converter - from pyearthtools.pipeline.operation import Operation +from pyearthtools.utils.data import converter XARRAY_OBJECTS = TypeVar("XARRAY_OBJECTS", xr.Dataset, xr.DataArray) FILE_TYPES = Union[str, Path] diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/filters.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/filters.py index d0c5b807..d195356e 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/filters.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/filters.py @@ -13,13 +13,11 @@ # limitations under the License. +import math from typing import Literal, Optional, TypeVar, Union import numpy as np import xarray as xr - -import math - from pyearthtools.pipeline.filters import Filter, PipelineFilterException T = TypeVar("T", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/join.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/join.py index eec4eb0f..b7092cf7 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/join.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/join.py @@ -14,10 +14,9 @@ from functools import reduce -from typing import TypeVar, Union, Optional, Any +from typing import Any, Optional, TypeVar, Union import xarray as xr - from pyearthtools.pipeline.branching.join import Joiner T = TypeVar("T", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/metadata.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/metadata.py index 5642492e..98392e83 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/metadata.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/metadata.py @@ -13,12 +13,10 @@ # limitations under the License. -from typing import TypeVar, Optional, Any, Literal - -import xarray as xr +from typing import Any, Literal, Optional, TypeVar import pyearthtools.data - +import xarray as xr from pyearthtools.pipeline.operation import Operation T = TypeVar("T", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/normalisation.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/normalisation.py index 21ab35cf..49eb8e17 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/normalisation.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/normalisation.py @@ -18,12 +18,9 @@ from typing import TypeVar, Union import xarray as xr - from pyearthtools.data.utils import parse_path - -from pyearthtools.utils.decorators import BackwardsCompatibility from pyearthtools.pipeline.operation import Operation - +from pyearthtools.utils.decorators import BackwardsCompatibility FILE = Union[str, Path] diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/remapping/base.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/remapping/base.py index 3398fe25..f3315e6a 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/remapping/base.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/remapping/base.py @@ -17,10 +17,10 @@ Remapping Operations. """ -from abc import abstractmethod, ABCMeta +from abc import ABCMeta, abstractmethod from typing import Type, TypeVar -import xarray as xr +import xarray as xr from pyearthtools.pipeline import Operation XR_TYPE = TypeVar("XR_TYPE", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/remapping/healpix.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/remapping/healpix.py index a9e88328..7a9d3230 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/remapping/healpix.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/remapping/healpix.py @@ -43,25 +43,21 @@ """ -from typing import TypeVar, Literal, Optional - import functools +import logging import warnings +from typing import Literal, Optional, TypeVar -import numpy as np -import healpy as hp -import xarray as xr -import reproject as rp import astropy as ap - -import logging - +import healpy as hp +import numpy as np import pyearthtools.data +import reproject as rp +import xarray as xr from pyearthtools.pipeline.warnings import PipelineWarning from .base import BaseRemap - XR_TYPE = TypeVar("XR_TYPE", xr.Dataset, xr.DataArray) HEALPIX_COORDS = ["face", "height", "width"] diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/reshape.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/reshape.py index 3fc8e572..182a1cf5 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/reshape.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/reshape.py @@ -15,9 +15,8 @@ from typing import Hashable, TypeVar, Union -import xarray as xr - import pyearthtools.data +import xarray as xr from pyearthtools.pipeline.operation import Operation T = TypeVar("T", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/select.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/select.py index eb99386e..93be766c 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/select.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/select.py @@ -13,12 +13,10 @@ # limitations under the License. -from typing import Any, Literal, Union, Optional - -import xarray as xr - +from typing import Any, Literal, Optional, Union import pyearthtools.data +import xarray as xr from pyearthtools.pipeline.operation import Operation diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/sort.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/sort.py index c6d8536c..a32e8cda 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/sort.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/sort.py @@ -13,10 +13,9 @@ # limitations under the License. -from typing import TypeVar, Optional +from typing import Optional, TypeVar import xarray as xr - from pyearthtools.pipeline.operation import Operation T = TypeVar("T", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/split.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/split.py index 5a240139..a6f288b9 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/split.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/split.py @@ -13,10 +13,9 @@ # limitations under the License. -from typing import TypeVar, Union, Optional, Any +from typing import Any, Optional, TypeVar, Union import xarray as xr - from pyearthtools.pipeline.branching.split import Spliter T = TypeVar("T", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/values.py b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/values.py index b2ac63a6..6698d994 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/values.py +++ b/packages/pipeline/src/pyearthtools/pipeline/operations/xarray/values.py @@ -13,13 +13,11 @@ # limitations under the License. -from typing import TypeVar, Union, Optional, Any, Literal +from typing import Any, Literal, Optional, TypeVar, Union -import xarray as xr import numpy as np - import pyearthtools.data - +import xarray as xr from pyearthtools.pipeline.operation import Operation T = TypeVar("T", xr.Dataset, xr.DataArray) diff --git a/packages/pipeline/src/pyearthtools/pipeline/parallel.py b/packages/pipeline/src/pyearthtools/pipeline/parallel.py index d9652635..cfd8d340 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/parallel.py +++ b/packages/pipeline/src/pyearthtools/pipeline/parallel.py @@ -23,17 +23,14 @@ no code is needed to be changed to run in serial. """ -from abc import abstractmethod import functools - -from importlib.util import find_spec - import logging -from typing import Callable, Literal, Optional, Type, TypeVar, Any, Union - -from pyearthtools.utils.decorators import classproperty +from abc import abstractmethod +from importlib.util import find_spec +from typing import Any, Callable, Literal, Optional, Type, TypeVar, Union import pyearthtools.utils +from pyearthtools.utils.decorators import classproperty Future = TypeVar("Future", Any, Any) @@ -159,9 +156,9 @@ def config(self): @classproperty def client(cls) -> "distributed.Client": # type: ignore # noqa: F821 """Get dask client""" - from dask.distributed import Client - import distributed import dask + import distributed + from dask.distributed import Client try: client = distributed.get_client() @@ -252,7 +249,7 @@ def config(self): return self._interface_kwargs.get("Delayed", {}) def run_delayed(self, func, *args, **kwargs): - from dask.delayed import tokenize, delayed + from dask.delayed import delayed, tokenize name = self.config.get("name", None) if name is not None: diff --git a/packages/pipeline/src/pyearthtools/pipeline/samplers.py b/packages/pipeline/src/pyearthtools/pipeline/samplers.py index 659de333..fec50fef 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/samplers.py +++ b/packages/pipeline/src/pyearthtools/pipeline/samplers.py @@ -20,8 +20,8 @@ """ from __future__ import annotations -from abc import ABCMeta, abstractmethod +from abc import ABCMeta, abstractmethod from typing import Any, Generator, Union from pyearthtools.pipeline.recording import PipelineRecordingMixin diff --git a/packages/pipeline/src/pyearthtools/pipeline/save.py b/packages/pipeline/src/pyearthtools/pipeline/save.py index f28ae75a..2dd0b305 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/save.py +++ b/packages/pipeline/src/pyearthtools/pipeline/save.py @@ -15,22 +15,17 @@ """Saving and Loading of `Pipelines`""" +import logging import os -from typing import Any, Union, Optional - -from pathlib import Path import warnings +from pathlib import Path +from typing import Any, Optional, Union +import pyearthtools.pipeline import yaml - -import logging - from pyearthtools.data.utils import parse_path - -from pyearthtools.utils.initialisation.imports import dynamic_import from pyearthtools.utils import initialisation - -import pyearthtools.pipeline +from pyearthtools.utils.initialisation.imports import dynamic_import CONFIG_KEY = "--CONFIG--" SUFFIX = ".epi" diff --git a/packages/pipeline/src/pyearthtools/pipeline/step.py b/packages/pipeline/src/pyearthtools/pipeline/step.py index 64c0f5a8..ccd7e63d 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/step.py +++ b/packages/pipeline/src/pyearthtools/pipeline/step.py @@ -13,17 +13,15 @@ # limitations under the License. +import warnings from abc import ABCMeta, abstractmethod - from functools import partial -from typing import Callable, Union, Optional, Literal, Type -import warnings - +from typing import Callable, Literal, Optional, Type, Union +from pyearthtools.pipeline.exceptions import PipelineFilterException, PipelineTypeError +from pyearthtools.pipeline.parallel import ParallelEnabledMixin from pyearthtools.pipeline.recording import PipelineRecordingMixin -from pyearthtools.pipeline.exceptions import PipelineTypeError, PipelineFilterException from pyearthtools.pipeline.warnings import PipelineWarning -from pyearthtools.pipeline.parallel import ParallelEnabledMixin class PipelineStep(PipelineRecordingMixin, ParallelEnabledMixin, metaclass=ABCMeta): diff --git a/packages/pipeline/src/pyearthtools/pipeline/validation.py b/packages/pipeline/src/pyearthtools/pipeline/validation.py index bf662884..99a6cf5d 100644 --- a/packages/pipeline/src/pyearthtools/pipeline/validation.py +++ b/packages/pipeline/src/pyearthtools/pipeline/validation.py @@ -15,7 +15,7 @@ from __future__ import annotations -from typing import Type, Iterable, Optional, Union, Any +from typing import Any, Iterable, Optional, Type, Union from pyearthtools.pipeline.exceptions import PipelineTypeError diff --git a/packages/pipeline/tests/fake_pipeline_steps.py b/packages/pipeline/tests/fake_pipeline_steps.py index 604885cb..f693de44 100644 --- a/packages/pipeline/tests/fake_pipeline_steps.py +++ b/packages/pipeline/tests/fake_pipeline_steps.py @@ -15,10 +15,8 @@ from __future__ import annotations import pyearthtools.utils - -from pyearthtools.pipeline import Operation from pyearthtools.data import Index - +from pyearthtools.pipeline import Operation pyearthtools.utils.config.set({"pipeline.run_parallel": False}) diff --git a/packages/pipeline/tests/operations/xarray/test_sort.py b/packages/pipeline/tests/operations/xarray/test_sort.py index dd890d9e..99bef02d 100644 --- a/packages/pipeline/tests/operations/xarray/test_sort.py +++ b/packages/pipeline/tests/operations/xarray/test_sort.py @@ -14,7 +14,6 @@ import pytest import xarray as xr - from pyearthtools.pipeline.operations.xarray import sort SIMPLE_DA1 = xr.DataArray( diff --git a/packages/pipeline/tests/pipeline/test_controller.py b/packages/pipeline/tests/pipeline/test_controller.py index 9e1e46d9..bcfeb878 100644 --- a/packages/pipeline/tests/pipeline/test_controller.py +++ b/packages/pipeline/tests/pipeline/test_controller.py @@ -1,5 +1,4 @@ import pytest - from pyearthtools.pipeline import controller # def test_PipelineIndex(): diff --git a/packages/pipeline/tests/pipeline/test_idx_mod.py b/packages/pipeline/tests/pipeline/test_idx_mod.py index ebbfc0d1..3d992ac2 100644 --- a/packages/pipeline/tests/pipeline/test_idx_mod.py +++ b/packages/pipeline/tests/pipeline/test_idx_mod.py @@ -15,13 +15,10 @@ from __future__ import annotations -import pytest - import pyearthtools.utils - -from pyearthtools.pipeline import Pipeline, modifications -from pyearthtools.pipeline import Operation +import pytest from pyearthtools.data import Index +from pyearthtools.pipeline import Operation, Pipeline, modifications class FakeIndex(Index): @@ -168,20 +165,20 @@ def test_TimeIdxModifier_basic(): import pyearthtools.data pipe = Pipeline(FakeIndex(), modifications.TimeIdxModifier("6 hours")) - assert pipe[pyearthtools.data.Petdt("2000-01-01T00")] == pyearthtools.data.Petdt("2000-01-01T06") + assert pipe[pyearthtools.data.time.Petdt("2000-01-01T00")] == pyearthtools.data.time.Petdt("2000-01-01T06") # def test_TimeIdxModifier_basic_tuple(): # import pyearthtools.data # pipe = Pipeline(FakeIndex(), pipelines.TimeIdxModifier((6, 'hours'))) -# assert pipe[pyearthtools.data.Petdt('2000-01-01T00')] == pyearthtools.data.Petdt('2000-01-01T06') +# assert pipe[pyearthtools.data.time.Petdt('2000-01-01T00')] == pyearthtools.data.time.Petdt('2000-01-01T06') def test_TimeIdxModifier_nested(): import pyearthtools.data pipe = Pipeline(FakeIndex(), modifications.TimeIdxModifier(("6 hours", "12 hours"))) - assert pipe[pyearthtools.data.Petdt("2000-01-01T00")] == ( - pyearthtools.data.Petdt("2000-01-01T06"), - pyearthtools.data.Petdt("2000-01-01T12"), + assert pipe[pyearthtools.data.time.Petdt("2000-01-01T00")] == ( + pyearthtools.data.time.Petdt("2000-01-01T06"), + pyearthtools.data.time.Petdt("2000-01-01T12"), ) diff --git a/packages/pipeline/tests/pipeline/test_sequence.py b/packages/pipeline/tests/pipeline/test_sequence.py index 325f1e29..89573742 100644 --- a/packages/pipeline/tests/pipeline/test_sequence.py +++ b/packages/pipeline/tests/pipeline/test_sequence.py @@ -14,22 +14,18 @@ from __future__ import annotations -from typing import Any -import pytest +from typing import Any import pyearthtools.utils +import pytest pyearthtools.utils.config.set({"pipeline.run_parallel": False}) import pyearthtools.data - -from pyearthtools.pipeline import Pipeline, exceptions, modifications -from pyearthtools.pipeline.modifications.idx_modification import SequenceRetrieval - -from pyearthtools.pipeline import Operation from pyearthtools.data import Index - +from pyearthtools.pipeline import Operation, Pipeline, exceptions, modifications +from pyearthtools.pipeline.modifications.idx_modification import SequenceRetrieval pyearthtools.utils.config.set({"pipeline.run_parallel": False}) diff --git a/packages/pipeline/tests/pipeline/test_temporal_idx_mod.py b/packages/pipeline/tests/pipeline/test_temporal_idx_mod.py index 09969b04..264c71cc 100644 --- a/packages/pipeline/tests/pipeline/test_temporal_idx_mod.py +++ b/packages/pipeline/tests/pipeline/test_temporal_idx_mod.py @@ -14,22 +14,18 @@ from __future__ import annotations -from typing import Any -import pytest +from typing import Any import pyearthtools.utils +import pytest pyearthtools.utils.config.set({"pipeline.run_parallel": False}) import pyearthtools.data - -from pyearthtools.pipeline import Pipeline, exceptions -from pyearthtools.pipeline.modifications.idx_modification import TemporalRetrieval - -from pyearthtools.pipeline import Operation from pyearthtools.data import Index - +from pyearthtools.pipeline import Operation, Pipeline, exceptions +from pyearthtools.pipeline.modifications.idx_modification import TemporalRetrieval pyearthtools.utils.config.set({"pipeline.run_parallel": False}) diff --git a/packages/pipeline/tests/test_controller/test_branchpoints.py b/packages/pipeline/tests/test_controller/test_branchpoints.py index 8898fc61..16e0bf3e 100644 --- a/packages/pipeline/tests/test_controller/test_branchpoints.py +++ b/packages/pipeline/tests/test_controller/test_branchpoints.py @@ -14,13 +14,14 @@ from __future__ import annotations -import pytest - import pyearthtools.utils - -from pyearthtools.pipeline import Pipeline, exceptions, branching - -from tests.fake_pipeline_steps import FakeIndex, MultiplicationOperation, MultiplicationOperationUnunifiedable +import pytest +from pyearthtools.pipeline import Pipeline, branching, exceptions +from tests.fake_pipeline_steps import ( + FakeIndex, + MultiplicationOperation, + MultiplicationOperationUnunifiedable, +) pyearthtools.utils.config.set({"pipeline.run_parallel": False}) diff --git a/packages/pipeline/tests/test_controller/test_joining.py b/packages/pipeline/tests/test_controller/test_joining.py index 4836ef4c..a1b85be4 100644 --- a/packages/pipeline/tests/test_controller/test_joining.py +++ b/packages/pipeline/tests/test_controller/test_joining.py @@ -13,14 +13,12 @@ # limitations under the License. from __future__ import annotations -from typing import Any -import pytest +from typing import Any import pyearthtools.utils - +import pytest from pyearthtools.pipeline import Pipeline, branching - from tests.fake_pipeline_steps import FakeIndex, MultiplicationOperation pyearthtools.utils.config.set({"pipeline.run_parallel": False}) diff --git a/packages/pipeline/tests/test_controller/test_split.py b/packages/pipeline/tests/test_controller/test_split.py index 99eb06cb..be943972 100644 --- a/packages/pipeline/tests/test_controller/test_split.py +++ b/packages/pipeline/tests/test_controller/test_split.py @@ -15,12 +15,9 @@ from __future__ import annotations -import pytest - import pyearthtools.utils - +import pytest from pyearthtools.pipeline import Pipeline, branching - from tests.fake_pipeline_steps import FakeIndex pyearthtools.utils.config.set({"pipeline.run_parallel": False}) diff --git a/packages/pipeline/tests/test_controller/test_unify.py b/packages/pipeline/tests/test_controller/test_unify.py index 3e0b1fde..aace5b66 100644 --- a/packages/pipeline/tests/test_controller/test_unify.py +++ b/packages/pipeline/tests/test_controller/test_unify.py @@ -14,12 +14,9 @@ from __future__ import annotations -import pytest - import pyearthtools.utils - +import pytest from pyearthtools.pipeline import Pipeline, branching, exceptions - from tests.fake_pipeline_steps import FakeIndex pyearthtools.utils.config.set({"pipeline.run_parallel": False}) diff --git a/packages/pipeline/tests/test_filters.py b/packages/pipeline/tests/test_filters.py index 710a485b..94bdbeb9 100644 --- a/packages/pipeline/tests/test_filters.py +++ b/packages/pipeline/tests/test_filters.py @@ -13,12 +13,10 @@ # limitations under the License. +import pyearthtools.utils import pytest - -from pyearthtools.pipeline import Pipeline, iterators, filters, exceptions, Operation from pyearthtools.data import Index - -import pyearthtools.utils +from pyearthtools.pipeline import Operation, Pipeline, exceptions, filters, iterators pyearthtools.utils.config.set({"pipeline.run_parallel": False}) diff --git a/packages/pipeline/tests/test_iteration.py b/packages/pipeline/tests/test_iteration.py index 1eb2e427..f9e23d7e 100644 --- a/packages/pipeline/tests/test_iteration.py +++ b/packages/pipeline/tests/test_iteration.py @@ -14,7 +14,6 @@ import pytest - from pyearthtools.pipeline import Pipeline, iterators, samplers from tests.fake_pipeline_steps import FakeIndex diff --git a/packages/pipeline/tests/test_operation/test_recognised_types.py b/packages/pipeline/tests/test_operation/test_recognised_types.py index 98e72492..e2e324c5 100644 --- a/packages/pipeline/tests/test_operation/test_recognised_types.py +++ b/packages/pipeline/tests/test_operation/test_recognised_types.py @@ -14,9 +14,8 @@ import pytest -from pyearthtools.pipeline import Operation, Pipeline - from pyearthtools.data import Index +from pyearthtools.pipeline import Operation, Pipeline class FakeIndex(Index): diff --git a/packages/training/src/pyearthtools/training/__init__.py b/packages/training/src/pyearthtools/training/__init__.py index cf5ecad6..77031d8d 100644 --- a/packages/training/src/pyearthtools/training/__init__.py +++ b/packages/training/src/pyearthtools/training/__init__.py @@ -21,12 +21,11 @@ """ # ruff: noqa: F401 +from pyearthtools.training import data from pyearthtools.training import logger as _ - -from pyearthtools.training import data, wrapper, manage - -from pyearthtools.training.wrapper import * # type: ignore # noqa: F403 +from pyearthtools.training import manage, wrapper from pyearthtools.training.dataindex import MLDataIndex +from pyearthtools.training.wrapper import * # type: ignore # noqa: F403 __version__ = "0.1.0" diff --git a/packages/training/src/pyearthtools/training/commands.py b/packages/training/src/pyearthtools/training/commands.py index 2f1f7e4e..16ae95cd 100644 --- a/packages/training/src/pyearthtools/training/commands.py +++ b/packages/training/src/pyearthtools/training/commands.py @@ -18,9 +18,10 @@ """ from __future__ import annotations -import click import sys +import click + @click.group(name="Trainer From Yaml") def entry_point(): diff --git a/packages/training/src/pyearthtools/training/data/__init__.py b/packages/training/src/pyearthtools/training/data/__init__.py index 9fad52cc..290822ea 100644 --- a/packages/training/src/pyearthtools/training/data/__init__.py +++ b/packages/training/src/pyearthtools/training/data/__init__.py @@ -15,13 +15,11 @@ # ruff: noqa: F401 -from pyearthtools.training.data.datamodule import PipelineDataModule +import logging from pyearthtools.training.data import default - -from pyearthtools.training.data.fileio import save, load - -import logging +from pyearthtools.training.data.datamodule import PipelineDataModule +from pyearthtools.training.data.fileio import load, save LOG = logging.getLogger(__name__) try: diff --git a/packages/training/src/pyearthtools/training/data/datamodule.py b/packages/training/src/pyearthtools/training/data/datamodule.py index a8b0bb7c..e1f2652c 100644 --- a/packages/training/src/pyearthtools/training/data/datamodule.py +++ b/packages/training/src/pyearthtools/training/data/datamodule.py @@ -20,16 +20,14 @@ from __future__ import annotations import functools -from typing import Optional, Callable, Any, TypeVar - -import numpy as np from pathlib import Path +from typing import Any, Callable, Optional, TypeVar +import numpy as np import pyearthtools.pipeline -from pyearthtools.pipeline import Pipeline, Iterator +from pyearthtools.pipeline import Iterator, Pipeline from pyearthtools.utils.initialisation import InitialisationRecordingMixin - CONFIG_KEY = "--CONFIG--" SUFFIX = ".datamodule" diff --git a/packages/training/src/pyearthtools/training/data/default/__init__.py b/packages/training/src/pyearthtools/training/data/default/__init__.py index 6ca30634..0ea7c1dd 100644 --- a/packages/training/src/pyearthtools/training/data/default/__init__.py +++ b/packages/training/src/pyearthtools/training/data/default/__init__.py @@ -20,5 +20,11 @@ # ruff: noqa: F401 -from pyearthtools.training.data.default.datamodule import PipelineDefaultDataModule, DataLoader -from pyearthtools.training.data.default.datasets import IndexableDataset, IterableDataset +from pyearthtools.training.data.default.datamodule import ( + DataLoader, + PipelineDefaultDataModule, +) +from pyearthtools.training.data.default.datasets import ( + IndexableDataset, + IterableDataset, +) diff --git a/packages/training/src/pyearthtools/training/data/default/datamodule.py b/packages/training/src/pyearthtools/training/data/default/datamodule.py index 8999281b..ab07e88b 100644 --- a/packages/training/src/pyearthtools/training/data/default/datamodule.py +++ b/packages/training/src/pyearthtools/training/data/default/datamodule.py @@ -14,17 +14,20 @@ from __future__ import annotations + import functools -from typing import Callable, Any +from typing import Any, Callable import numpy as np import xarray as xr - from pyearthtools.pipeline.controller import Pipeline from pyearthtools.pipeline.iterators import Iterator - from pyearthtools.training.data.datamodule import PipelineDataModule -from pyearthtools.training.data.default.datasets import IndexableDataset, IterableDataset, BaseDefault +from pyearthtools.training.data.default.datasets import ( + BaseDefault, + IndexableDataset, + IterableDataset, +) def map_function(obj, function: Callable[[Any], Any], **kwargs): diff --git a/packages/training/src/pyearthtools/training/data/fileio.py b/packages/training/src/pyearthtools/training/data/fileio.py index a36136b8..8dbde9c3 100644 --- a/packages/training/src/pyearthtools/training/data/fileio.py +++ b/packages/training/src/pyearthtools/training/data/fileio.py @@ -15,15 +15,14 @@ from __future__ import annotations -from typing import Optional, Any import os - from pathlib import Path -import yaml +from typing import Any, Optional import pyearthtools.pipeline -from pyearthtools.utils.initialisation import Dumper, Loader, update_contents +import yaml from pyearthtools.training.data.datamodule import PipelineDataModule +from pyearthtools.utils.initialisation import Dumper, Loader, update_contents CONFIG_KEY = "--CONFIG--" SUFFIX = ".edm" diff --git a/packages/training/src/pyearthtools/training/data/lightning/__init__.py b/packages/training/src/pyearthtools/training/data/lightning/__init__.py index 0a66d44d..88c5b985 100644 --- a/packages/training/src/pyearthtools/training/data/lightning/__init__.py +++ b/packages/training/src/pyearthtools/training/data/lightning/__init__.py @@ -16,4 +16,7 @@ # ruff: noqa: F401 from pyearthtools.training.data.lightning.datamodule import PipelineLightningDataModule -from pyearthtools.training.data.lightning.datasets import PytorchDataset, PytorchIterable +from pyearthtools.training.data.lightning.datasets import ( + PytorchDataset, + PytorchIterable, +) diff --git a/packages/training/src/pyearthtools/training/data/lightning/datamodule.py b/packages/training/src/pyearthtools/training/data/lightning/datamodule.py index 4a6fa966..3aa6bbbd 100644 --- a/packages/training/src/pyearthtools/training/data/lightning/datamodule.py +++ b/packages/training/src/pyearthtools/training/data/lightning/datamodule.py @@ -15,16 +15,17 @@ from __future__ import annotations - import lightning as L from lightning.pytorch.utilities import CombinedLoader -from torch.utils.data import DataLoader - from pyearthtools.pipeline.controller import Pipeline from pyearthtools.pipeline.iterators import Iterator - from pyearthtools.training.data.datamodule import PipelineDataModule -from pyearthtools.training.data.lightning.datasets import PytorchDataset, PytorchIterable, BasePytorchPipeline +from pyearthtools.training.data.lightning.datasets import ( + BasePytorchPipeline, + PytorchDataset, + PytorchIterable, +) +from torch.utils.data import DataLoader class PipelineLightningDataModule(PipelineDataModule, L.LightningDataModule): diff --git a/packages/training/src/pyearthtools/training/data/lightning/datasets.py b/packages/training/src/pyearthtools/training/data/lightning/datasets.py index d0ee7998..247a0b7e 100644 --- a/packages/training/src/pyearthtools/training/data/lightning/datasets.py +++ b/packages/training/src/pyearthtools/training/data/lightning/datasets.py @@ -15,10 +15,9 @@ from __future__ import annotations -from torch.utils.data import IterableDataset, get_worker_info, Dataset - import pyearthtools from pyearthtools.pipeline import Pipeline +from torch.utils.data import Dataset, IterableDataset, get_worker_info class BasePytorchPipeline: diff --git a/packages/training/src/pyearthtools/training/dataindex.py b/packages/training/src/pyearthtools/training/dataindex.py index cf793550..65f2c080 100644 --- a/packages/training/src/pyearthtools/training/dataindex.py +++ b/packages/training/src/pyearthtools/training/dataindex.py @@ -21,18 +21,16 @@ from __future__ import annotations import os - from pathlib import Path -import yaml from typing import Any import pyearthtools.data -from pyearthtools.data import Petdt, Transform, TransformCollection, TimeDelta +import pyearthtools.training.wrapper +import yaml +from pyearthtools.data import Petdt, TimeDelta, Transform, TransformCollection from pyearthtools.data.indexes import TimeIndex from pyearthtools.data.indexes.cacheIndex import BaseCacheIndex -import pyearthtools.training.wrapper - ATTRIBUTE_MARK = pyearthtools.data.transforms.attributes.set_attributes( purpose="Research Use Only.", contact="For further information or support, contact the Data Science and Emerging Technologies Team.", diff --git a/packages/training/src/pyearthtools/training/manage.py b/packages/training/src/pyearthtools/training/manage.py index 6f821786..fe4048f4 100644 --- a/packages/training/src/pyearthtools/training/manage.py +++ b/packages/training/src/pyearthtools/training/manage.py @@ -4,11 +4,12 @@ """ from __future__ import annotations -from typing import Any, overload, Union, Optional, Callable, TypeVar -import xarray as xr -import numpy as np import itertools +from typing import Any, Callable, Optional, TypeVar, Union, overload + +import numpy as np +import xarray as xr TORCH_INSTALLED = True try: diff --git a/packages/training/src/pyearthtools/training/modules/loss/__init__.py b/packages/training/src/pyearthtools/training/modules/loss/__init__.py index e1e44742..af582d2d 100644 --- a/packages/training/src/pyearthtools/training/modules/loss/__init__.py +++ b/packages/training/src/pyearthtools/training/modules/loss/__init__.py @@ -15,13 +15,12 @@ import importlib -from pyearthtools.training.modules.loss.extremes import ExtremeLoss +from pyearthtools.training import modules from pyearthtools.training.modules.loss.centre_weighted import centre_weighted +from pyearthtools.training.modules.loss.component import ComponentLoss +from pyearthtools.training.modules.loss.extremes import ExtremeLoss from pyearthtools.training.modules.loss.rmse import RMSELoss from pyearthtools.training.modules.loss.structure import SSIMLoss -from pyearthtools.training.modules.loss.component import ComponentLoss - -from pyearthtools.training import modules def _get_callable(module: str): diff --git a/packages/training/src/pyearthtools/training/modules/loss/centre_weighted.py b/packages/training/src/pyearthtools/training/modules/loss/centre_weighted.py index a3a067ee..6282f48b 100644 --- a/packages/training/src/pyearthtools/training/modules/loss/centre_weighted.py +++ b/packages/training/src/pyearthtools/training/modules/loss/centre_weighted.py @@ -14,11 +14,12 @@ import functools +import math from typing import Callable, Union -from torch import nn + import numpy as np import torch -import math +from torch import nn def sin(m_adjust=1, drop_rate=0.5, dimensions: tuple[int] = [-2, -1]): diff --git a/packages/training/src/pyearthtools/training/modules/loss/component.py b/packages/training/src/pyearthtools/training/modules/loss/component.py index 82619c21..ba9e79c5 100644 --- a/packages/training/src/pyearthtools/training/modules/loss/component.py +++ b/packages/training/src/pyearthtools/training/modules/loss/component.py @@ -13,9 +13,9 @@ # limitations under the License. -from torch import nn -import torch import pyearthtools.training +import torch +from torch import nn class ComponentLoss(nn.Module): diff --git a/packages/training/src/pyearthtools/training/modules/loss/structure.py b/packages/training/src/pyearthtools/training/modules/loss/structure.py index d0f4ae17..7741d0ec 100644 --- a/packages/training/src/pyearthtools/training/modules/loss/structure.py +++ b/packages/training/src/pyearthtools/training/modules/loss/structure.py @@ -13,11 +13,10 @@ # limitations under the License. +import einops import torch from piqa import SSIM -import einops - class SSIMLoss(torch.nn.Module): """ diff --git a/packages/training/src/pyearthtools/training/wrapper/__init__.py b/packages/training/src/pyearthtools/training/wrapper/__init__.py index e5101c40..abcebc4a 100644 --- a/packages/training/src/pyearthtools/training/wrapper/__init__.py +++ b/packages/training/src/pyearthtools/training/wrapper/__init__.py @@ -16,12 +16,10 @@ # ruff: noqa: F401 -from pyearthtools.training.wrapper.wrapper import ModelWrapper - from pyearthtools.training.wrapper import predict, train, utils - -from pyearthtools.training.wrapper.train import TrainingWrapper from pyearthtools.training.wrapper.predict import Predictor +from pyearthtools.training.wrapper.train import TrainingWrapper +from pyearthtools.training.wrapper.wrapper import ModelWrapper try: ONNX_IMPORTED = True diff --git a/packages/training/src/pyearthtools/training/wrapper/lightning/__init__.py b/packages/training/src/pyearthtools/training/wrapper/lightning/__init__.py index 9b082220..68d41818 100644 --- a/packages/training/src/pyearthtools/training/wrapper/lightning/__init__.py +++ b/packages/training/src/pyearthtools/training/wrapper/lightning/__init__.py @@ -19,5 +19,7 @@ Pytorch Lighting Wrappers """ -from pyearthtools.training.wrapper.lightning.predict import LightingPrediction as Predict +from pyearthtools.training.wrapper.lightning.predict import ( + LightingPrediction as Predict, +) from pyearthtools.training.wrapper.lightning.train import LightingTraining as Train diff --git a/packages/training/src/pyearthtools/training/wrapper/lightning/predict.py b/packages/training/src/pyearthtools/training/wrapper/lightning/predict.py index cea782e8..1b8bc49d 100644 --- a/packages/training/src/pyearthtools/training/wrapper/lightning/predict.py +++ b/packages/training/src/pyearthtools/training/wrapper/lightning/predict.py @@ -15,17 +15,14 @@ from __future__ import annotations -from functools import cached_property import logging import warnings - +from functools import cached_property from typing import Any -import numpy as np import lightning as L - +import numpy as np from pyearthtools.data.patterns.utils import parse_root_dir - from pyearthtools.pipeline.controller import Pipeline from pyearthtools.training.data.lightning import PipelineLightningDataModule from pyearthtools.training.wrapper.lightning.wrapper import LightningWrapper diff --git a/packages/training/src/pyearthtools/training/wrapper/lightning/train.py b/packages/training/src/pyearthtools/training/wrapper/lightning/train.py index 0dacca6f..42dc431c 100644 --- a/packages/training/src/pyearthtools/training/wrapper/lightning/train.py +++ b/packages/training/src/pyearthtools/training/wrapper/lightning/train.py @@ -14,16 +14,14 @@ from __future__ import annotations +import importlib.util import os +import warnings from pathlib import Path from typing import Any, Optional -import warnings -import importlib.util import lightning as L -from lightning.pytorch import callbacks -from lightning.pytorch import loggers - +from lightning.pytorch import callbacks, loggers from pyearthtools.pipeline.controller import Pipeline from pyearthtools.training.data.lightning import PipelineLightningDataModule from pyearthtools.training.wrapper.lightning.wrapper import LightningWrapper diff --git a/packages/training/src/pyearthtools/training/wrapper/lightning/wrapper.py b/packages/training/src/pyearthtools/training/wrapper/lightning/wrapper.py index 0c947fbe..b57cc33e 100644 --- a/packages/training/src/pyearthtools/training/wrapper/lightning/wrapper.py +++ b/packages/training/src/pyearthtools/training/wrapper/lightning/wrapper.py @@ -14,15 +14,14 @@ from __future__ import annotations + +import warnings from pathlib import Path from typing import Any, Optional -import warnings import lightning as L import torch - from pyearthtools.data.utils import parse_path - from pyearthtools.pipeline.controller import Pipeline from pyearthtools.training.data.lightning import PipelineLightningDataModule from pyearthtools.training.wrapper.wrapper import ModelWrapper diff --git a/packages/training/src/pyearthtools/training/wrapper/onnx.py b/packages/training/src/pyearthtools/training/wrapper/onnx.py index b8ea5b14..41711f39 100644 --- a/packages/training/src/pyearthtools/training/wrapper/onnx.py +++ b/packages/training/src/pyearthtools/training/wrapper/onnx.py @@ -15,17 +15,14 @@ from __future__ import annotations -from pathlib import Path import logging +from pathlib import Path from typing import Optional import onnxruntime as ort - from pyearthtools.pipeline.controller import Pipeline - -from pyearthtools.training.wrapper.wrapper import ModelWrapper from pyearthtools.training.data.datamodule import PipelineDataModule - +from pyearthtools.training.wrapper.wrapper import ModelWrapper LOG = logging.getLogger(__name__) diff --git a/packages/training/src/pyearthtools/training/wrapper/predict/__init__.py b/packages/training/src/pyearthtools/training/wrapper/predict/__init__.py index 01c64ca9..bcae9567 100644 --- a/packages/training/src/pyearthtools/training/wrapper/predict/__init__.py +++ b/packages/training/src/pyearthtools/training/wrapper/predict/__init__.py @@ -19,15 +19,14 @@ Prediction Wrappers """ -from pyearthtools.utils.decorators import BackwardsCompatibility - from pyearthtools.training.wrapper.predict.predict import Predictor from pyearthtools.training.wrapper.predict.timeseries import ( - TimeSeriesPredictor, + ManualTimeSeriesPredictor, TimeSeriesAutoRecurrentPredictor, TimeSeriesManagedPredictor, - ManualTimeSeriesPredictor, + TimeSeriesPredictor, ) +from pyearthtools.utils.decorators import BackwardsCompatibility @BackwardsCompatibility(TimeSeriesPredictor) diff --git a/packages/training/src/pyearthtools/training/wrapper/predict/coupled.py b/packages/training/src/pyearthtools/training/wrapper/predict/coupled.py index 1a809f1f..1abfc974 100644 --- a/packages/training/src/pyearthtools/training/wrapper/predict/coupled.py +++ b/packages/training/src/pyearthtools/training/wrapper/predict/coupled.py @@ -14,22 +14,19 @@ from __future__ import annotations -import functools - -from typing import Literal, TypeVar, Any, Optional +import functools from abc import abstractmethod +from typing import Any, Literal, Optional, TypeVar -from pyearthtools.data.time import TimeDelta -import xarray as xr import numpy as np import tqdm.auto as tqdm - +import xarray as xr +from pyearthtools.data.time import TimeDelta from pyearthtools.pipeline.controller import Pipeline -from pyearthtools.training.wrapper.wrapper import ModelWrapper -from pyearthtools.training.wrapper.predict.timeseries import TimeSeriesPredictor - from pyearthtools.training.manage import Variables +from pyearthtools.training.wrapper.predict.timeseries import TimeSeriesPredictor +from pyearthtools.training.wrapper.wrapper import ModelWrapper XR_TYPE = TypeVar("XR_TYPE", xr.Dataset, xr.DataArray) diff --git a/packages/training/src/pyearthtools/training/wrapper/predict/predict.py b/packages/training/src/pyearthtools/training/wrapper/predict/predict.py index 7a574d1e..802b595b 100644 --- a/packages/training/src/pyearthtools/training/wrapper/predict/predict.py +++ b/packages/training/src/pyearthtools/training/wrapper/predict/predict.py @@ -15,15 +15,12 @@ from __future__ import annotations -from typing import Any - from abc import ABCMeta - - -from pyearthtools.utils.initialisation import InitialisationRecordingMixin +from typing import Any from pyearthtools.pipeline.controller import Pipeline from pyearthtools.training.wrapper.wrapper import ModelWrapper +from pyearthtools.utils.initialisation import InitialisationRecordingMixin class Predictor(InitialisationRecordingMixin, metaclass=ABCMeta): diff --git a/packages/training/src/pyearthtools/training/wrapper/predict/timeseries.py b/packages/training/src/pyearthtools/training/wrapper/predict/timeseries.py index eda989f2..8442ee64 100644 --- a/packages/training/src/pyearthtools/training/wrapper/predict/timeseries.py +++ b/packages/training/src/pyearthtools/training/wrapper/predict/timeseries.py @@ -14,25 +14,20 @@ from __future__ import annotations -import functools - -from typing import Literal, TypeVar, Any, Optional +import functools +import logging from abc import abstractmethod +from typing import Any, Literal, Optional, TypeVar -import xarray as xr import numpy as np import tqdm.auto as tqdm -import logging - - -from pyearthtools.data import TimeDelta, Petdt, TimeRange - +import xarray as xr +from pyearthtools.data import Petdt, TimeDelta, TimeRange from pyearthtools.pipeline.controller import Pipeline -from pyearthtools.training.wrapper.wrapper import ModelWrapper -from pyearthtools.training.wrapper.predict.predict import Predictor - from pyearthtools.training.manage import Variables +from pyearthtools.training.wrapper.predict.predict import Predictor +from pyearthtools.training.wrapper.wrapper import ModelWrapper XR_TYPE = TypeVar("XR_TYPE", xr.Dataset, xr.DataArray) LOG = logging.getLogger("pyearthtools.training") @@ -83,7 +78,7 @@ def __init__( fix_time_dim (bool, optional): Fix time dimension after prediction. Defaults to True. interval (int | str | TimeDelta, optional): - Interval of temporal predictions, must be passable by `pyearthtools.data.TimeDelta`. Defaults to 1. + Interval of temporal predictions, must be passable by `pyearthtools.data.time.TimeDelta`. Defaults to 1. time_dim (str, optional): Name of time dimension in undone data. Defaults to "time". """ @@ -214,7 +209,7 @@ def __init__( fix_time_dim (bool, optional): Fix time dimension after prediction. Defaults to True. interval (int | str | TimeDelta, optional): - Interval of temporal predictions, must be passable by `pyearthtools.data.TimeDelta`. Defaults to 1. + Interval of temporal predictions, must be passable by `pyearthtools.data.time.TimeDelta`. Defaults to 1. time_dim (str, optional): Name of time dimension in undone data. Defaults to "time". combine (Optional[Literal['stack', 'combine']], optional): diff --git a/packages/training/src/pyearthtools/training/wrapper/utils.py b/packages/training/src/pyearthtools/training/wrapper/utils.py index 6027d7b7..43ed3573 100644 --- a/packages/training/src/pyearthtools/training/wrapper/utils.py +++ b/packages/training/src/pyearthtools/training/wrapper/utils.py @@ -14,12 +14,12 @@ from __future__ import annotations -import functools -from typing import Callable +import functools import math +from typing import Callable -from pyearthtools.data import TimeDelta, Petdt +from pyearthtools.data import Petdt, TimeDelta def parse_recurrent(interval: int | TimeDelta = 1): diff --git a/packages/training/src/pyearthtools/training/wrapper/wrapper.py b/packages/training/src/pyearthtools/training/wrapper/wrapper.py index 6bb2a0c8..4fe69533 100644 --- a/packages/training/src/pyearthtools/training/wrapper/wrapper.py +++ b/packages/training/src/pyearthtools/training/wrapper/wrapper.py @@ -16,10 +16,9 @@ from abc import ABCMeta, abstractmethod -from pyearthtools.utils.initialisation import InitialisationRecordingMixin from pyearthtools.pipeline import Pipeline - from pyearthtools.training.data import PipelineDataModule +from pyearthtools.utils.initialisation import InitialisationRecordingMixin class ModelWrapper(InitialisationRecordingMixin, metaclass=ABCMeta): diff --git a/packages/tutorial/src/pyearthtools/tutorial/ERA5DataClass.py b/packages/tutorial/src/pyearthtools/tutorial/ERA5DataClass.py index 7c766d08..635da7ff 100644 --- a/packages/tutorial/src/pyearthtools/tutorial/ERA5DataClass.py +++ b/packages/tutorial/src/pyearthtools/tutorial/ERA5DataClass.py @@ -28,20 +28,19 @@ from __future__ import annotations import functools - from pathlib import Path from typing import Any, Literal - import pyearthtools.data - from pyearthtools.data import Petdt +from pyearthtools.data.archive import register_archive from pyearthtools.data.exceptions import DataNotFoundError from pyearthtools.data.indexes import ArchiveIndex, decorators from pyearthtools.data.transforms import Transform, TransformCollection -from pyearthtools.data.archive import register_archive - -from pyearthtools.tutorial.ancilliary.ERA5lowres import ERA5_SINGLE_VARIABLES, ERA5_PRESSURE_VARIABLES +from pyearthtools.tutorial.ancilliary.ERA5lowres import ( + ERA5_PRESSURE_VARIABLES, + ERA5_SINGLE_VARIABLES, +) # This tells pyearthtools what the actual resolution or time-step of the data is inside the files ERA_RESOLUTION = (1, "hour") diff --git a/packages/utils/src/pyearthtools/utils/__init__.py b/packages/utils/src/pyearthtools/utils/__init__.py index 14aa1972..e2332503 100644 --- a/packages/utils/src/pyearthtools/utils/__init__.py +++ b/packages/utils/src/pyearthtools/utils/__init__.py @@ -22,14 +22,19 @@ __version__ = "0.1.0" import importlib - -from pyearthtools.utils import parameter, repr_utils, context, decorators, initialisation, config, logger -from pyearthtools.utils.initialisation import load, save, dynamic_import - - -import pyearthtools import importlib.util -from pyearthtools.utils import data +import pyearthtools +from pyearthtools.utils import ( + config, + context, + data, + decorators, + initialisation, + logger, + parameter, + repr_utils, +) +from pyearthtools.utils.initialisation import dynamic_import, load, save setattr(pyearthtools, "config", config) diff --git a/packages/utils/src/pyearthtools/utils/context.py b/packages/utils/src/pyearthtools/utils/context.py index f734c3e0..7e4f9f63 100644 --- a/packages/utils/src/pyearthtools/utils/context.py +++ b/packages/utils/src/pyearthtools/utils/context.py @@ -17,10 +17,10 @@ Contexts """ from __future__ import annotations -from typing import Any, Type, Callable -import logging +import logging from contextlib import ContextDecorator +from typing import Any, Callable, Type class ChangeValue(ContextDecorator): diff --git a/packages/utils/src/pyearthtools/utils/data/__init__.py b/packages/utils/src/pyearthtools/utils/data/__init__.py index eb465162..453ef212 100644 --- a/packages/utils/src/pyearthtools/utils/data/__init__.py +++ b/packages/utils/src/pyearthtools/utils/data/__init__.py @@ -17,5 +17,5 @@ Common Data tools """ -from pyearthtools.utils.data.tesselator import Tesselator from pyearthtools.utils.data import converter +from pyearthtools.utils.data.tesselator import Tesselator diff --git a/packages/utils/src/pyearthtools/utils/data/converter.py b/packages/utils/src/pyearthtools/utils/data/converter.py index c8d255b1..5ea6065d 100644 --- a/packages/utils/src/pyearthtools/utils/data/converter.py +++ b/packages/utils/src/pyearthtools/utils/data/converter.py @@ -17,17 +17,15 @@ from __future__ import annotations -from abc import ABCMeta, abstractmethod + +import importlib import importlib.util import json -from pathlib import Path -from typing import Any, Union, Literal - -from collections import OrderedDict import warnings - -import importlib - +from abc import ABCMeta, abstractmethod +from collections import OrderedDict +from pathlib import Path +from typing import Any, Literal, Union import numpy as np import xarray as xr diff --git a/packages/utils/src/pyearthtools/utils/data/tesselator/__init__.py b/packages/utils/src/pyearthtools/utils/data/tesselator/__init__.py index 04d62b8e..0a62a294 100644 --- a/packages/utils/src/pyearthtools/utils/data/tesselator/__init__.py +++ b/packages/utils/src/pyearthtools/utils/data/tesselator/__init__.py @@ -20,15 +20,14 @@ """ from __future__ import annotations -from typing import Union import warnings +from typing import Union import numpy as np import xarray as xr - from pyearthtools.utils.data.tesselator import _patching -from pyearthtools.utils.warnings import TesselatorWarning from pyearthtools.utils.exceptions import TesselatorException +from pyearthtools.utils.warnings import TesselatorWarning class Tesselator: diff --git a/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/patches.py b/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/patches.py index fa82e537..5d686277 100644 --- a/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/patches.py +++ b/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/patches.py @@ -23,14 +23,13 @@ import numpy as np import xarray as xr - from pyearthtools.utils.data.tesselator._patching import ( DEFAULT_FORMAT_PATCH, DEFAULT_FORMAT_PATCH_AFTER, DEFAULT_FORMAT_PATCH_ORGANISE, ) from pyearthtools.utils.data.tesselator._patching.reorder import reorder -from pyearthtools.utils.data.tesselator._patching.subset import cut_center, center +from pyearthtools.utils.data.tesselator._patching.subset import center, cut_center def factors(value: int) -> list[list[int, int]]: diff --git a/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/reorder.py b/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/reorder.py index 28e0b6e4..d2469fcb 100644 --- a/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/reorder.py +++ b/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/reorder.py @@ -18,7 +18,6 @@ """ from __future__ import annotations - import numpy as np diff --git a/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/subset.py b/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/subset.py index 69a84e97..31409c69 100644 --- a/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/subset.py +++ b/packages/utils/src/pyearthtools/utils/data/tesselator/_patching/subset.py @@ -23,10 +23,9 @@ from typing import Iterable, Union import numpy as np - -from pyearthtools.utils.exceptions import TesselatorException from pyearthtools.utils.data.tesselator._patching import DEFAULT_FORMAT_SUBSET from pyearthtools.utils.data.tesselator._patching.reorder import move_to_end, reorder +from pyearthtools.utils.exceptions import TesselatorException def cut_center(data: np.ndarray, size: Union[int, tuple[int, int]]) -> np.ndarray: diff --git a/packages/utils/src/pyearthtools/utils/decorators.py b/packages/utils/src/pyearthtools/utils/decorators.py index cfd28fb9..fd8635ab 100644 --- a/packages/utils/src/pyearthtools/utils/decorators.py +++ b/packages/utils/src/pyearthtools/utils/decorators.py @@ -16,9 +16,8 @@ from __future__ import annotations import functools -from typing import Any, Callable - import warnings +from typing import Any, Callable class classproperty(property): diff --git a/packages/utils/src/pyearthtools/utils/iPython.py b/packages/utils/src/pyearthtools/utils/iPython.py index f2e4f4f9..f699ab34 100644 --- a/packages/utils/src/pyearthtools/utils/iPython.py +++ b/packages/utils/src/pyearthtools/utils/iPython.py @@ -14,9 +14,8 @@ import numpy as np - -from PIL.Image import fromarray from IPython import get_ipython +from PIL.Image import fromarray def display_np_arrays_as_images(): diff --git a/packages/utils/src/pyearthtools/utils/initialisation/__init__.py b/packages/utils/src/pyearthtools/utils/initialisation/__init__.py index 77a67ac3..a33abe93 100644 --- a/packages/utils/src/pyearthtools/utils/initialisation/__init__.py +++ b/packages/utils/src/pyearthtools/utils/initialisation/__init__.py @@ -18,11 +18,10 @@ """ -from pyearthtools.utils.initialisation.mixin import InitialisationRecordingMixin -from pyearthtools.utils.initialisation.load import load, save, update_contents -from pyearthtools.utils.initialisation.yaml import Loader, Dumper - from pyearthtools.utils.initialisation.imports import dynamic_import +from pyearthtools.utils.initialisation.load import load, save, update_contents +from pyearthtools.utils.initialisation.mixin import InitialisationRecordingMixin +from pyearthtools.utils.initialisation.yaml import Dumper, Loader OVERRIDE_KEY = "_pyearthtools_initialisation" diff --git a/packages/utils/src/pyearthtools/utils/initialisation/imports.py b/packages/utils/src/pyearthtools/utils/initialisation/imports.py index e0627216..80cb1549 100644 --- a/packages/utils/src/pyearthtools/utils/initialisation/imports.py +++ b/packages/utils/src/pyearthtools/utils/initialisation/imports.py @@ -19,10 +19,9 @@ from __future__ import annotations -import builtins +import builtins import importlib - from types import ModuleType from typing import Callable diff --git a/packages/utils/src/pyearthtools/utils/initialisation/init_parsing.py b/packages/utils/src/pyearthtools/utils/initialisation/init_parsing.py index c668d6a6..78e0b995 100644 --- a/packages/utils/src/pyearthtools/utils/initialisation/init_parsing.py +++ b/packages/utils/src/pyearthtools/utils/initialisation/init_parsing.py @@ -21,22 +21,22 @@ # See https://github.com/Lightning-AI/pytorch-lightning/blob/master/src/lightning/pytorch/utilities/parsing.py -from collections import OrderedDict +import inspect import types +from collections import OrderedDict from typing import ( Any, Dict, List, Literal, + Mapping, MutableMapping, Optional, Sequence, Tuple, Type, Union, - Mapping, ) -import inspect class InitialisationRecord(Mapping): diff --git a/packages/utils/src/pyearthtools/utils/initialisation/load.py b/packages/utils/src/pyearthtools/utils/initialisation/load.py index 2abff80b..cced9d77 100644 --- a/packages/utils/src/pyearthtools/utils/initialisation/load.py +++ b/packages/utils/src/pyearthtools/utils/initialisation/load.py @@ -15,15 +15,13 @@ """Loading utils""" +import logging import os -from typing import Any, Union, Optional - -from pathlib import Path import re +from pathlib import Path +from typing import Any, Optional, Union -import logging import yaml - from pyearthtools.utils.initialisation.yaml import Dumper, Loader LOG = logging.getLogger("pyearthtools.utils") diff --git a/packages/utils/src/pyearthtools/utils/initialisation/mixin.py b/packages/utils/src/pyearthtools/utils/initialisation/mixin.py index bf4f9240..ccfe7282 100644 --- a/packages/utils/src/pyearthtools/utils/initialisation/mixin.py +++ b/packages/utils/src/pyearthtools/utils/initialisation/mixin.py @@ -14,8 +14,7 @@ import copy -from typing import Literal, Sequence, Any, Optional, TypeVar - +from typing import Any, Literal, Optional, Sequence, TypeVar import pyearthtools.utils from pyearthtools.utils.initialisation import init_parsing diff --git a/packages/utils/src/pyearthtools/utils/initialisation/yaml.py b/packages/utils/src/pyearthtools/utils/initialisation/yaml.py index e88e8dd2..85e2da22 100644 --- a/packages/utils/src/pyearthtools/utils/initialisation/yaml.py +++ b/packages/utils/src/pyearthtools/utils/initialisation/yaml.py @@ -13,13 +13,12 @@ # limitations under the License. -from typing import Literal, Sequence, Any, Union, Optional, TypeVar from collections.abc import Mapping - from pathlib import Path -import yaml +from typing import Any, Literal, Optional, Sequence, TypeVar, Union import pyearthtools.utils +import yaml from pyearthtools.utils.initialisation.imports import dynamic_import from pyearthtools.utils.initialisation.mixin import InitialisationRecordingMixin diff --git a/packages/utils/src/pyearthtools/utils/logger.py b/packages/utils/src/pyearthtools/utils/logger.py index 2d2be100..6d19fb43 100644 --- a/packages/utils/src/pyearthtools/utils/logger.py +++ b/packages/utils/src/pyearthtools/utils/logger.py @@ -15,16 +15,16 @@ from __future__ import annotations -import os -import socket +import datetime as dt import logging import logging.handlers +import os +import socket from typing import Literal -import datetime as dt + import yaml from pyearthtools.utils import config - LOGGING_LEVELS = Literal["CRITICAL", "FATAL", "ERROR", "WARN", "WARNING", "INFO", "DEBUG", "NOTSET"] diff --git a/packages/utils/src/pyearthtools/utils/parameter.py b/packages/utils/src/pyearthtools/utils/parameter.py index ef572220..7a66542a 100644 --- a/packages/utils/src/pyearthtools/utils/parameter.py +++ b/packages/utils/src/pyearthtools/utils/parameter.py @@ -19,11 +19,11 @@ """ +import concurrent.futures import itertools from typing import Callable -import tqdm.auto as tqdm -import concurrent.futures +import tqdm.auto as tqdm _executor = concurrent.futures.ThreadPoolExecutor() diff --git a/packages/utils/src/pyearthtools/utils/parsing/names.py b/packages/utils/src/pyearthtools/utils/parsing/names.py index 9785348e..eb98d462 100644 --- a/packages/utils/src/pyearthtools/utils/parsing/names.py +++ b/packages/utils/src/pyearthtools/utils/parsing/names.py @@ -13,8 +13,8 @@ # limitations under the License. -from typing import Callable import types +from typing import Callable def function_name(object: Callable) -> str: diff --git a/packages/utils/src/pyearthtools/utils/repr_utils/__init__.py b/packages/utils/src/pyearthtools/utils/repr_utils/__init__.py index 78db54fc..065914c5 100644 --- a/packages/utils/src/pyearthtools/utils/repr_utils/__init__.py +++ b/packages/utils/src/pyearthtools/utils/repr_utils/__init__.py @@ -15,5 +15,4 @@ from .html import provide_html from .html import provide_html as html - from .standard import provide_repr as default diff --git a/packages/utils/src/pyearthtools/utils/repr_utils/standard.py b/packages/utils/src/pyearthtools/utils/repr_utils/standard.py index 739b9c35..dc1326ad 100644 --- a/packages/utils/src/pyearthtools/utils/repr_utils/standard.py +++ b/packages/utils/src/pyearthtools/utils/repr_utils/standard.py @@ -13,9 +13,8 @@ # limitations under the License. -from typing import Optional - import copy +from typing import Optional def dynamic_padding(name, length_): diff --git a/packages/utils/tests/pyearthtools/utils/data/tesselator/test_reorder.py b/packages/utils/tests/pyearthtools/utils/data/tesselator/test_reorder.py index b557589c..f06f3c6a 100644 --- a/packages/utils/tests/pyearthtools/utils/data/tesselator/test_reorder.py +++ b/packages/utils/tests/pyearthtools/utils/data/tesselator/test_reorder.py @@ -1,6 +1,11 @@ import numpy as np import pytest -from pyearthtools.utils.data.tesselator._patching.reorder import setup_formats, reorder, move_to_end +from pyearthtools.utils.data.tesselator._patching.reorder import ( + move_to_end, + reorder, + setup_formats, +) + # Test setup_formats function. def test_setup_formats(): diff --git a/packages/utils/tests/pyearthtools/utils/data/tesselator/test_subset.py b/packages/utils/tests/pyearthtools/utils/data/tesselator/test_subset.py index 9fd1166c..9a398454 100644 --- a/packages/utils/tests/pyearthtools/utils/data/tesselator/test_subset.py +++ b/packages/utils/tests/pyearthtools/utils/data/tesselator/test_subset.py @@ -1,6 +1,6 @@ import numpy as np import pytest -from pyearthtools.utils.data.tesselator._patching.subset import cut_center, center +from pyearthtools.utils.data.tesselator._patching.subset import center, cut_center from pyearthtools.utils.exceptions import TesselatorException diff --git a/packages/utils/tests/pyearthtools/utils/data/tesselator/test_tesselator.py b/packages/utils/tests/pyearthtools/utils/data/tesselator/test_tesselator.py index 1e6e77d3..66887a87 100644 --- a/packages/utils/tests/pyearthtools/utils/data/tesselator/test_tesselator.py +++ b/packages/utils/tests/pyearthtools/utils/data/tesselator/test_tesselator.py @@ -14,9 +14,7 @@ import numpy as np - import pytest - from pyearthtools.utils.data import Tesselator tesselator_tests = [ diff --git a/packages/utils/tests/pyearthtools/utils/data/test_converter.py b/packages/utils/tests/pyearthtools/utils/data/test_converter.py index c7a1bc9f..651f8cfc 100644 --- a/packages/utils/tests/pyearthtools/utils/data/test_converter.py +++ b/packages/utils/tests/pyearthtools/utils/data/test_converter.py @@ -13,9 +13,9 @@ # limitations under the License. -from pyearthtools.utils.data import converter -import xarray as xr import pytest +import xarray as xr +from pyearthtools.utils.data import converter SIMPLE_DATA_ARRAY = xr.DataArray([1, 2, 3, 4, 5]) SIMPLE_DATA_SET = xr.Dataset({"Entry": SIMPLE_DATA_ARRAY}) diff --git a/packages/utils/tests/pyearthtools/utils/parsing/test_names.py b/packages/utils/tests/pyearthtools/utils/parsing/test_names.py index cc7121df..ade86c43 100644 --- a/packages/utils/tests/pyearthtools/utils/parsing/test_names.py +++ b/packages/utils/tests/pyearthtools/utils/parsing/test_names.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pyearthtools.utils.parsing import names import pyearthtools.utils.parameter +from pyearthtools.utils.parsing import names def test_function_name():