Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/data/src/pyearthtools/data/archive/extensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def decorator(archive_index: Any):
def sample() -> pyearthtools.data.Index:
if sample_kwargs is not None:
return archive_index(**sample_kwargs)
raise RuntimeError(f"Keyword arguments were not given to create a `sample` index.")
raise RuntimeError("Keyword arguments were not given to create a `sample` index.")

setattr(archive_index, "sample", sample)

Expand Down
15 changes: 7 additions & 8 deletions packages/data/src/pyearthtools/data/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,8 @@
from typing import Optional

import yaml
import json
from pathlib import Path
from typing import Any, Callable, Optional
from typing import Any, Callable
import types
import warnings
from functools import lru_cache
Expand All @@ -57,7 +56,7 @@
from pyearthtools.utils.initialisation.imports import dynamic_import

import pyearthtools.data
from pyearthtools.data.collection import Collection, LabelledCollection
from pyearthtools.data.collection import Collection

from pyearthtools.utils.decorators import alias_arguments

Expand Down Expand Up @@ -464,7 +463,7 @@ def append(

elif isinstance(other, pyearthtools.data.DataIndex) and not hasattr(other, "catalog"):
raise AttributeError(
f"Object to append appears to be a `pyearthtools.data.DataIndex`, but has no `catalog`.\n"
"Object to append appears to be a `pyearthtools.data.DataIndex`, but has no `catalog`.\n"
"Ensure, `.record_initialisation()` has been run in the `__init__` of the Index."
)

Expand Down Expand Up @@ -560,7 +559,7 @@ def save(self, output_file: str | Path | io.IOBase | None = None, direct_load: b

try:
yaml.dump(save_catalog, fileIO, sort_keys=False)
except PermissionError as e:
except PermissionError:
warnings.warn(
f"Could not save to {output_file!s}, due to a PermissionError",
UserWarning,
Expand Down Expand Up @@ -614,7 +613,7 @@ def load(
if len(possible_catalogs) > 1:
raise FileNotFoundError(f"Multiple catalogs found in {catalog_to_load!s}")
raise FileNotFoundError(
f"Given file was actually a directory, and no valid `pyearthtools` catalog was found inside."
"Given file was actually a directory, and no valid `pyearthtools` catalog was found inside."
"\nEnsure the catalog ends in '.cat'"
)

Expand All @@ -640,7 +639,7 @@ def load(

if direct_load:
if len(new_catalog) > 1:
raise TypeError(f"Catalog cannot be direct loaded, and 'direct_load' is True")
raise TypeError("Catalog cannot be direct loaded, and 'direct_load' is True")
return new_catalog[0].function
return new_catalog

Expand Down Expand Up @@ -686,7 +685,7 @@ def __str__(self):

@property
def _doc_(self):
return f"Catalog of Data"
return "Catalog of Data"

@property
def _info_(self):
Expand Down
6 changes: 3 additions & 3 deletions packages/data/src/pyearthtools/data/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ def setup(verbose):
import pyearthtools.data

if pyearthtools.data.static.geographic._download_all(verbose=verbose):
print(f"Successfully downloaded all files")
print("Successfully downloaded all files")
else:
print(f"Failed to download all files")
print("Failed to download all files")


def split_dictionary(dictionary: dict[str, dict] = {}, **kwargs) -> list[list[str]]:
Expand Down Expand Up @@ -104,7 +104,7 @@ def create_structure(top, disallowed, save, verbose):
structure_dict: dict[str, dict | list] = {}
structure_d: dict[str, dict[str, Any] | list[str]] = structure(top, disallowed=disallowed, verbose=verbose) # type: ignore

response = input(f"Would you like to specify the order? (Yes/No): ")
response = input("Would you like to specify the order? (Yes/No): ")
order = []
if "y" in response.lower():
for level in split_dictionary(structure_d): # type: ignore
Expand Down
2 changes: 0 additions & 2 deletions packages/data/src/pyearthtools/data/decorators.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,3 @@
"All decorators once here have been moved from `pyearthtools.data.decorators` to `pyearthtools.utils.decorators`, and will be removed here in the future",
FutureWarning,
)

from pyearthtools.utils.decorators import alias_arguments
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

import pyearthtools.data
from pyearthtools.data.indexes import (
alias_arguments,
check_arguments,
VariableDefault,
VARIABLE_DEFAULT,
Expand Down Expand Up @@ -188,7 +187,7 @@ def __init__(
def _get_latest(self) -> str:
"""Get latest datetime from `ecmwf-opendata`"""
if self._client is None:
raise ImportError(f"`ecwmwf.opendata` was not imported, cannot download new data.")
raise ImportError("`ecwmwf.opendata` was not imported, cannot download new data.")
return str(self._client.latest(self._request_base))

def latest(self) -> xr.Dataset:
Expand All @@ -198,7 +197,7 @@ def latest(self) -> xr.Dataset:
def download(self, querytime: str | pyearthtools.data.Petdt) -> xr.Dataset:
"""Download data from `ecwmf-opendata`"""
if self._client is None:
raise ImportError(f"`ecwmwf.opendata` was not imported, cannot download new data.")
raise ImportError("`ecwmwf.opendata` was not imported, cannot download new data.")

if querytime == "l1atest":
querytime = self._get_latest()
Expand Down
1 change: 0 additions & 1 deletion packages/data/src/pyearthtools/data/indexes/combine.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

from pyearthtools.data.indexes import Index, AdvancedTimeDataIndex
from pyearthtools.data import Petdt
from pyearthtools.data.operations import SpatialInterpolation, TemporalInterpolation
from pyearthtools.data.transforms.transform import Transform, TransformCollection


Expand Down
1 change: 0 additions & 1 deletion packages/data/src/pyearthtools/data/indexes/intake.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
from pyearthtools.data.transforms import Transform, TransformCollection
from pyearthtools.data.patterns.argument import flattened_combinations

from pyearthtools.data.indexes.utilities.delete_files import delete_path

LOG = logging.getLogger("pyearthtools.data")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
from typing import Any, Type

from pyearthtools.data.exceptions import InvalidIndexError
from pyearthtools.data.indexes.utilities.fileload import open_static


class VariableDefault:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def get_structure(top: str | Path, disallowed: list[str], verbose: bool = False)
for component in Path(dirpath).relative_to(top).parts:
if component in disallowed:
continue
if not component in sub_dict:
if component not in sub_dict:
sub_dict[component] = {} if filter_disallowed(dirnames, disallowed) else None
sub_dict = sub_dict[component]
return structure
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from typing import Literal
import xarray as xr

from pyearthtools.data.time import Petdt, TimeDelta, TimeResolution, TimeRange
from pyearthtools.data.time import Petdt, TimeDelta, TimeResolution
from pyearthtools.data.operations.utils import identify_time_dimension

from pyearthtools.data.modifications import Modification, register_modification
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,10 @@ def aggregation(
Dataset with aggregation method applied
"""
if not reduce_dims and not preserve_dims:
raise ValueError(f"Either 'reduce_dims' or 'preserve_dims' must be given ")
raise ValueError("Either 'reduce_dims' or 'preserve_dims' must be given ")

if reduce_dims and preserve_dims:
raise ValueError(f"Both 'reduce_dims' and 'preserve_dims' cannot be given ")
raise ValueError("Both 'reduce_dims' and 'preserve_dims' cannot be given ")

if reduce_dims:
aggregation_func = aggr_trans.over(aggregation, reduce_dims) # type: ignore
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,9 @@
import xarray as xr


from pyearthtools.data.exceptions import DataNotFoundError, InvalidIndexError, InvalidDataError
from pyearthtools.data.exceptions import DataNotFoundError, InvalidDataError
from pyearthtools.data.time import Petdt, TimeDelta, TimeRange
from pyearthtools.data.transforms.transform import Transform, TransformCollection
from pyearthtools.data.warnings import IndexWarning

from pyearthtools.data.operations import index_routines
from pyearthtools.data.operations.utils import identify_time_dimension
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,15 @@
import builtins
import os
from pathlib import Path
from typing import Callable, Union

import numpy as np
from tqdm.auto import tqdm, trange
from tqdm.auto import trange
import xarray as xr

import pyearthtools.data

from pyearthtools.data.time import Petdt, TimeDelta
from pyearthtools.data.exceptions import (
DataNotFoundError,
InvalidIndexError,
run_and_catch_exception,
)
from pyearthtools.data.transforms import Transform, TransformCollection

Expand Down Expand Up @@ -214,7 +210,7 @@ def aggregation(

if save_file is not None:
dataset.to_netcdf(save_file, mode="w")
for file in (Path(save_location).with_suffix("") / f"components/").glob("temp_file_*.nc"):
for file in (Path(save_location).with_suffix("") / "components/").glob("temp_file_*.nc"):
os.remove(file)
return dataset

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@

from __future__ import annotations

import functools
import logging
from typing import Iterable

Expand Down Expand Up @@ -129,7 +128,7 @@ def series(
verbose=verbose,
**kwargs,
)
except NotImplementedError as e:
except NotImplementedError:
data = _get_series(
DataFunction,
start,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ def TemporalInterpolation(
reference_dataset = listed_datasets.pop(0)

if "time" not in reference_dataset:
raise ValueError(f"reference dataset contains no dimension 'time'. Cannot interpolate")
raise ValueError("reference dataset contains no dimension 'time'. Cannot interpolate")

last_index = None
aggregation = pyearthtools.data.transforms.aggregation.over(aggregation_function, dimension="time", **kwargs)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from __future__ import annotations

from typing import Iterable, Union
from typing import Iterable

import numpy as np
import xarray as xr
Expand Down
1 change: 0 additions & 1 deletion packages/data/src/pyearthtools/data/operations/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@


from __future__ import annotations
from typing import Hashable

import xarray as xr

Expand Down
2 changes: 0 additions & 2 deletions packages/data/src/pyearthtools/data/save/dask.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@

from __future__ import annotations

from pathlib import Path
from typing import Any

from pyearthtools.data.save.array import save as numpy_save

Expand Down
2 changes: 1 addition & 1 deletion packages/data/src/pyearthtools/data/save/dispatcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
from dask.delayed import Delayed
import dask.array as da
from pyearthtools.data.save import dask as dask_save
except (ImportError, ModuleNotFoundError) as e:
except (ImportError, ModuleNotFoundError):
DASK_IMPORTED = False

DISPATCH = {
Expand Down
6 changes: 3 additions & 3 deletions packages/data/src/pyearthtools/data/time.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,7 @@ def is_time(time_to_parse: Any) -> bool:
try:
Petdt(time_to_parse)
return True
except Exception as e:
except Exception:
return False

def at_resolution(
Expand Down Expand Up @@ -993,7 +993,7 @@ def __iter__(self) -> Generator[Petdt, None, None]:
pbar.update(1)
else:
if self.kwargs:
raise TypeError(f"kwargs given but not using `tqdm`, no execution path for kwargs.")
raise TypeError("kwargs given but not using `tqdm`, no execution path for kwargs.")

for i in samples:
yield i
Expand All @@ -1016,4 +1016,4 @@ def TimeDeltaConstructer(loader: yaml.loader.Loader, tag_suffix: str, node):


initialisation.Dumper.add_multi_representer(TimeDelta, TimeDeltaRepresenter)
initialisation.Loader.add_multi_constructor(f"!TimeDelta", TimeDeltaConstructer)
initialisation.Loader.add_multi_constructor("!TimeDelta", TimeDeltaConstructer)
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

from __future__ import annotations

from typing import Callable, Optional, Union
from typing import Callable, Optional

import xarray as xr

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ def __init__(self, dtype: str | dict[str, str] | None = None, **variables: Any):
self.record_initialisation()

if not dtype and not variables:
raise ValueError(f"Either `dtype` or `**variables` must be given.")
raise ValueError("Either `dtype` or `**variables` must be given.")

self._dtype = dtype
self._variables = variables
Expand Down
4 changes: 2 additions & 2 deletions packages/data/src/pyearthtools/data/transforms/coordinates.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ def __init__(
coordinates.update(coords)

if not coordinates:
raise ValueError(f"No coordinates to reindex at, must be given either with `coordinates` or `kwargs`.")
raise ValueError("No coordinates to reindex at, must be given either with `coordinates` or `kwargs`.")
self._coordinates = coordinates

@property
Expand All @@ -193,7 +193,7 @@ def _info_(self):

def apply(self, dataset: xr.Dataset):
for coord, index_op in self._coordinates.items():
if not coord in dataset.coords:
if coord not in dataset.coords:
continue

if isinstance(index_op, str):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@


from __future__ import annotations
from typing import Any, Optional, Literal, TypeVar
from typing import Optional, Literal, TypeVar

import xarray as xr

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,17 +185,17 @@ def __init__(
self.record_initialisation()

if not xESMF_IMPORTED:
raise ImportError(f"xesmf could not be imported")
raise ImportError("xesmf could not be imported")
if not reference_dataset and not coords:
raise KeyError(f"Either 'reference_dataset' or 'coords' must be given")
raise KeyError("Either 'reference_dataset' or 'coords' must be given")

def get_reference(reference_dataset: Optional[xr.Dataset] = None, coords: Optional[dict] = None):
if reference_dataset:
return reference_dataset

try:
return xr.Dataset(coords)
except ValueError as e:
except ValueError:
pass

if coords:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@
from pyearthtools.data.transforms.transform import (
FunctionTransform,
Transform,
get_default_transforms,
)

from pyearthtools.data.transforms.default import get_default_transforms

from pyearthtools.data.indexes.utilities.fileload import open_files

from pyearthtools.utils.initialisation.imports import dynamic_import
Expand Down Expand Up @@ -519,7 +521,7 @@ def __call__(self, method: str | dict | tuple, default: str | None = None) -> Tr
raise TypeError(f"{type(method)!r} for 'method' not understood")

def __repr__(self):
return f"Normalisation Class waiting upon a request for a method, either call with a method or use property."
return "Normalisation Class waiting upon a request for a method, either call with a method or use property."

def __del__(self):
if hasattr(self, "temp_dir"):
Expand Down
Loading