Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions notebooks/tutorial/Accessing_ERA5_Data.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -2332,7 +2332,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "pet-tutorials",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
Expand All @@ -2346,7 +2346,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
"version": "3.12.9"
}
},
"nbformat": 4,
Expand Down
8 changes: 4 additions & 4 deletions notebooks/tutorial/CNN_model_training.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -782,10 +782,10 @@
" * Input data: (-1, 1) tuple is used to select a single timestamp at -1hrs\n",
" * Target data: (6, 1) tuple is used to select a single timestamp at +5hrs ahead. \n",
"\n",
"### Aditional Steps:\n",
"### Additional Steps:\n",
"1. Export to a NumPy array.\n",
"2. Rearrange the axes of the NumPy array.\n",
"3. Remove dimensions of size 1 via a \"squish\" (equivalent to a NumPy \"squeeze\") operation."
"3. Remove dimensions of size 1 via a \"squeeze\" operation."
]
},
{
Expand Down Expand Up @@ -1347,7 +1347,7 @@
" ),\n",
" pyearthtools.pipeline.operations.xarray.conversion.ToNumpy(),\n",
" pyearthtools.pipeline.operations.numpy.reshape.Rearrange(\"c t h w -> t c h w\"),\n",
" pyearthtools.pipeline.operations.numpy.reshape.Squish(axis=0),\n",
" pyearthtools.pipeline.operations.numpy.reshape.Squeeze(axis=0),\n",
")\n",
"data_preparation"
]
Expand Down Expand Up @@ -8350,7 +8350,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.12.9"
}
},
"nbformat": 4,
Expand Down
4 changes: 2 additions & 2 deletions notebooks/tutorial/Data_Pipelines.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@
" # These methods will be explained when we create a pipeline for machine learning. \n",
" # pyearthtools.pipeline.operations.xarray.reshape.CoordinateFlatten('level'),\n",
" # pyearthtools.pipeline.operations.xarray.conversion.ToNumpy(),\n",
" # pyearthtools.pipeline.operations.numpy.reshape.Squish(1),\n",
" # pyearthtools.pipeline.operations.numpy.reshape.Squeeze(1),\n",
")\n"
]
},
Expand Down Expand Up @@ -2081,7 +2081,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.12.9"
}
},
"nbformat": 4,
Expand Down
2 changes: 1 addition & 1 deletion notebooks/tutorial/Downloading_ERA5.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.12.9"
}
},
"nbformat": 4,
Expand Down
2 changes: 1 addition & 1 deletion notebooks/tutorial/Interfacing_to_Data_at_NCI.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -1246,7 +1246,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.12.9"
}
},
"nbformat": 4,
Expand Down
2 changes: 1 addition & 1 deletion notebooks/tutorial/MultipleSources.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -4534,7 +4534,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.12.9"
}
},
"nbformat": 4,
Expand Down
2 changes: 1 addition & 1 deletion notebooks/tutorial/Working_with_Climate_Data.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -5483,7 +5483,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.12.9"
}
},
"nbformat": 4,
Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
4 changes: 2 additions & 2 deletions old_docs/docs/documentation/pipeline/available.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ Here are the default operations included with `pyearthtools.pipeline`, accessibl
| filters | Filter data when iterating | `DropAnyNan`, `DropAllNan`, `DropValue`, `Shape` |
| join | Combine tuples of `np.ndarrays` | `Stack`, `VStack`, `HStack`, `Concatenate` |
| normalisation | Normalise arrays | `Anomaly`, `Deviation`, `Division`, `Evaluated` |
| reshape | Reshape numpy array | `Rearrange`, `Squish`, `Expand`, `Flatten`, `SwapAxis` |
| reshape | Reshape numpy array | `Rearrange`, `Squeeze`, `Expand`, `Flatten`, `SwapAxis` |
| select | Select elements from array | `Select`, `Slice` |
| split | Split numpy arrays into tuples | `OnAxis`, `OnSlice`, `VSplit`, `HSplit` |
| values | Modify values of arrays | `FillNan`, `MaskValue`, `ForceNormalised` |
Expand All @@ -67,7 +67,7 @@ Here are the default operations included with `pyearthtools.pipeline`, accessibl
| filters | Filter data when iterating | `DropAnyNan`, `DropAllNan`, `DropValue`, `Shape` |
| join | Combine tuples of `np.ndarrays` | `Stack`, `VStack`, `HStack`, `Concatenate` |
| normalisation | Normalise arrays | `Anomaly`, `Deviation`, `Division`, `Evaluated` |
| reshape | Reshape numpy array | `Rearrange`, `Squish`, `Expand`, `Flatten`, `SwapAxis` |
| reshape | Reshape numpy array | `Rearrange`, `Squeeze`, `Expand`, `Flatten`, `SwapAxis` |
| select | Select elements from array | `Select`, `Slice` |
| split | Split numpy arrays into tuples | `OnAxis`, `OnSlice`, `VSplit`, `HSplit` |
| values | Modify values of arrays | `FillNan`, `MaskValue`, `ForceNormalised` |
Expand Down
38 changes: 22 additions & 16 deletions old_docs/docs/documentation/pipeline/creating.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,27 +48,29 @@ pyearthtools.pipeline.Pipeline(
(
pyearthtools.data.archive.ERA5(['tcwv', 'skt', 'sp']),
pyearthtools.pipeline.operations.Transforms(
apply = pyearthtools.pipeline.operations.transform.AddCoordinates(('latitude', 'longitude'))),
apply=pyearthtools.pipeline.operations.transform.AddCoordinates(('latitude', 'longitude'))),
pyearthtools.pipeline.operations.xarray.Sort(('var_latitude', 'var_longitude', 'tcwv', 'skt', 'sp'))),
(
pyearthtools.data.archive.ERA5(['t', 'u', 'v'], level_value = [1,50,150,250,400,600,750,900,1000]),
pyearthtools.data.archive.ERA5(['t', 'u', 'v'], level_value=[1, 50, 150, 250, 400, 600, 750, 900, 1000]),
pyearthtools.pipeline.operations.xarray.Sort(('t', 'u', 'v'))
),
(
(
pyearthtools.data.archive.ERA5(
['mtnlwrf', 'msdwswrf', 'msdwlwrf', 'mtpr', 'mslhf', 'msshf', 'mtnswrf', 'mtdwswrf', 'msnswrf', 'msnlwrf'],
transforms = pyearthtools.data.transforms.derive(
mtupswrf = 'mtnswrf - mtdwswrf',
msupswrf = 'msnswrf - msdwswrf',
msuplwrf = 'msnlwrf - msdwlwrf',
drop = True
pyearthtools.data.archive.ERA5(
['mtnlwrf', 'msdwswrf', 'msdwlwrf', 'mtpr', 'mslhf', 'msshf', 'mtnswrf', 'mtdwswrf', 'msnswrf',
'msnlwrf'],
transforms=pyearthtools.data.transforms.derive(
mtupswrf='mtnswrf - mtdwswrf',
msupswrf='msnswrf - msdwswrf',
msuplwrf='msnlwrf - msdwlwrf',
drop=True
)
),
pyearthtools.data.archive.ERA5('!accumulate[period:"6 hours"]:tp>tp_accum')
pyearthtools.data.archive.ERA5('!accumulate[period:"6 hours"]:tp>tp_accum')
),
pyearthtools.pipeline.operations.xarray.Merge(),
pyearthtools.pipeline.operations.xarray.Sort(('mslhf', 'msshf', 'msuplwrf', 'msupswrf', 'mtnlwrf', 'mtpr', 'mtupswrf', 'tp_accum'))
pyearthtools.pipeline.operations.xarray.Sort(
('mslhf', 'msshf', 'msuplwrf', 'msupswrf', 'mtnlwrf', 'mtpr', 'mtupswrf', 'tp_accum'))
),
(
pyearthtools.data.archive.ERA5(['mtdwswrf', 'z_surface', 'lsm', 'ci']),
Expand All @@ -78,15 +80,19 @@ pyearthtools.pipeline.Pipeline(
pyearthtools.pipeline.operations.xarray.reshape.CoordinateFlatten('level', skip_missing=True),
pyearthtools.pipeline.operations.xarray.reshape.Dimensions(('time', 'latitude', 'longitude')),
pyearthtools.pipeline.operations.Transforms(
apply = pyearthtools.data.transform.coordinates.pad(coordinates = {'latitude': 1, 'longitude': 1}, mode = 'wrap') + pyearthtools.data.transforms.values.fill(coordinates = ['latitude', 'longitude'], direction = 'forward') + pyearthtools.data.transforms.interpolation.like(pipe['2020-01-01T00'], drop_coords = 'time')),
apply=pyearthtools.data.transform.coordinates.pad(coordinates={'latitude': 1, 'longitude': 1},
mode='wrap') + pyearthtools.data.transforms.values.fill(
coordinates=['latitude', 'longitude'],
direction='forward') + pyearthtools.data.transforms.interpolation.like(pipe['2020-01-01T00'],
drop_coords='time')),

pyearthtools.pipeline.operations.xarray.Merge(),
pyearthtools.pipeline.operations.xarray.Sort(order, safe = True),
pyearthtools.pipeline.operations.xarray.Sort(order, safe=True),

pyearthtools.pipeline.operations.xarray.conversion.ToDask(),
pyearthtools.pipeline.operations.dask.reshape.Squish(axis=1),
pyearthtools.pipeline.modifications.Cache('temp', pattern_kwargs = dict(extension = 'npy'))
pyearthtools.pipeline.modifications.TemporalRetrieval(((-6, 1), (6,1))),
pyearthtools.pipeline.operations.dask.reshape.Squeeze(axis=1),
pyearthtools.pipeline.modifications.Cache('temp', pattern_kwargs=dict(extension='npy'))
pyearthtools.pipeline.modifications.TemporalRetrieval(((-6, 1), (6, 1))),
)
```
![Pipeline Graph](./assets/ComplexPipeline.svg)
Expand Down
10 changes: 5 additions & 5 deletions old_docs/docs/documentation/pipeline/details/operation.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,24 +60,24 @@ inverse = example_operation.T

## Example Implementation

Here is the implementation of `numpy.reshape.Squish`, to flatten a one element axis of an array
Here is the implementation of `numpy.reshape.Squeeze`, to flatten a one element axis of an array

```python
from typing import Union, Optional, Any
import numpy as np

from pyearthtools.pipeline.operation import Operation

class Squish(Operation):
class Squeeze(Operation):
"""
Operation to Squish one Dimensional axis at 'axis' location
Operation to Squeeze one Dimensional axis at 'axis' location
"""

_override_interface = ["Delayed", "Serial"] # Which parallel interfaces to use in order of priority.
_interface_kwargs = {"Delayed": {"name": "Squish"}}
_interface_kwargs = {"Delayed": {"name": "Squeeze"}}

def __init__(self, axis: Union[tuple[int, ...], int]) -> None:
"""Squish Dimension of Data
"""Squeeze Dimension of Data

Args:
axis (Union[tuple[int, ...], int]):
Expand Down
13 changes: 7 additions & 6 deletions old_docs/docs/documentation/pipeline/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,21 @@ Here is an example pipeline, for use with PanguWeather.
import pyearthtools.data
import pyearthtools.pipeline


data_preperation = pyearthtools.pipeline.Pipeline(
(
pyearthtools.data.archive.ERA5(['msl', '10u', '10v', '2t']),
pyearthtools.data.archive.ERA5(['z', 'q', 't', 'u', 'v'], level_value = [50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 850, 925, 1000])
pyearthtools.data.archive.ERA5(['z', 'q', 't', 'u', 'v'],
level_value=[50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 850, 925, 1000])
),
pyearthtools.pipeline.operations.xarray.Merge(),
pyearthtools.pipeline.operations.xarray.Sort(['msl', '10u', '10v', '2t', 'z', 'q', 't', 'u', 'v']),
pyearthtools.pipeline.operations.Transforms(
apply = pyearthtools.data.transforms.coordinates.standard_longitude(type = '0-360') + pyearthtools.data.transforms.coordinates.ReIndex(level = 'reversed')
),
pyearthtools.pipeline.operations.xarray.reshape.CoordinateFlatten(coordinate = 'level'),
apply=pyearthtools.data.transforms.coordinates.standard_longitude(
type='0-360') + pyearthtools.data.transforms.coordinates.ReIndex(level='reversed')
),
pyearthtools.pipeline.operations.xarray.reshape.CoordinateFlatten(coordinate='level'),
pyearthtools.pipeline.operations.xarray.conversion.ToNumpy(),
pyearthtools.pipeline.operations.numpy.reshape.Squish(axis = 1),
pyearthtools.pipeline.operations.numpy.reshape.Squeeze(axis=1),
)
```

Expand Down
2 changes: 1 addition & 1 deletion packages/pipeline/assets/pipeline_example.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
| filters | Filter data when iterating | `DropAnyNan`, `DropAllNan`, `DropValue`, `Shape` |
| join | Combine tuples of `np.ndarrays` | `Stack`, `VStack`, `HStack`, `Concatenate` |
| normalisation | Normalise arrays | `Anomaly`, `Deviation`, `Division`, `Evaluated` |
| reshape | Reshape numpy array | `Rearrange`, `Squish`, `Expand`, `Flatten`, `SwapAxis` |
| reshape | Reshape numpy array | `Rearrange`, `Squeeze`, `Expand`, `Flatten`, `SwapAxis` |
| select | Select elements from array | `Select`, `Slice` |
| split | Split numpy arrays into tuples | `OnAxis`, `OnSlice`, `VSplit`, `HSplit` |
| values | Modify values of arrays | `FillNan`, `MaskValue`, `ForceNormalised` |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,16 +110,16 @@ def undo_func(self, data: Union[np.ndarray, da.Array]):
return self._rearrange(data, pattern)


class Squish(DaskOperation):
class Squeeze(DaskOperation):
"""
Operation to Squish one Dimensional axis at 'axis' location
Operation to Squeeze one Dimensional axis at 'axis' location
"""

_override_interface = ["Serial"]
_numpy_counterpart = "reshape.Squish"
_numpy_counterpart = "reshape.Squeeze"

def __init__(self, axis: Union[tuple[int, ...], int]) -> None:
"""Squish Dimension of Data
"""Squeeze Dimension of Data

Args:
axis (Union[tuple[int, ...], int]):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
| filters | Filter data when iterating | `DropAnyNan`, `DropAllNan`, `DropValue`, `Shape` |
| join | Combine tuples of `np.ndarrays` | `Stack`, `VStack`, `HStack`, `Concatenate` |
| normalisation | Normalise arrays | `Anomaly`, `Deviation`, `Division`, `Evaluated` |
| reshape | Reshape numpy array | `Rearrange`, `Squish`, `Expand`, `Flatten`, `SwapAxis` |
| reshape | Reshape numpy array | `Rearrange`, `Squeeze`, `Expand`, `Flatten`, `SwapAxis` |
| select | Select elements from array | `Select`, `Slice` |
| split | Split numpy arrays into tuples | `OnAxis`, `OnSlice`, `VSplit`, `HSplit` |
| values | Modify values of arrays | `FillNan`, `MaskValue`, `ForceNormalised` |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,21 +94,21 @@ def undo_func(self, data: np.ndarray):
return self._rearrange(data, pattern)


class Squish(Operation):
class Squeeze(Operation):
"""
Operation to Squish one Dimensional axis at 'axis' location
Operation to Squeeze one-Dimensional axes at 'axis' location

"""

_override_interface = ["Delayed", "Serial"]
_interface_kwargs = {"Delayed": {"name": "Squish"}}
_interface_kwargs = {"Delayed": {"name": "Squeeze"}}

def __init__(self, axis: Union[tuple[int, ...], int]) -> None:
"""Squish Dimension of Data
"""Squeeze Dimension of Data, removing dimensions of length 1.

Args:
axis (Union[tuple[int, ...], int]):
Axis to squish at
Axis to squeeze at
"""
super().__init__(
split_tuples=True,
Expand Down