ACCESS-Community-Hub · tennlee · May 22, 2025 · May 22, 2025 · May 22, 2025 · May 22, 2025
diff --git a/notebooks/tutorial/FourCastMini Demo.ipynb b/notebooks/tutorial/FourCastMini Demo.ipynb
diff --git a/packages/bundled_models/fourcastnext/Training/configs/config.yaml b/packages/bundled_models/fourcastnext/Training/configs/config.yaml
@@ -10,7 +10,7 @@ defaults:
 experiment_name: ${now:%Y-%m-%d}/${now:%H-%M-%S}
 path: /scratch/kd24/${oc.env:USER}/ML/FourCastNeXt/Training/${experiment_name}
 
-fit: False
+fit: false
 
 hydra:
   verbose: false

diff --git a/packages/bundled_models/fourcastnext/Training/configs/data/module/default.yaml b/packages/bundled_models/fourcastnext/Training/configs/data/module/default.yaml
@@ -1,2 +1,2 @@
-num_workers: 12
-batch_size: 8
+num_workers: 4
+batch_size: 4
diff --git a/packages/bundled_models/fourcastnext/Training/configs/data/splits/default.yaml b/packages/bundled_models/fourcastnext/Training/configs/data/splits/default.yaml
@@ -1,4 +1,4 @@
-random: True
+random: true
 random_seed: 42
 train:
   - 1980

diff --git a/packages/bundled_models/fourcastnext/Training/configs/data/splits/short_training.yaml b/packages/bundled_models/fourcastnext/Training/configs/data/splits/short_training.yaml
@@ -0,0 +1,10 @@
+random: true
+random_seed: 42
+train:
+  - 2000
+  - 2015
+  - 6 hours
+valid:
+  - 2018
+  - 2020
+  - 6 hours
diff --git a/packages/bundled_models/fourcastnext/Training/configs/model/default.yaml b/packages/bundled_models/fourcastnext/Training/configs/model/default.yaml
@@ -1,10 +1,9 @@
-
+---
 _target_: fourcastnext.lightning_model.FourCastNextLM
 _recursive_: true
 
-
 # Model parameters to init AFNONet
 model_params:
-  img_size : ${data.img_size}
-  in_channels : ${data.in_channels}
-  out_channels : ${data.out_channels}
+  img_size: ${data.img_size}
+  in_channels: ${data.in_channels}
+  out_channels: ${data.out_channels}
diff --git a/packages/bundled_models/fourcastnext/Training/configs/trainer/default.yaml b/packages/bundled_models/fourcastnext/Training/configs/trainer/default.yaml
@@ -1,5 +1,6 @@
+---
 precision: 16-mixed
-max_epochs: 200
+max_epochs: 5
 
 checkpointing:
 - monitor: "train_loss"
@@ -18,5 +19,5 @@ checkpointing:
   mode: "max"
   dirpath: "{path}/Checkpoints/Epoch"
   filename: "model-{epoch:02d}"
-  save_on_train_epoch_end: True
+  save_on_train_epoch_end: true
   save_top_k: 50
diff --git a/...es/bundled_models/fourcastnext/Training/limited_variables_early_stopping/data/lowres.yaml b/...es/bundled_models/fourcastnext/Training/limited_variables_early_stopping/data/lowres.yaml
@@ -0,0 +1,9 @@
+# Use the ERA5 limited variable pipeline
+
+pipelines: pipelines/limited_variables.pipe
+
+img_size:
+  - 64
+  - 28
+in_channels: 4
+out_channels: 4
diff --git a/...ed_models/fourcastnext/Training/limited_variables_early_stopping/data/module/default.yaml b/...ed_models/fourcastnext/Training/limited_variables_early_stopping/data/module/default.yaml
@@ -1,2 +1,2 @@
 num_workers: 4
-batch_size: 1
+batch_size: 4
diff --git a/...ed_models/fourcastnext/Training/limited_variables_early_stopping/data/splits/default.yaml b/...ed_models/fourcastnext/Training/limited_variables_early_stopping/data/splits/default.yaml
@@ -1,8 +1,8 @@
-random: True
+random: true
 random_seed: 42
 train:
-  - 2000
-  - 2015
+  - 1980
+  - 2018
   - 6 hours
 valid:
   - 2018

diff --git a/...castnext/Training/limited_variables_early_stopping/data/splits/short_training_splits.yaml b/...castnext/Training/limited_variables_early_stopping/data/splits/short_training_splits.yaml
@@ -0,0 +1,11 @@
+---
+random: true
+random_seed: 42
+train:
+  - 2000
+  - 2015
+  - 6 hours
+valid:
+  - 2018
+  - 2020
+  - 6 hours
diff --git a/...odels/fourcastnext/Training/limited_variables_early_stopping/limited_vars_early_stop.yaml b/...odels/fourcastnext/Training/limited_variables_early_stopping/limited_vars_early_stop.yaml
@@ -10,7 +10,7 @@ defaults:
 experiment_name: ${now:%Y-%m-%d}/${now:%H-%M-%S}
 path: /scratch/kd24/${oc.env:USER}/ML/FourCastNeXt/Training/${experiment_name}
 
-fit: False
+fit: false
 
 hydra:
   verbose: false

diff --git a/packages/bundled_models/fourcastnext/Training/limited_variables_early_stopping/lowres.yaml b/packages/bundled_models/fourcastnext/Training/limited_variables_early_stopping/lowres.yaml
@@ -0,0 +1,19 @@
+---
+defaults:
+ - _self_
+ - trainer: few_epochs
+ - model: default
+ - data: lowres
+
+ - data/splits: short_training_splits
+ - data/module: default
+
+experiment_name: ${now:%Y-%m-%d}/${now:%H-%M-%S}
+path: ./Experiments/FourCastNeXt/Training/${experiment_name}
+
+fit: false
+
+hydra:
+  verbose: false
+  run:
+    dir: ${path}
diff --git a/.../bundled_models/fourcastnext/Training/limited_variables_early_stopping/model/default.yaml b/.../bundled_models/fourcastnext/Training/limited_variables_early_stopping/model/default.yaml
@@ -1,10 +1,9 @@
-
+---
 _target_: fourcastnext.lightning_model.FourCastNextLM
 _recursive_: true
 
-
 # Model parameters to init AFNONet
 model_params:
-  img_size : ${data.img_size}
-  in_channels : ${data.in_channels}
-  out_channels : ${data.out_channels}
+  img_size: ${data.img_size}
+  in_channels: ${data.in_channels}
+  out_channels: ${data.out_channels}
diff --git a/...undled_models/fourcastnext/Training/limited_variables_early_stopping/trainer/default.yaml b/...undled_models/fourcastnext/Training/limited_variables_early_stopping/trainer/default.yaml
@@ -18,5 +18,5 @@ checkpointing:
   mode: "max"
   dirpath: "{path}/Checkpoints/Epoch"
   filename: "model-{epoch:02d}"
-  save_on_train_epoch_end: True
+  save_on_train_epoch_end: true
   save_top_k: 50
diff --git a/...led_models/fourcastnext/Training/limited_variables_early_stopping/trainer/few_epochs.yaml b/...led_models/fourcastnext/Training/limited_variables_early_stopping/trainer/few_epochs.yaml
@@ -0,0 +1,23 @@
+---
+precision: 16-mixed
+max_epochs: 5
+
+checkpointing:
+- monitor: "train_loss"
+  mode: "min"
+  dirpath: "{path}/Checkpoints/Train"
+  filename: "model-{epoch:02d}-{step:02d}"
+  every_n_train_steps: 1000
+  save_top_k: 10
+- monitor: "valid_loss"
+  mode: "min"
+  dirpath: "{path}/Checkpoints/Valid"
+  filename: "model-{epoch:02d}-{step:02d}-{valid_loss}"
+  every_n_train_steps: 5000
+  save_top_k: 10
+- monitor: "epoch"
+  mode: "max"
+  dirpath: "{path}/Checkpoints/Epoch"
+  filename: "model-{epoch:02d}"
+  save_on_train_epoch_end: true
+  save_top_k: 50
diff --git a/packages/bundled_models/fourcastnext/Training/pipelines/low_res_demo_subset.pipe b/packages/bundled_models/fourcastnext/Training/pipelines/low_res_demo_subset.pipe
@@ -0,0 +1,46 @@
+!pyearthtools@pyearthtools.pipeline.controller.Pipeline
+__args: !!python/tuple
+- !pyearthtools@pyearthtools.data.archive.era5_demo_subset
+  level_value: null
+  product: reanalysis
+  transforms: null
+  variables:
+  - msl
+  - 10u
+  - 10v
+  - 2t
+- !pyearthtools@pyearthtools.pipeline.operations.transforms.Transforms
+  apply: !pyearthtools@pyearthtools.data.transforms.coordinates.StandardLongitude
+    type: -180-180
+  transforms: null
+  undo: null
+- !pyearthtools@fourcastnext.CropToRectangleSmall
+  warn: true
+- !pyearthtools@pyearthtools.pipeline.modifications.idx_modification.TemporalRetrieval
+  concat: true
+  delta_unit: null
+  merge_function: null
+  merge_kwargs:
+    axis: 1
+  samples: !!python/tuple
+  - !!python/tuple
+    - -6
+    - 1
+  - !!python/tuple
+    - 6
+    - 2
+    - 6
+- !pyearthtools@pyearthtools.pipeline.operations.xarray.conversion.ToNumpy
+  warn: true
+- !pyearthtools@pyearthtools.pipeline.operations.numpy.reshape.Rearrange
+  rearrange: c t h w -> t c h w
+  rearrange_kwargs: null
+  reverse_rearrange: null
+  skip: false
+exceptions_to_ignore: null
+iterator: null
+sampler: !pyearthtools@pyearthtools.pipeline.samplers.Default {}
+
+--CONFIG--
+VERSION: 1.0.1
+import: []
diff --git a/packages/bundled_models/fourcastnext/src/fourcastnext/__init__.py b/packages/bundled_models/fourcastnext/src/fourcastnext/__init__.py
@@ -48,3 +48,28 @@ def undo_func(self, dataset_to_undo, **kwargs):
 
         # Just return the cropped data, cannot 'undo' this one
         return dataset_to_undo
+
+
+class CropToRectangleSmall(Operation):
+    """Cut with Bounding box"""
+
+    def __init__(self, warn=True):
+        """
+        Default ERA5 is 721x1440. FourCastNeXt needs to be able to use 2x2 kernels, so needs an even number grid
+        dimension. For now, this class just disposes of the surplus pixels. In future the cropping strategy
+        from the paper could be implemented, or a complex regrid could be performed to resample to an even grid
+        """
+        super().__init__()
+        self.record_initialisation()
+
+    def apply_func(self, dataset: xr.Dataset):
+        subset_dataset = dataset.isel(
+            latitude=slice(0, -4),
+        )
+
+        return subset_dataset
+
+    def undo_func(self, dataset_to_undo, **kwargs):
+
+        # Just return the cropped data, cannot 'undo' this one
+        return dataset_to_undo
diff --git a/packages/bundled_models/fourcastnext/src/fourcastnext/configs/Data/low_res.pipe b/packages/bundled_models/fourcastnext/src/fourcastnext/configs/Data/low_res.pipe
@@ -0,0 +1,46 @@
+!pyearthtools@pyearthtools.pipeline.controller.Pipeline
+__args: !!python/tuple
+- !pyearthtools@pyearthtools.data.archive.era5_demo_subset
+  level_value: null
+  product: reanalysis
+  transforms: null
+  variables:
+  - msl
+  - 10u
+  - 10v
+  - 2t
+- !pyearthtools@pyearthtools.pipeline.operations.transforms.Transforms
+  apply: !pyearthtools@pyearthtools.data.transforms.coordinates.StandardLongitude
+    type: -180-180
+  transforms: null
+  undo: null
+- !pyearthtools@fourcastnext.CropToRectangleSmall
+  warn: true
+- !pyearthtools@pyearthtools.pipeline.modifications.idx_modification.TemporalRetrieval
+  concat: true
+  delta_unit: null
+  merge_function: null
+  merge_kwargs:
+    axis: 1
+  samples: !!python/tuple
+  - !!python/tuple
+    - -6
+    - 1
+  - !!python/tuple
+    - 6
+    - 2
+    - 6
+- !pyearthtools@pyearthtools.pipeline.operations.xarray.conversion.ToNumpy
+  warn: true
+- !pyearthtools@pyearthtools.pipeline.operations.numpy.reshape.Rearrange
+  rearrange: c t h w -> t c h w
+  rearrange_kwargs: null
+  reverse_rearrange: null
+  skip: false
+exceptions_to_ignore: null
+iterator: null
+sampler: !pyearthtools@pyearthtools.pipeline.samplers.Default {}
+
+--CONFIG--
+VERSION: 1.0.1
+import: []
diff --git a/packages/data/.yamllint.yml b/packages/data/.yamllint.yml
@@ -1,4 +1,4 @@
-
+---
 extends: default
 
 rules:

diff --git a/packages/data/src/pyearthtools/data/data.yaml b/packages/data/src/pyearthtools/data/data.yaml
@@ -6,24 +6,24 @@ data:                             # Data configuration
       chunks: 'auto'
       combine_attrs: 'drop_conflicts'
     xarray_mf:
-      parallel: False
+      parallel: false
 
 
   save:                           # Save files default kwargs
     xarray:
       engine: 'netcdf4'
     zarr:
-      compute: True
+      compute: true
 
   search_function: 'filesystem'  # Search function to use to find files
 
-  experimental: False            # Experimental feature toggle
+  experimental: false            # Experimental feature toggle
 
 
   series:
     warning_threshold: 5         # .series warning threshold
 
-  future_warning: True           # Show FutureWarning of pyearthtools's development
+  future_warning: true           # Show FutureWarning of pyearthtools's development
 
   patterns:                      # Pattern defaults
     default_extension: .nc
diff --git a/packages/data/src/pyearthtools/data/static/_geographic/Australia/Capital/download.yaml b/packages/data/src/pyearthtools/data/static/_geographic/Australia/Capital/download.yaml
@@ -1,4 +1,3 @@
-
 ---
 # Download Config For Capital Cities Data
 url: https://www.abs.gov.au/statistics/standards/australian-statistical-geography-standard-asgs-edition-3/jul2021-jun2026/access-and-downloads/digital-boundary-files/GCCSA_2021_AUST_SHP_GDA2020.zip

diff --git a/packages/data/src/pyearthtools/data/static/_geographic/Australia/States/download.yaml b/packages/data/src/pyearthtools/data/static/_geographic/Australia/States/download.yaml
@@ -1,4 +1,3 @@
-
 ---
 # Download Config For States Data
 url: https://www.abs.gov.au/statistics/standards/australian-statistical-geography-standard-asgs-pyearthtoolsion-3/jul2021-jun2026/access-and-downloads/digital-boundary-files/STE_2021_AUST_SHP_GDA2020.zip

diff --git a/packages/data/src/pyearthtools/data/static/_geographic/Australia/download.yaml b/packages/data/src/pyearthtools/data/static/_geographic/Australia/download.yaml
@@ -1,6 +1,5 @@
-
 ---
 # Download Config For Australia Data
 url: https://www.abs.gov.au/statistics/standards/australian-statistical-geography-standard-asgs-edition-3/jul2021-jun2026/access-and-downloads/digital-boundary-files/AUS_2021_AUST_SHP_GDA2020.zip
 save_name: Australia.zip
-zip: true
+zip: true
diff --git a/packages/data/src/pyearthtools/data/static/_geographic/World/download.yaml b/packages/data/src/pyearthtools/data/static/_geographic/World/download.yaml
@@ -1,4 +1,3 @@
-
 ---
 # Download Config For World Data
 url: https://public.opendatasoft.com/api/explore/v2.1/catalog/datasets/world-administrative-boundaries/exports/shp?lang=en&timezone=Australia%2FSydney

diff --git a/packages/data/src/pyearthtools/data/transforms/RegionLookup.yaml b/packages/data/src/pyearthtools/data/transforms/RegionLookup.yaml
@@ -11,7 +11,7 @@ Australia:
 
 Oceania: [-50, -5, 90, 180]
 
-Adelaide: [-39,-30, 131, 140]
+Adelaide: [-39, -30, 131, 140]
 Brisbane: [-30, -20, 145, 155]
 Sydney: [-38, -28, 145, 155]
 Melbourne: [-42, -32, 139, 151]

diff --git a/packages/nci_site_archive/mkdocs.yml b/packages/nci_site_archive/mkdocs.yml
@@ -57,7 +57,7 @@ plugins:
         docstring_style: "google"
         paths: [src]
         members_order: source
-        show_submodules: True
+        show_submodules: true
         options:
           allow_inspection: false
           show_source: false
-Original file line number
+Diff line change
@@ -1,4 +1,4 @@
+    ---
     extends: default
     rules:
@@ Expand Down @@