FIDUCEO · TomBlock · Feb 21, 2023 · Feb 22, 2023 · Mar 3, 2023 · Mar 3, 2023
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,6 @@
 target/
 /ingestion-tool/src/temp/
 temp.txt
+temp.java
+temp.xml
 /out/
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,4 +1,36 @@
+### Updates from version 1.5.8 to 1.6.0
+* Migrated to SNAP 11, several refactorings and library updates
+* The VariableCache now uses an LRU cache with a listener to automatically free the oldest cached, no longer
+  needed variables and file handles when the maximum cache size is reached.
+* fix wrong corner point order of era5 interpolation in SatelliteFields.  
+  Instead, the four era5 interpolation vertices are read directly from the variable
+* Remove reading of subset from entire era5 variable array and remove to read the entire era5 variable in VariableCache
+* Remove creation of rectangles for InterpolationContext and remove field era5Region in InterpolationContext
+* era5 post processing .. can now also handle satellite longitude data which not fits the range [-180 to 180].
+  In such cases (e.g. Windsat-Coriolis [0 to 360]), longitude data will be converted so that it fits into the
+  required range of [-180 to 180] to be able to create correct interpolation of era5 data.
+* era5 post processing .. In era5-post-processing-general-info.xml a fill value per
+  variable can be defined.
+* era5 post processing .. can now load generalized information from an optional
+  era5-post-processing-general-info.xml in the config directory.
+  A general info file can contain variable definition for satellite-fields. 
+* Upgrade Mockito to version 4.11.0 and also include artifact mockito-inline for static mocking
+* era5 post processing .. setter and getter of variable names in class SatelliteFieldsConfiguration was
+  generalized. E.g. config.set_an_ml_q_name("abcde") was replaced by config.setVarName("an_ml_q", "abcde")  
+* era5 post processing .. "length" attribute of tag <satellite-fields><z_dim name='up' length='???' />" is no
+  longer optional. 
+* era5 post processing .. The translation of variable name to variable name for file access has been made
+  switchable. 
+* post processing .. added PostProcessingContext to method PostProcessingPlugin.createPostProcessing(...).
+  Now plugins can perform extended initializations. E.g. by loading extra files from config directory. 
+* post processing .. the config directory has been added to the PostProcessingContext so that plugins can load 
+  additional generalized information required for the plugin from this directory.
+
 ### Updates from version 1.5.7 to 1.5.8
+* added support for SMAP Salinity L2C data  
+* added functionality to configure several global attributes before computing MMD files.
+* added support for NOAA TAO insitu data 
+* added support for NOAA NDBC insitu data
 * added support for Windsat Coriolis data
 * added conda environment
 * extended workflow to run without scheduler on plain console

diff --git a/CITATION.cff b/CITATION.cff
@@ -0,0 +1,37 @@
+cff-version: 1.2.0
+title: Multisensor Matchup System (MMS)
+message: >-
+  If you use this software, please cite it using the
+  metadata from this file.
+type: software
+authors:
+  - given-names: Thomas
+    family-names: Block
+    email: tom.block@brockmann-consult.de
+    affiliation: Brockmann Consult GmbH
+  - given-names: Sabine
+    family-names: Embacher
+    email: sabine.embacher@brockmann-consult.de
+    affiliation: Brockmann Consult GmbH
+identifiers:
+  - type: doi
+    value: 10.5281/zenodo.8116892
+repository-code: 'https://github.com/bcdev/MMS'
+url: 'https://www.brockmann-consult.de/'
+abstract: >-
+  The Multisensor matchup System (MMS) is a software suite
+  that allows to detect simulatneous measurements of remote
+  sensing (satellite) and in-situ instruments. 
+
+
+  Its main purpose is to support the validation of satellite
+  based instruments by generating cross-sensor matches of
+  either instruments from different satellite platforms or
+  between satellite and ground-based measurements.
+keywords:
+  - matchup
+  - calibration
+  - validation
+  - earth observation
+  - satellite
+license: GPL-3.0
diff --git a/LICENSE.txt b/LICENSE.txt
diff --git a/MMS_Manual_v_1_5.pdf b/MMS_Manual_v_1_5.pdf
diff --git a/MMS_Manual_v_1_6.pdf b/MMS_Manual_v_1_6.pdf
diff --git a/build_and_install_on_cems.sh b/build_and_install_on_cems.sh
@@ -6,5 +6,5 @@ mvn clean install package assembly:directory
 # echo "clean up bin dir"
 echo "copy build result to bin dir"
 rm -rf /gws/nopw/j04/fiduceo/Software/mms/bin/*
-cp -a target/fiduceo-master-1.5.8-MMS/* /gws/nopw/j04/fiduceo/Software/mms/bin
+cp -a target/fiduceo-master-1.6.2-MMS/* /gws/nopw/j04/fiduceo/Software/mms/bin
 
diff --git a/build_and_install_on_cems_sst.sh b/build_and_install_on_cems_sst.sh
@@ -6,5 +6,5 @@ mvn clean install package assembly:directory
 # echo "clean up bin dir"
 echo "copy build result to bin dir"
 rm -rf /gws/nopw/j04/esacci_sst/mms_new/bin/*
-cp -a target/fiduceo-master-1.5.8-MMS/* /gws/nopw/j04/esacci_sst/mms_new/bin
+cp -a target/fiduceo-master-1.6.2-MMS/* /gws/nopw/j04/esacci_sst/mms_new/bin
 
diff --git a/cems/pom.xml b/cems/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <artifactId>fiduceo-master</artifactId>
         <groupId>com.bc.fiduceo</groupId>
-        <version>1.5.8</version>
+        <version>1.6.2</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

diff --git a/cems/src/main/bin/mms-env.sh b/cems/src/main/bin/mms-env.sh
@@ -8,13 +8,14 @@
 
 # project and user settings
 # -------------------------
-export PROJECT=bc_fiduceo   # only LSF
-export MMS_USER=tblock01    # only SLURM
+export PROJECT=bc_fiduceo     # only LSF
+export MMS_USER=tblock01      # only SLURM
+export MMS_ACCOUNT=esacci_sst # only SLURM_2
 export MMS_ENV_NAME=mms-env.sh
 
 # Java and Python runtime definitions
 # -----------------------------------
-export MMS_JAVA_EXEC='/gws/nopw/j04/esacci_sst/mms_new/software/jdk1.8.0_202/bin/java'
+export MMS_JAVA_EXEC='/gws/nopw/j04/esacci_sst/mms_new/software/openjdk-11.0.26'
 
 export PM_EXE_DIR=/gws/nopw/j04/esacci_sst/mms_new/bin
 export PM_PYTHON_EXEC='/gws/nopw/j04/esacci_sst/mms_new/software/conda_envs/sst-cci-mms/bin/python'
@@ -23,7 +24,8 @@ export PATH=${PM_EXE_DIR}:$PATH
 
 # export scheduling engine
 # ------------------------
-export SCHEDULER='SLURM'
+export SCHEDULER='SLURM_2'
+# export SCHEDULER='SLURM'
 # export SCHEDULER='LSF'
 
 # ensure that processes exit
@@ -35,6 +37,9 @@ fi
 
 export PM_LOG_DIR=${WORKING_DIR}/log
 
+# --------------------------------------
+# --- LSF ------------------------------
+# --------------------------------------
 if [ "$SCHEDULER" == "LSF" ]; then
 
     submit_job() {
@@ -61,6 +66,9 @@ if [ "$SCHEDULER" == "LSF" ]; then
         fi
     }
 
+# --------------------------------------
+# --- SLURM ----------------------------
+# --------------------------------------
 elif [ "$SCHEDULER" == "SLURM" ]; then
 
     submit_job() {
@@ -87,6 +95,35 @@ elif [ "$SCHEDULER" == "SLURM" ]; then
             fi
         }
 
+# --------------------------------------
+# --- SLURM_2 --------------------------
+# --------------------------------------
+elif [ "$SCHEDULER" == "SLURM_2" ]; then
+
+submit_job() {
+            jobname=$1
+            command=$2
+
+            bsubmit="sbatch -A ${MMS_ACCOUNT} --mem=20000 -p standard -q short -n 1 -t 04:00:00 -o ${PM_LOG_DIR}/${jobname}.out -e ${PM_LOG_DIR}/${jobname}.err --job-name ${jobname} ${PM_EXE_DIR}/${command} ${@:3}"
+
+            rm -f ${PM_LOG_DIR}/${jobname}.out
+            rm -f ${PM_LOG_DIR}/${jobname}.err
+
+            # line contains the console output of the bsub command
+            line=`${bsubmit}`
+
+            if echo ${line} | grep -qF 'Submitted batch job'
+            then
+                # extract the job_id from the bsub message, concatenate '_' and jobname to form an identifier
+                # and dump to std_out to be fetched by pmonitor
+                job_id=`echo ${line} | awk '{ print substr($4,0,length($4)) }'`
+                echo "${job_id}_${jobname}"
+            else
+                echo "`date -u +%Y%m%d-%H%M%S` - submit of ${jobname} failed: ${line}"
+                exit 1
+            fi
+        }
+
 else
     echo "Invalid scheduler"
     exit 1

diff --git a/cems/src/main/bin/post_processing_run.sh b/cems/src/main/bin/post_processing_run.sh
@@ -8,6 +8,6 @@ end_date=$3
 job_config=$4
 config_dir=$5
 
-echo "`date -u +%Y%m%d-%H%M%S` ingestion ${start_date} - ${end_date} ..."
+echo "`date -u +%Y%m%d-%H%M%S` post_processing ${start_date} - ${end_date} ..."
 
 ${PM_EXE_DIR}/post-processing-tool.sh -i ${input_dir} -start ${start_date} -end ${end_date} -j ${job_config} -c ${config_dir}
diff --git a/cems/src/main/python/jasmin/jasmin_job_monitor.py b/cems/src/main/python/jasmin/jasmin_job_monitor.py
@@ -48,8 +48,10 @@ def _create_scheduler_interface():
                 return  LSFInterface()
             elif scheduler_name == "SLURM":
                 return SLURMInterface()
+            elif scheduler_name == "SLURM_2":
+                return SLURMInterface()
             else:
-                raise ValueError("Environment variable 'SCHEDULER' invalid")
+                raise ValueError("Environment variable 'SCHEDULER' invalid" + scheduler_name)
         else:
             raise ValueError("Environment variable 'SCHEDULER' is not set")
 
@@ -188,10 +190,13 @@ def resolve_status_from_log(self, check_log_dict):
 class SLURMInterface:
 
     user_name = None
+    scheduler_name = None
 
     def __init__(self):
         if "MMS_USER" in os.environ:
             self.user_name = os.environ["MMS_USER"]
+            if "SCHEDULER" in os.environ:
+                self.scheduler_name = os.environ["SCHEDULER"]
         else:
             raise RuntimeError("Missing environment variable 'MMS_USER'")
 
@@ -224,7 +229,12 @@ def _extract_id_and_status(self, line):
         if len(tokens) < 7:
             raise ValueError("unable to handle 'squeue' result: " + line)
 
-        status_code = self._status_to_enum(tokens[4])
+        if self.scheduler_name == "SLURM_2":
+            status_token = tokens[7]
+        else:
+            status_token = tokens[4]
+
+        status_code = self._status_to_enum(status_token)
         return {tokens[0]: status_code}
 
     @staticmethod

diff --git a/cems/src/main/python/jasmin/jasmin_job_monitor_test.py b/cems/src/main/python/jasmin/jasmin_job_monitor_test.py
@@ -2,8 +2,8 @@
 import io
 import unittest
 
-from jasmin.jasmin_job_monitor import JasminJobMonitor, LSFInterface, SLURMInterface
-from jasmin.status_codes import StatusCodes
+from jasmin_job_monitor import JasminJobMonitor, LSFInterface, SLURMInterface
+from status_codes import StatusCodes
 
 
 class JasminJobMonitorTest(unittest.TestCase):
@@ -62,6 +62,25 @@ def test_parse_job_status_SLURM(self):
         finally:
             del os.environ["MMS_USER"]
 
+    def test_parse_job_status_SLURM_2(self):
+        output = "   JOBID PARTIT       QOS                 NAME       USER NODE  CPUS ST         TIME    TIME_LEFT PRIORITY NODELIST(REASON)\n" \
+                 "8466477 standa     short ingest-slstr-s3a-uor   tblock01    1     1  R         0:37      3:59:23    77911 (NonZeroExitCode)\n" \
+               "8466478 standa     short ingest-slstr-s3a-uor   tblock01    1     1  F         0:37      3:59:23    77911 (NonZeroExitCode)"
+
+
+        try:
+            os.environ["MMS_USER"] = "HarryPotter"
+            os.environ["SCHEDULER"] = "SLURM_2"
+
+            scheduler = SLURMInterface()
+            job_status_dict = scheduler.parse_jobs_call(output)
+            self.assertEqual(2, len(job_status_dict))
+            self.assertEqual(StatusCodes.RUNNING, job_status_dict["8466477"])
+            self.assertEqual(StatusCodes.FAILED, job_status_dict["8466478"])
+        finally:
+            del os.environ["MMS_USER"]
+            del os.environ["SCHEDULER"]
+
     def test_status_to_enum_LSF(self):
         scheduler = LSFInterface()
 

diff --git a/cems/src/main/python/workflow.py b/cems/src/main/python/workflow.py
@@ -408,16 +408,23 @@ def run_matchup(self, hosts, num_parallel_tasks, simulation=False, logdir='trace
 
         monitor.wait_for_completion()
 
-    def run_post_processing(self, hosts, num_parallel_tasks, simulation=False, logdir='trace'):
+    def run_post_processing(self, hosts, num_parallel_tasks, simulation=False, logdir='trace', synchronous=False):
         """
-
         :param hosts: list
         :param num_parallel_tasks: int
         :param simulation: bool
         :param logdir: str
+        :param synchronous: bool:
+                decides whether post_processing_run.sh (True) or post_processing_start.sh (False) is called.
         :return:
         """
-        monitor = self._get_monitor(hosts, [('post_processing_start.sh', num_parallel_tasks)], logdir, simulation)
+
+        if synchronous:
+            runs_script = 'post_processing_run.sh'
+        else:
+            runs_script = 'post_processing_start.sh'
+
+        monitor = self._get_monitor(hosts, [(runs_script, num_parallel_tasks)], logdir, simulation, synchronous)
         production_period = self.get_production_period()
         date = production_period.get_start_date()
         while date < production_period.get_end_date():
@@ -430,7 +437,7 @@ def run_post_processing(self, hosts, num_parallel_tasks, simulation=False, logdi
             pre_condition = 'mmd-' + start_string + '-' + end_string
             post_condition = 'post-processing-' + start_string + '-' + end_string + '-' + self.usecase_config
 
-            job = Job(job_name, 'post_processing_start.sh', [pre_condition], [post_condition],
+            job = Job(job_name, runs_script, [pre_condition], [post_condition],
                       [self.input_dir, start_string, end_string, self.usecase_config, self._get_config_dir()])
             monitor.execute(job)
 

diff --git a/core/pom.xml b/core/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>fiduceo-master</artifactId>
         <groupId>com.bc.fiduceo</groupId>
-        <version>1.5.8</version>
+        <version>1.6.2</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -39,13 +39,13 @@
         </dependency>
 
         <dependency>
-            <groupId>org.esa.s3tbx</groupId>
-            <artifactId>s3tbx-avhrr-reader</artifactId>
+            <groupId>eu.esa.opt</groupId>
+            <artifactId>opttbx-avhrr-reader</artifactId>
         </dependency>
 
         <dependency>
-            <groupId>org.esa.s3tbx</groupId>
-            <artifactId>s3tbx-sentinel3-reader</artifactId>
+            <groupId>eu.esa.opt</groupId>
+            <artifactId>opttbx-sentinel3-reader</artifactId>
         </dependency>
 
         <dependency>
@@ -89,14 +89,10 @@
             <artifactId>netcdfAll</artifactId>
         </dependency>
 
-        <!--<dependency>-->
-            <!--<groupId>edu.ucar</groupId>-->
-            <!--<artifactId>netcdf4</artifactId>-->
-        <!--</dependency>-->
-        <!--<dependency>-->
-        <!--<groupId>edu.ucar</groupId>-->
-        <!--<artifactId>cdm</artifactId>-->
-        <!--</dependency>-->
+        <dependency>
+            <groupId>org.jdom</groupId>
+            <artifactId>jdom2</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>junit</groupId>
@@ -110,14 +106,20 @@
         </dependency>
         <dependency>
             <groupId>org.hamcrest</groupId>
-            <artifactId>hamcrest-all</artifactId>
+            <artifactId>hamcrest</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>com.google.jimfs</groupId>
             <artifactId>jimfs</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+            <version>1.10.0</version>
+            <scope>test</scope>
+        </dependency>
 
     </dependencies>