eCLM preprocessing (#16)

* add eclm prepro

* update .gitignore

* add debugmode for pre

* not loading enviroment in pre.job

* add eCLM_static-file-generator as submodule

Author: s-poll

.gitignore

@@ -1 +1,8 @@
+# logs
+ctl/logs
+# run directories
 run/
+# data directories
+dta/restart
+dta/simres
+dta/forcing

.gitmodules

@@ -34,3 +34,6 @@
 	path = dta/geo/parflow
 	url = https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/tsmp2-static-files/extpar_parflow_cordex-eur-11u.git
 	branch = master
+[submodule "src/eCLM_atmforcing"]
+	path = src/eCLM_atmforcing
+	url = https://github.com/HPSCTerrSys/eCLM_static-file-generator.git

ctl/control_tsmp2.sh

@@ -44,7 +44,7 @@ partition="" # compute partition
 account="" # SET compute account. $BUDGET_ACCOUNTS / slts is used, if not set.
 
 # wallclock
-pre_wallclock=00:05:00
+pre_wallclock=00:35:00
 sim_wallclock=00:25:00 # needs to be format hh:mm:ss
 pos_wallclock=00:05:00
 vis_wallclock=00:05:00
@@ -140,6 +140,8 @@ datep1=$(date -u -d -I "+${startdate} + ${simlength}")
 datem1=$(date -u -d -I "+${startdate} - ${simlength}")
 simlensec=$(( $(date -u -d "${datep1}" +%s)-$(date -u -d "${startdate}" +%s) ))
 simlenhr=$(($simlensec/3600 | bc -l))
+simlenmon=$(( (10#$(date -u -d "${datep1}" +%Y)-10#$(date -u -d "${startdate}" +%Y))*12 + \
+               10#$(date -u -d "${datep1}" +%m)-10#$(date -u -d "${startdate}" +%m) ))
 dateymd=$(date -u -d "${startdate}" +%Y%m%d)
 dateshort=$(date -u -d "${startdate}" +%Y%m%d%H%M%S)
 
@@ -168,7 +170,18 @@ jobprestring="${jobgenstring} \
               --ntasks=${npnode}"
 
 # Submit to pre.job
-submit_pre=$(sbatch ${jobprestring} ${ctl_dir}/pre_ctl/pre.job 2>&1)
+if (! ${debugmode}) ; then
+  # Submit to sim.job
+  submit_pre=$(sbatch ${jobprestring} ${ctl_dir}/pre_ctl/pre.job 2>&1)
+  echo $submit_pre" for preprocessing"
+else
+  # Set lpre run & cleanup to false and source pre.job
+  lpre[1]=false
+  lpre[2]=false
+  lprestr="${lpre[@]}"
+  source ${ctl_dir}/pre_ctl/pre.job
+fi
+
 echo $submit_pre" for preprocessing"
 
 # get jobid

ctl/pre_ctl/pre.job

@@ -18,7 +18,7 @@ IFS=' ' read -r -a lvis <<< "$lvisstr"
 ###
 # source environment
 ###
-source ${tsmp2_env} # should be changed to pre-processing env.
+#source ${tsmp2_env} # chang pre-processing env. according to comp. model
 
 ###
 # Start config, run, cleanup

ctl/pre_ctl/pre_cleanup.sh

@@ -6,5 +6,26 @@ pre_cleanup(){
 
 echo "Start Pre-processing cleanup"
 
-} # pre_cleanup
+####################
+# CLM
+####################
+if [[ "${modelid}" == *clm* ]]; then
+
+echo "clean-up clm forcing"
+
+for idate in ${listfrcfile[@]}; do
+  ifile=${pre_dir}/${idate}/${idate}.nc
+  cp ${ifile} ${eclmfrc_dir}
+done
+unset ifile
+
+fi
 
+# move logs
+mkdir -p ${eclmfrc_dir}/log/
+mv ${log_dir}/${SLURM_JOB_NAME}_${SLURM_JOB_ID}.{err,out} ${eclmfrc_dir}/log/.
+
+# remove working directory
+rm -rf ${pre_dir:?}
+
+} # pre_cleanup

ctl/pre_ctl/pre_config.sh

@@ -6,5 +6,72 @@ pre_config(){
 
 echo "Start Pre-processing Config"
 
+####################
+# General
+####################
+
+# create a new simulation run directory
+echo "pre_dir: "$pre_dir
+if [ -e "${pre_dir}" ]; then
+  mv ${pre_dir} ${pre_dir}_bku$(date '+%Y%m%d%H%M%S')
+fi
+mkdir -p $pre_dir
+
+# change to run directory
+cd ${pre_dir}
+
+####################
+# CLM
+####################
+if [[ "${modelid}" == *clm* ]]; then
+
+echo "start create clm forcing"
+
+# directories
+lsmforcgensrc_dir=${ctl_dir}/../src/eCLM_atmforcing/mkforcing
+eclmfrc_dir=${frc_dir}/eclm/forcing/
+cdsapi_dtadir=${ctl_dir}/../src/eCLM_atmforcing/mkforcing/cdsapidwn
+
+# check if forcing files already exists
+unset listfrcfile
+if [ ! -e "${eclmfrc_dir}/$(date -u -d "${startdate}" +%Y-%m).nc" ]; then
+   listfrcfile+=("$(date -u -d "${startdate}" +%Y-%m)")
+fi
+if [[ "${simlenmon}" -ge 1 ]]; then
+   for imon in $(seq 1 $simlenmon);do
+      dateloop=$(date -u -d "${startdate} +${imon} month" +%Y-%m)
+      if [ ! -e "${eclmfrc_dir}/${dateloop}.nc" ]; then
+	  listfrcfile+=("${dateloop}")
+      fi
+   done
+fi
+
+echo "List of forcing files: "${listfrcfile[@]}
+
+fi
+
+
+if ${debugmode}; then
+
+# create job submission script (pre.job)
+echo "#!/usr/bin/env bash" > pre.job
+echo "#SBATCH ${jobprestring//[$'\t\r\n']}" >> pre.job
+
+# add modelid, which is needed
+echo "" >> pre.job
+echo "modelid=${modelid}" >> pre.job
+echo "lsmforcgensrc_dir=${lsmforcgensrc_dir}" >> pre.job
+echo "cdsapi_dtadir=${cdsapi_dtadir}" >> pre.job
+echo "listfrcfile=(${listfrcfile[*]})" >> pre.job
+
+# cat pre run script into submission script
+cat ${ctl_dir}/pre_ctl/pre_run.sh | tail -n +2 >> pre.job # start from line 2
+
+sed -i "s/pre_run(){//" pre.job
+sed -i "s/} # pre_run//" pre.job
+sed -i "s#${log_dir}#${pre_dir}#g" pre.job
+
+fi # debugmode
+
 } # pre_config
 

ctl/pre_ctl/pre_run.sh

@@ -6,5 +6,42 @@ pre_run(){
 
 echo "Start Pre-processing run"
 
+####################
+# CLM
+####################
+if [[ "${modelid}" == *clm* ]]; then
+
+echo "Start processing clm"
+
+module use $OTHERSTAGES
+module load Stages/2022  NVHPC/22.9  ParaStationMPI/5.5.0-1 CDO/2.0.2 NCO
+
+
+# list contain dates needed for forcing
+for yyyymm in "${listfrcfile[@]}"; do
+   year="${yyyymm%%-*}"
+   month="${yyyymm#*-}"
+   mkdir -p $yyyymm # temporary directory for scripts
+
+   # extract data from ERA5 meteocloud
+   srun --exclusive -n 1 ${lsmforcgensrc_dir}/extract_ERA5_meteocloud.sh iyear=$year imonth=$month \
+        outdir=${pre_dir}/${yyyymm} #quiet=y
+
+#  # needs to be done in advance
+#   # download from CDSAPI needs
+#   python ${lsmforcgensrc_dir}/download_ERA5_input.py $year $month ${pre_dir}/${yyyymm}
+
+   # link cdsapi data for preparation script
+   ln -s ${cdsapi_dtadir}/download_era5_${year}_$month.zip ${yyyymm}/
+
+   # preparation script
+   srun --exclusive -n 1 ${lsmforcgensrc_dir}/prepare_ERA5_input.sh pathdata=${pre_dir}/${yyyymm} \
+	                  iyear=$year imonth=$month wrkdir=${yyyymm}
+
+done
+unset yyyymm year month
+
+fi
+
 } # pre_run