diff --git a/.gitignore b/.gitignore index 7e3b24e..d1f3dab 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,8 @@ +# logs +ctl/logs +# run directories run/ +# data directories +dta/restart +dta/simres +dta/forcing diff --git a/.gitmodules b/.gitmodules index 74afad5..fb97e80 100644 --- a/.gitmodules +++ b/.gitmodules @@ -34,3 +34,6 @@ path = dta/geo/parflow url = https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/tsmp2-static-files/extpar_parflow_cordex-eur-11u.git branch = master +[submodule "src/eCLM_atmforcing"] + path = src/eCLM_atmforcing + url = https://github.com/HPSCTerrSys/eCLM_static-file-generator.git diff --git a/ctl/control_tsmp2.sh b/ctl/control_tsmp2.sh index a7ceef3..04ae9f0 100755 --- a/ctl/control_tsmp2.sh +++ b/ctl/control_tsmp2.sh @@ -44,7 +44,7 @@ partition="" # compute partition account=$BUDGET_ACCOUNTS # SET compute account. If not set, slts is used # wallclock -pre_wallclock=00:05:00 +pre_wallclock=00:35:00 sim_wallclock=00:25:00 # needs to be format hh:mm:ss pos_wallclock=00:05:00 vis_wallclock=00:05:00 @@ -155,6 +155,8 @@ datep1=$(date -u -d -I "+${startdate} + ${simlength}") datem1=$(date -u -d -I "+${startdate} - ${simlength}") simlensec=$(( $(date -u -d "${datep1}" +%s)-$(date -u -d "${startdate}" +%s) )) simlenhr=$(($simlensec/3600 | bc -l)) +simlenmon=$(( (10#$(date -u -d "${datep1}" +%Y)-10#$(date -u -d "${startdate}" +%Y))*12 + \ + 10#$(date -u -d "${datep1}" +%m)-10#$(date -u -d "${startdate}" +%m) )) dateymd=$(date -u -d "${startdate}" +%Y%m%d) dateshort=$(date -u -d "${startdate}" +%Y%m%d%H%M%S) @@ -183,7 +185,18 @@ jobprestring="${jobgenstring} \ --ntasks=${npnode}" # Submit to pre.job -submit_pre=$(sbatch ${jobprestring} ${ctl_dir}/pre_ctl/pre.job 2>&1) +if (! ${debugmode}) ; then + # Submit to sim.job + submit_pre=$(sbatch ${jobprestring} ${ctl_dir}/pre_ctl/pre.job 2>&1) + echo $submit_pre" for preprocessing" +else + # Set lpre run & cleanup to false and source pre.job + lpre[1]=false + lpre[2]=false + lprestr="${lpre[@]}" + source ${ctl_dir}/pre_ctl/pre.job +fi + echo $submit_pre" for preprocessing" # get jobid diff --git a/ctl/pre_ctl/pre.job b/ctl/pre_ctl/pre.job index 0c13dfd..2a27490 100755 --- a/ctl/pre_ctl/pre.job +++ b/ctl/pre_ctl/pre.job @@ -18,7 +18,7 @@ IFS=' ' read -r -a lvis <<< "$lvisstr" ### # source environment ### -source ${tsmp2_env} # should be changed to pre-processing env. +#source ${tsmp2_env} # chang pre-processing env. according to comp. model ### # Start config, run, cleanup diff --git a/ctl/pre_ctl/pre_cleanup.sh b/ctl/pre_ctl/pre_cleanup.sh index e388dc7..fc4ecdc 100755 --- a/ctl/pre_ctl/pre_cleanup.sh +++ b/ctl/pre_ctl/pre_cleanup.sh @@ -6,5 +6,26 @@ pre_cleanup(){ echo "Start Pre-processing cleanup" -} # pre_cleanup +#################### +# CLM +#################### +if [[ "${modelid}" == *clm* ]]; then + +echo "clean-up clm forcing" + +for idate in ${listfrcfile[@]}; do + ifile=${pre_dir}/${idate}/${idate}.nc + cp ${ifile} ${eclmfrc_dir} +done +unset ifile + +fi +# move logs +mkdir -p ${eclmfrc_dir}/log/ +mv ${log_dir}/${SLURM_JOB_NAME}_${SLURM_JOB_ID}.{err,out} ${eclmfrc_dir}/log/. + +# remove working directory +rm -rf ${pre_dir:?} + +} # pre_cleanup diff --git a/ctl/pre_ctl/pre_config.sh b/ctl/pre_ctl/pre_config.sh index b6cdeff..d823a09 100755 --- a/ctl/pre_ctl/pre_config.sh +++ b/ctl/pre_ctl/pre_config.sh @@ -6,5 +6,72 @@ pre_config(){ echo "Start Pre-processing Config" +#################### +# General +#################### + +# create a new simulation run directory +echo "pre_dir: "$pre_dir +if [ -e "${pre_dir}" ]; then + mv ${pre_dir} ${pre_dir}_bku$(date '+%Y%m%d%H%M%S') +fi +mkdir -p $pre_dir + +# change to run directory +cd ${pre_dir} + +#################### +# CLM +#################### +if [[ "${modelid}" == *clm* ]]; then + +echo "start create clm forcing" + +# directories +lsmforcgensrc_dir=${ctl_dir}/../src/eCLM_atmforcing/mkforcing +eclmfrc_dir=${frc_dir}/eclm/forcing/ +cdsapi_dtadir=${ctl_dir}/../src/eCLM_atmforcing/mkforcing/cdsapidwn + +# check if forcing files already exists +unset listfrcfile +if [ ! -e "${eclmfrc_dir}/$(date -u -d "${startdate}" +%Y-%m).nc" ]; then + listfrcfile+=("$(date -u -d "${startdate}" +%Y-%m)") +fi +if [[ "${simlenmon}" -ge 1 ]]; then + for imon in $(seq 1 $simlenmon);do + dateloop=$(date -u -d "${startdate} +${imon} month" +%Y-%m) + if [ ! -e "${eclmfrc_dir}/${dateloop}.nc" ]; then + listfrcfile+=("${dateloop}") + fi + done +fi + +echo "List of forcing files: "${listfrcfile[@]} + +fi + + +if ${debugmode}; then + +# create job submission script (pre.job) +echo "#!/usr/bin/env bash" > pre.job +echo "#SBATCH ${jobprestring//[$'\t\r\n']}" >> pre.job + +# add modelid, which is needed +echo "" >> pre.job +echo "modelid=${modelid}" >> pre.job +echo "lsmforcgensrc_dir=${lsmforcgensrc_dir}" >> pre.job +echo "cdsapi_dtadir=${cdsapi_dtadir}" >> pre.job +echo "listfrcfile=(${listfrcfile[*]})" >> pre.job + +# cat pre run script into submission script +cat ${ctl_dir}/pre_ctl/pre_run.sh | tail -n +2 >> pre.job # start from line 2 + +sed -i "s/pre_run(){//" pre.job +sed -i "s/} # pre_run//" pre.job +sed -i "s#${log_dir}#${pre_dir}#g" pre.job + +fi # debugmode + } # pre_config diff --git a/ctl/pre_ctl/pre_run.sh b/ctl/pre_ctl/pre_run.sh index b182fc9..4351fa1 100755 --- a/ctl/pre_ctl/pre_run.sh +++ b/ctl/pre_ctl/pre_run.sh @@ -6,5 +6,42 @@ pre_run(){ echo "Start Pre-processing run" +#################### +# CLM +#################### +if [[ "${modelid}" == *clm* ]]; then + +echo "Start processing clm" + +module use $OTHERSTAGES +module load Stages/2022 NVHPC/22.9 ParaStationMPI/5.5.0-1 CDO/2.0.2 NCO + + +# list contain dates needed for forcing +for yyyymm in "${listfrcfile[@]}"; do + year="${yyyymm%%-*}" + month="${yyyymm#*-}" + mkdir -p $yyyymm # temporary directory for scripts + + # extract data from ERA5 meteocloud + srun --exclusive -n 1 ${lsmforcgensrc_dir}/extract_ERA5_meteocloud.sh iyear=$year imonth=$month \ + outdir=${pre_dir}/${yyyymm} #quiet=y + +# # needs to be done in advance +# # download from CDSAPI needs +# python ${lsmforcgensrc_dir}/download_ERA5_input.py $year $month ${pre_dir}/${yyyymm} + + # link cdsapi data for preparation script + ln -s ${cdsapi_dtadir}/download_era5_${year}_$month.zip ${yyyymm}/ + + # preparation script + srun --exclusive -n 1 ${lsmforcgensrc_dir}/prepare_ERA5_input.sh pathdata=${pre_dir}/${yyyymm} \ + iyear=$year imonth=$month wrkdir=${yyyymm} + +done +unset yyyymm year month + +fi + } # pre_run diff --git a/src/eCLM_atmforcing b/src/eCLM_atmforcing new file mode 160000 index 0000000..858634d --- /dev/null +++ b/src/eCLM_atmforcing @@ -0,0 +1 @@ +Subproject commit 858634df52d9a224099b622965934674bdd248cb