NVHPC toolchain support

.gitmodules

@@ -17,7 +17,7 @@
 [submodule "models/eCLM"]
 	path = models/eCLM
 	url = https://github.com/HPSCTerrSys/eCLM.git
-	branch = beta-0.4
+	branch = dev-nvhpc-support
 [submodule "models/oasis3-mct"]
         path = models/oasis3-mct
         url = https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct

cmake/BuildOASIS3MCT.cmake

@@ -44,6 +44,9 @@ if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
   file(APPEND  ${OASIS_MAKE_INC} "FCBASEFLAGS     = ${OPTIM} -I. -ffree-line-length-none -fallow-argument-mismatch ${OpenMP_Fortran_FLAGS}\n")
 elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM")
   file(APPEND  ${OASIS_MAKE_INC} "FCBASEFLAGS     = ${OPTIM} -I. -xCORE-AVX2 -assume byterecl -mt_mpi ${OpenMP_Fortran_FLAGS}\n")
+elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
+  # nvfortran doesn't support real(16) / quad precision; hence the __NO_16BYTE_REALS here.
+  file(APPEND  ${OASIS_MAKE_INC} "FCBASEFLAGS     = -D__NO_16BYTE_REALS ${OPTIM} -I. ${OpenMP_Fortran_FLAGS}\n")
 else()
   message(FATAL_ERROR "Fortran compiler '${CMAKE_Fortran_COMPILER_ID}' is not supported.")
 endif()
@@ -68,7 +71,7 @@ ExternalProject_Add(OASIS3_MCT
   SOURCE_DIR        ${OASIS_SRC}
   BUILD_IN_SOURCE   FALSE
   CONFIGURE_COMMAND ""
-  BUILD_COMMAND     make -f ${OASIS_SRC}/util/make_dir/TopMakefileOasis3 static-libs -C ${OASIS_BLD_DIR}
+  BUILD_COMMAND     make -f ${OASIS_SRC}/util/make_dir/TopMakefileOasis3 realclean static-libs -C ${OASIS_BLD_DIR}
   INSTALL_COMMAND   ""
 )
 

cmake/BuildPDAF.cmake

@@ -24,6 +24,7 @@ endif()
 # LAPACK is required
 # For eCLM-PDAF, this setting has to be consistent with MKL/LAPACK
 # loading in `eCLM/src/clm5/CMakelists.txt`
+# https://cmake.org/cmake/help/latest/module/FindLAPACK.html
 find_package(LAPACK REQUIRED)
 
 # OpenMP is required
@@ -73,6 +74,12 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   list(APPEND PDAF_LINK_LIBS "-mkl")
   list(APPEND PDAF_LINK_LIBS "${LAPACK_LIBRARIES}")
   message(WARNING "LAPACK_LIBRARIES: ${LAPACK_LIBRARIES}")
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC")
+  # MKL command
+  list(APPEND PDAF_LINK_LIBS "${LAPACK_LIBRARIES}")
+  list(APPEND PDAF_LINK_LIBS "${LAPACK_LINKER_FLAGS}")
+  message(STATUS "LAPACK_LIBRARIES: ${LAPACK_LIBRARIES}")
+  message(STATUS "LAPACK_LINKER_FLAGS: ${LAPACK_LINKER_FLAGS}")
 else()
   message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
 endif()
@@ -147,6 +154,22 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   list(APPEND PDAF_FOPT "-fallow-argument-mismatch")
   list(APPEND PDAF_FOPT "-fcommon")
 
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC")
+
+  # Using NVHPC
+  if (CMAKE_BUILD_TYPE STREQUAL "RELEASE")
+    # Release optimization flags
+    list(APPEND PDAF_FOPT "-Ofast")
+  elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+    # Debug optimization flags
+    list(APPEND PDAF_FOPT "-O0")
+    list(APPEND PDAF_FOPT "-g")
+  else()
+    message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
+  endif()
+
+  list(APPEND PDAF_FOPT "-fPIC")
+
 else()
   message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
 endif()
@@ -195,6 +218,22 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   list(APPEND PDAF_COPT "-mcmodel=large")
   list(APPEND PDAF_COPT "-fcommon")
 
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC")
+
+  # Using NVHPC
+  if (CMAKE_BUILD_TYPE STREQUAL "RELEASE")
+    # Release optimization flags
+    list(APPEND PDAF_COPT "-Ofast")
+  elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+    # Debug optimization flags
+    list(APPEND PDAF_COPT "-O0")
+    list(APPEND PDAF_COPT "-g")
+  else()
+    message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
+  endif()
+
+  list(APPEND PDAF_COPT "-fPIC")
+
 else()
   message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
 endif()
@@ -213,6 +252,11 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
 
   list(APPEND PDAF_DOUBLEPRECISION "-fdefault-real-8")
 
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC")
+
+  # doubleprecision flag!?
+  list(APPEND PDAF_DOUBLEPRECISION "-r8")
+
 else()
   message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
 endif()

cmake/BuildParFlow.cmake

@@ -35,7 +35,6 @@ else()
   else()
     set(PF_ACC_BACKEND "none") 
   endif()
-  #TODO: Add support for 'kokkos' backend
 endif()
 
 # Set compiler flags
@@ -48,7 +47,12 @@ if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
     endif()
     set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none")
 elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM")
-    set(PF_CFLAGS "-Wall -Werror -Wno-unused-function -Wno-unused-variable")
+elseif(CMAKE_C_COMPILER_ID STREQUAL "NVHPC")
+    if (NOT ${ParFlowGPU})
+      # TODO: Perhaps there's a case for using NVHPC to target CPU. This would require
+      #       fiddling with libraries+environment so we don't support it for now.
+      message(FATAL_ERROR "NVHPC is only valid for ParflowGPU builds.")
+    endif()
 else()
   message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.")
 endif()

env/jsc.2025.intel.psmpi

@@ -22,6 +22,8 @@ else
   module load Intel
 fi
 module load ParaStationMPI
+module load ScaLAPACK/2.2.0-fb
+
 
 # Basic scripting and build tools
 module load Python

env/jsc.2025.nvhpc.openmpi

@@ -0,0 +1,65 @@
+# --------------------------------------------------------------------------
+# Loads IntelLLVM+ParaStationMPI build environment for TSMP2.
+# This environment is tailored for JURECA [1] and JUWELS [2] supercomputers.
+#
+# [1] https://apps.fz-juelich.de/jsc/software/jureca/index.xhtml
+# [2] https://apps.fz-juelich.de/jsc/software/juwels/index.xhtml
+# 
+# Usage: source jsc.2025.intel.psmpi
+# --------------------------------------------------------------------------
+
+# Load Stages/2025
+module --force purge
+module use $OTHERSTAGES
+module load Stages/2025
+module load NVHPC/25.5-CUDA-12
+module load OpenMPI
+module load ScaLAPACK/2.2.0-fb
+
+# Basic scripting and build tools
+module load Python
+module load CMake
+module load git
+
+# Storage libraries
+module load HDF5
+module load netCDF
+module load netCDF-Fortran
+module load PnetCDF
+
+# ParFlow additional libraries
+module load CUDA
+module load UCX-settings/RC-CUDA
+module load Hypre/2.31.0
+
+# TODO: Verify these values
+if [[ $SYSTEMNAME == "jedi" || $SYSTEMNAME == "jupiter" ]]; then
+  export CUDAARCHS="90"
+else
+  export CUDAARCHS="80"
+fi
+export CMAKE_CUDA_RUNTIME_LIBRARY="Shared"
+
+
+# ICON additional libraries
+module load ecCodes
+
+# Set default MPI compilers
+export OMPI_CC=nvc
+export OMPI_CXX=nvc++
+export OMPI_FC=nvfortran
+export CC=mpicc
+export FC=mpif90
+export CXX=mpicxx
+export MPI_HOME=$EBROOTOPENMPI
+
+# Display compiler settings
+module list
+echo "=========================== COMPILER SETTINGS  =======================" 
+echo "   Machine: ${SYSTEMNAME} on Stages/$STAGE"
+echo "   MPI lib: $(mpirun --version | head -n 1)"
+echo "         C: $($CC --version | head -n 2 | tail -n 1)"
+echo "       C++: $($CXX --version | head -n 2 | tail -n 1)"
+echo "   Fortran: $($FC --version | head -n 2 | tail -n 1)"
+echo "======================================================================"
+

env/jsc.2025.nvhpc.psmpi

@@ -0,0 +1,61 @@
+# --------------------------------------------------------------------------
+# Loads IntelLLVM+ParaStationMPI build environment for TSMP2.
+# This environment is tailored for JURECA [1] and JUWELS [2] supercomputers.
+#
+# [1] https://apps.fz-juelich.de/jsc/software/jureca/index.xhtml
+# [2] https://apps.fz-juelich.de/jsc/software/juwels/index.xhtml
+# 
+# Usage: source jsc.2025.intel.psmpi
+# --------------------------------------------------------------------------
+
+# Load Stages/2025
+module --force purge
+module use $OTHERSTAGES
+module load Stages/2025
+module load NVHPC/25.5-CUDA-12
+module load ParaStationMPI
+module load ScaLAPACK/2.2.0-fb
+
+# Basic scripting and build tools
+module load Python
+module load CMake
+module load git
+
+# Storage libraries
+module load HDF5
+module load netCDF
+module load netCDF-Fortran
+module load PnetCDF
+
+# ParFlow additional libraries (TODO)
+module load CUDA
+module load UCX-settings/RC-CUDA
+module load Hypre/2.31.0
+
+# TODO: Verify these values
+if [[ $SYSTEMNAME == "jedi" || $SYSTEMNAME == "jupiter" ]]; then
+  export CUDAARCHS="90"
+else
+  export CUDAARCHS="80"
+fi
+export CMAKE_CUDA_RUNTIME_LIBRARY="Shared"
+
+# ICON additional libraries
+module load ecCodes
+
+# Set default MPI compilers
+export CC=mpicc
+export FC=mpif90
+export CXX=mpicxx
+export MPI_HOME=$EBROOTPSMPI
+
+# Display compiler settings
+module list
+echo "========================= COMPILER SETTINGS =========================="
+echo "   Machine: ${SYSTEMNAME} on Stages/$STAGE"
+echo "   MPI lib: $(mpichversion | head -n 1 | tr -d =)"
+echo "         C: $($CC --version 2>/dev/null | head -n 2 | tail -n 1)"
+echo "       C++: $($CXX --version 2>/dev/null | head -n 2 | tail -n 1)"
+echo "   Fortran: $($FC --version 2>/dev/null | head -n 2 | tail -n 1)"
+echo "======================================================================"
+echo ""