diff --git a/.gitmodules b/.gitmodules index 04c4249..40b8a34 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,7 +17,7 @@ [submodule "models/eCLM"] path = models/eCLM url = https://github.com/HPSCTerrSys/eCLM.git - branch = rc-0.1 + branch = dev-scalasca-profiling [submodule "models/oasis3-mct"] path = models/oasis3-mct url = https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct diff --git a/build_eclm_profile.sh b/build_eclm_profile.sh new file mode 100755 index 0000000..9b5719d --- /dev/null +++ b/build_eclm_profile.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Script for manually building eCLM +# +set -eo pipefail + +if [[ $1 == "nvhpc" ]]; then + COMPILER="NVHPC" + ENV_FILE=jsc.2026.nvhpc.openmpi +else + COMPILER="GNU" + ENV_FILE=jsc.2026.gnu.openmpi +fi +MAKE_JOBS=32 +TSMP2_ROOT=`git rev-parse --show-toplevel` + +# +# Source env for eCLM and Parflow build +# +cd ${TSMP2_ROOT} +echo "Sourcing ${ENV_FILE} ..." +source env/${ENV_FILE} --profile + +MODEL_ID=${SYSTEMNAME^^}_eCLM_S${STAGE}_${COMPILER}-OPENMPI_SCALASCA +BUILD_DIR=${TSMP2_ROOT}/bld_manual/${MODEL_ID} +if [[ -z ${INSTALL_DIR// } ]]; then + INSTALL_DIR=${TSMP2_ROOT}/bin/${MODEL_ID} +else + INSTALL_DIR=${INSTALL_DIR}/${MODEL_ID} +fi +echo "TSMP2 binaries will be installed to ${INSTALL_DIR}" + +# +# Build eCLM +# +echo "" +echo "Building eCLM with Score-P ..." +echo "" +cd ${TSMP2_ROOT} +rm -rf ${BLD_eCLM} +BLD_eCLM=${BUILD_DIR}/eCLM +SCOREP_WRAPPER=off cmake -S ${TSMP2_ROOT}/models/eCLM/src \ + -B ${BLD_eCLM} \ + -DCMAKE_BUILD_TYPE="PROFILE" \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_C_COMPILER=$CC \ + -DCMAKE_Fortran_COMPILER=$FC \ + -DCOUP_OAS_PFL=False + +cd ${BLD_eCLM} +make SCOREP_WRAPPER_INSTRUMENTER_FLAGS="--thread=pthread --verbose" VERBOSE=1 -j${MAKE_JOBS} install +echo "Successfully built eCLM!" + +# +# Copy env file to INSTALL_DIR +# +cp ${TSMP2_ROOT}/env/${ENV_FILE} ${INSTALL_DIR} + +echo "" +echo "Successfully built eCLM to ${INSTALL_DIR}" +echo "" diff --git a/build_eclmparflowgpu_profile.sh b/build_eclmparflowgpu_profile.sh new file mode 100755 index 0000000..580afc4 --- /dev/null +++ b/build_eclmparflowgpu_profile.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# +# Script for manually building OASIS3-MCT, eCLM, and Parflow. +# +set -eo pipefail + +if [[ $1 == "nvhpc" ]]; then + COMPILER="NVHPC" + ENV_FILE=jsc.2026.nvhpc.openmpi +else + COMPILER="GNU" + ENV_FILE=jsc.2026.gnu.openmpi +fi + +MAKE_JOBS=32 +TSMP2_ROOT=`git rev-parse --show-toplevel` + +# +# Source env for eCLM and Parflow build +# +cd ${TSMP2_ROOT} +echo "Sourcing ${ENV_FILE} ..." +source env/${ENV_FILE} --parflowgpu --profile + +MODEL_ID=${SYSTEMNAME^^}_eCLM-ParflowGPU_S${STAGE}_${COMPILER}-OPENMPI_SCALASCA +BUILD_DIR=${TSMP2_ROOT}/bld_manual/${MODEL_ID} +if [[ -z ${INSTALL_DIR// } ]]; then + INSTALL_DIR=${TSMP2_ROOT}/bin/${MODEL_ID} +else + INSTALL_DIR=${INSTALL_DIR}/${MODEL_ID} +fi +echo "TSMP2 binaries will be installed to ${INSTALL_DIR}" + +# +# Build OASIS3-MCT +# +echo "" +echo "Building OASIS3-MCT with Score-P ..." +echo "" +cd ${TSMP2_ROOT}/models/oasis3-mct +git pull origin v5.0_patched && git checkout tsmp-patches-v0.2 +export OASIS_ROOT=$(pwd) INSTALL_PREFIX=${INSTALL_DIR} +cd util/make_dir +make realclean -f TopMakefileOasis3 +INSTALL_PREFIX=${INSTALL_DIR} make -f TopMakefileOasis3 -j${MAKE_JOBS} static-libs +echo "Successfully built OASIS3-MCT!" + +echo "" +echo "Building Parflow with Score-P ..." +echo "" +cd ${TSMP2_ROOT} +BLD_ParFlow=${BUILD_DIR}/ParFlow +rm -rf ${BLD_ParFlow} +SCOREP_WRAPPER=off cmake -S ${TSMP2_ROOT}/models/parflow \ + -B ${BLD_ParFlow} \ + -DCMAKE_BUILD_TYPE="RELEASE" \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ + -DCMAKE_C_COMPILER=$CC \ + -DCMAKE_Fortran_COMPILER=$FC \ + -DCMAKE_CXX_COMPILER=$CXX \ + -DPARFLOW_ENABLE_HYPRE=ON \ + -DPARFLOW_ENABLE_NETCDF=ON \ + -DPARFLOW_AMPS_SEQUENTIAL_IO=ON \ + -DPARFLOW_ENABLE_TIMING=TRUE \ + -DPARFLOW_ACCELERATOR_BACKEND="cuda" \ + -DMPIEXEC_EXECUTABLE=$(which srun) \ + -DMPIEXEC_NUMPROC_FLAG="--ntasks" \ + -DPARFLOW_ENABLE_SLURM=TRUE \ + -DUMPIRE_ROOT=${EBROOTUMPIRE} \ + -DSUNDIALS_ROOT=${EBROOTSUNDIALS} \ + -DPARFLOW_AMPS_LAYER=oas3 \ + -DOAS3_ROOT=${INSTALL_DIR} \ + -DPARFLOW_HAVE_ECLM=ON +cd ${BLD_ParFlow} +make SCOREP_WRAPPER_INSTRUMENTER_FLAGS="--thread=pthread --verbose" VERBOSE=1 -j${MAKE_JOBS} install +echo "Successfully built ParFlowGPU!" + +# +# Build eCLM +# +echo "" +echo "Building eCLM with Score-P ..." +echo "" +cd ${TSMP2_ROOT} +rm -rf ${BLD_eCLM} +BLD_eCLM=${BUILD_DIR}/eCLM +SCOREP_WRAPPER=off cmake -S ${TSMP2_ROOT}/models/eCLM/src \ + -B ${BLD_eCLM} \ + -DCMAKE_BUILD_TYPE="PROFILE" \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_C_COMPILER=$CC \ + -DCMAKE_Fortran_COMPILER=$FC \ + -DCOUP_OAS_PFL=True + +cd ${BLD_eCLM} +make SCOREP_WRAPPER_INSTRUMENTER_FLAGS="--thread=pthread --verbose" VERBOSE=1 -j${MAKE_JOBS} install +echo "Successfully built eCLM!" + +# +# Copy env file to INSTALL_DIR +# +cp ${TSMP2_ROOT}/env/${ENV_FILE} ${INSTALL_DIR} + +echo "" +echo "Successfully built eCLM-ParflowGPU to ${INSTALL_DIR}" +echo "" diff --git a/build_tsmp2.sh b/build_tsmp2.sh index 3698f76..19b2e79 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -36,7 +36,7 @@ function help_tsmp2() { echo " --OASIS_SRC Set OASIS3-MCT directory" echo " --PDAF_SRC Set PDAF_SRC directory" echo " --no_update Skip component model download" - echo " --build_type Set build configuration: 'DEBUG' 'RELEASE'" + echo " --build_type Set build configuration: 'DEBUG', 'RELEASE', or 'PROFILE'" echo " --build_dir Set build dir cmake, if not set bld/_ is used. Build artifacts will be generated in this folder." echo " --install_dir Set install dir cmake, if not set bin/_ is used. Model executables and libraries will be installed here" echo " --clean_first Delete build_dir if it already exists" @@ -208,8 +208,19 @@ if [ "${update_compsrc}" != n ]; then dwn_compsrc clm35 clm35_src "CLM3.5" fi +# 2. Determine build configuration +if [[ -z "$build_type" ]];then + build_type="RELEASE" +fi +if [[ ${build_type^^} == "DEBUG" || ${build_type^^} == "RELEASE" || ${build_type^^} == "PROFILE" ]]; then + cmake_build_type="${build_type^^}" +else + echo "ABORT: Unsupported build_type=${build_type}" + exit 1 +fi + # -# 2. Source environment file +# 3. Source environment file # if [[ ! -z "${env}" ]]; then @@ -231,6 +242,7 @@ if [[ ! -f "${env}" ]]; then exit 1 fi + if [[ -n "${env}" ]]; then message "Sourcing environment..." @@ -242,6 +254,15 @@ if [[ -n "${env}" ]]; then else source "${env}" fi + if [[ ${build_type^^} == "PROFILE" ]];then + env_profile_params="--profile" + if [[ ":${TSMP2_ENV_FILE}:" != *"jsc.2026.gnu.openmpi"* ]]; then + message "ERROR: Profiling builds are only currently supported on jsc.2026.gnu.openmpi." + exit 1 + fi + fi + message "SOURCE command: source ${env} ${env_gpu_params} ${env_profile_params}" + source "${env}" ${env_gpu_params} ${env_profile_params} fi # TSMP2_ENV_FILE should be set either (1) through --env, (2) as a shell variable, @@ -252,9 +273,14 @@ if [[ -z "${TSMP2_ENV_FILE}" ]]; then fi # -# 3. Set CMake build directory +# 4. Set CMake build directory # -BUILD_ID="${SYSTEMNAME^^}_${model_id}" +if [[ ${build_type^^} == "RELEASE" ]]; then + BUILD_ID="${SYSTEMNAME^^}_${model_id}" +else + BUILD_ID="${SYSTEMNAME^^}_${build_type^^}_${model_id}" +fi + if [[ -z "${build_dir}" ]]; then cmake_build_dir="${cmake_tsmp2_dir}/bld/${BUILD_ID}" else @@ -296,18 +322,9 @@ fi build_log="$(dirname ${cmake_build_dir})/${BUILD_ID}_$(date +%Y-%m-%d_%H-%M).log" # -# 4. Set the rest of the CMake options +# 5. Set the rest of the CMake options # message "Setting CMAKE options..." -if [[ -z "$build_type" ]];then - build_type="RELEASE" -fi -if [[ ${build_type^^} == "DEBUG" || ${build_type^^} == "RELEASE" ]]; then - cmake_build_type="${build_type^^}" -else - echo "ABORT: Unsupported build_type=${build_type}" - exit 1 -fi if [ -z "${verbose_makefile}" ]; then cmake_verbose_makefile="OFF" @@ -332,7 +349,7 @@ fi message "Parallel make jobs set to ${parallel_jobs}" # -# 5. CMake configure +# 6. CMake configure # message "" message "====================" @@ -357,7 +374,7 @@ cmake ${cmake_conf} |& tee -a "${build_log}" message "== CMAKE GENERATE PROJECT finished" # -# 6. CMake build and install +# 7. CMake build and install # message "CMAKE build:" message "cmake --build ${cmake_build_dir} |& tee -a $build_log" @@ -372,7 +389,7 @@ cmake --install ${cmake_build_dir} |& tee -a $build_log message "== CMAKE INSTALL finished" # -# 7. Post-installation steps +# 8. Post-installation steps # message "Copying build log and environment file to ${cmake_install_dir}..." if [[ -n "${env}" ]]; then diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index eee469e..44d153d 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -4,7 +4,7 @@ find_package(OpenMP REQUIRED) if (CMAKE_BUILD_TYPE STREQUAL "DEBUG") set(OPTIM "-g") set(MCT_DEBUGFLAG "--enable-debugging") -elseif (CMAKE_BUILD_TYPE STREQUAL "RELEASE") +elseif (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "PROFILE") set(OPTIM "-O2") set(MCT_DEBUGFLAG "") else() diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 546480b..a3ebf51 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -41,7 +41,10 @@ endif() # Set compiler flags if(CMAKE_C_COMPILER_ID STREQUAL "GNU") # Flags were based from https://github.com/parflow/parflow/blob/c8aa8d7140db19153194728b8fa9136b95177b6d/.github/workflows/linux.yml#L486 - set(PF_CFLAGS "-Wall -Werror -Wno-unused-result -Wno-unused-function -Wno-stringop-overread") + set(PF_CFLAGS "-Wall -Wno-unused-result -Wno-unused-function -Wno-stringop-overread") + if (NOT CMAKE_BUILD_TYPE STREQUAL "PROFILE") + string(APPEND PF_CFLAGS " -Werror") #compile warnings not ignored for DEBUG and RELEASE builds + endif() if(${PDAF}) # parflow-pdaf fork currently ignores unused variables string(APPEND PF_CFLAGS " -Wno-unused-variable") @@ -88,6 +91,17 @@ if (DEFINED ENV{SUNDIALS_ROOT} AND NOT ${PDAF}) set(PF_SUNDIALS_FLAG "-DSUNDIALS_ROOT=$ENV{SUNDIALS_ROOT}") endif() +# Profiling build (Experimental) +if (CMAKE_BUILD_TYPE STREQUAL "PROFILE") + set(PF_C_COMPILER $ENV{TSMP2_PROFILE_CC}) + set(PF_CXX_COMPILER $ENV{TSMP2_PROFILE_CXX}) + set(PF_Fortran_COMPILER $ENV{TSMP2_PROFILE_FC}) +else() + set(PF_C_COMPILER ${CMAKE_C_COMPILER}) + set(PF_CXX_COMPILER ${CMAKE_CXX_COMPILER}) + set(PF_Fortran_COMPILER ${CMAKE_Fortran_COMPILER}) +endif() + # Pass options to ParFlow CMake ExternalProject_Add(ParFlow PREFIX ParFlow diff --git a/cmake/BuildeCLM.cmake b/cmake/BuildeCLM.cmake index 941aa59..83341d7 100644 --- a/cmake/BuildeCLM.cmake +++ b/cmake/BuildeCLM.cmake @@ -14,12 +14,21 @@ if(${PDAF}) list(APPEND PDAF_FLAGS -DUSE_PDAF=True) endif() +# Experimental +if (CMAKE_BUILD_TYPE STREQUAL "PROFILE") + set(eCLM_C_COMPILER $ENV{TSMP2_PROFILE_CC}) + set(eCLM_Fortran_COMPILER $ENV{TSMP2_PROFILE_FC}) +else() + set(eCLM_C_COMPILER ${CMAKE_C_COMPILER}) + set(eCLM_Fortran_COMPILER ${CMAKE_Fortran_COMPILER}) +endif() + ExternalProject_Add(eCLM PREFIX eCLM SOURCE_DIR ${eCLM_SRC} SOURCE_SUBDIR src - CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} + CMAKE_ARGS -DCMAKE_C_COMPILER=${eCLM_C_COMPILER} + -DCMAKE_Fortran_COMPILER=${eCLM_Fortran_COMPILER} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_PREFIX_PATH=${OASIS_ROOT} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} diff --git a/env/jsc.2025.gnu.openmpi b/env/jsc.2025.gnu.openmpi index a3e705f..766f957 100644 --- a/env/jsc.2025.gnu.openmpi +++ b/env/jsc.2025.gnu.openmpi @@ -53,12 +53,20 @@ module load Tcl module load ecCodes # Set default MPI compilers -export OMPI_CC=gcc -export OMPI_CXX=g++ -export OMPI_FC=gfortran -export CC=mpicc -export FC=mpif90 -export CXX=mpicxx +if [[ "$1" == "--profile" || "$2" == "--profile" ]]; then + module load Scalasca + export CC=$(which scorep-mpicc) + export FC=$(which scorep-mpif90) + export CXX=$(which scorep-mpicxx) + unset OMPI_CC OMPI_FC OMPI_CXX +else + export OMPI_CC=gcc + export OMPI_CXX=g++ + export OMPI_FC=gfortran + export CC=mpicc + export FC=mpif90 + export CXX=mpicxx +fi export MPI_HOME=$EBROOTOPENMPI export MPIEXEC_EXECUTABLE=$(which srun) export TSMP2_PARALLEL_JOBS=16 diff --git a/env/jsc.2026.gnu.openmpi b/env/jsc.2026.gnu.openmpi index b2b9ace..44d2819 100644 --- a/env/jsc.2026.gnu.openmpi +++ b/env/jsc.2026.gnu.openmpi @@ -52,9 +52,16 @@ fi module load ecCodes # Set default MPI compilers -export CC=mpicc -export FC=mpif90 -export CXX=mpicxx +if [[ "$1" == "--profile" || "$2" == "--profile" ]]; then + module load Scalasca + export CC=$(which scorep-mpicc) + export FC=$(which scorep-mpif90) + export CXX=$(which scorep-mpicxx) +else + export CC=mpicc + export FC=mpif90 + export CXX=mpicxx +fi # OMPI_* variables affect OpenMPI backend compilers. They should be # unset so that the correct default backend compilers are selected. diff --git a/env/jsc.2026.nvhpc.openmpi b/env/jsc.2026.nvhpc.openmpi index 31c2804..9c2c65e 100644 --- a/env/jsc.2026.nvhpc.openmpi +++ b/env/jsc.2026.nvhpc.openmpi @@ -50,9 +50,16 @@ export CMAKE_CUDA_RUNTIME_LIBRARY="Shared" module load ecCodes # Set default MPI compilers -export CC=mpicc -export FC=mpifort -export CXX=mpicxx +if [[ "$1" == "--profile" || "$2" == "--profile" ]]; then + module load Scalasca + export CC=$(which scorep-mpicc) + export FC=$(which scorep-mpif90) + export CXX=$(which scorep-mpicxx) +else + export CC=mpicc + export FC=mpifort + export CXX=mpicxx +fi # OMPI_* variables affect OpenMPI backend compilers. They should be # unset so that the correct default backend compilers are selected. diff --git a/models/eCLM b/models/eCLM index 9c8f1ae..74a0c8e 160000 --- a/models/eCLM +++ b/models/eCLM @@ -1 +1 @@ -Subproject commit 9c8f1ae22afc9e47ffee46f24da4fb367736c77f +Subproject commit 74a0c8e9e5c778ef4a1fd35bddf0c272997ee886