Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
[submodule "models/eCLM"]
path = models/eCLM
url = https://github.com/HPSCTerrSys/eCLM.git
branch = rc-0.1
branch = dev-scalasca-profiling
[submodule "models/oasis3-mct"]
path = models/oasis3-mct
url = https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct
Expand Down
62 changes: 62 additions & 0 deletions build_eclm_profile.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/usr/bin/env bash
#
# Script for manually building eCLM
#
set -eo pipefail

if [[ $1 == "nvhpc" ]]; then
COMPILER="NVHPC"
ENV_FILE=jsc.2026.nvhpc.openmpi
else
COMPILER="GNU"
ENV_FILE=jsc.2026.gnu.openmpi
fi
MAKE_JOBS=32
TSMP2_ROOT=`git rev-parse --show-toplevel`

#
# Source env for eCLM and Parflow build
#
cd ${TSMP2_ROOT}
echo "Sourcing ${ENV_FILE} ..."
source env/${ENV_FILE} --profile

MODEL_ID=${SYSTEMNAME^^}_eCLM_S${STAGE}_${COMPILER}-OPENMPI_SCALASCA
BUILD_DIR=${TSMP2_ROOT}/bld_manual/${MODEL_ID}
if [[ -z ${INSTALL_DIR// } ]]; then
INSTALL_DIR=${TSMP2_ROOT}/bin/${MODEL_ID}
else
INSTALL_DIR=${INSTALL_DIR}/${MODEL_ID}
fi
echo "TSMP2 binaries will be installed to ${INSTALL_DIR}"

#
# Build eCLM
#
echo ""
echo "Building eCLM with Score-P ..."
echo ""
cd ${TSMP2_ROOT}
rm -rf ${BLD_eCLM}
BLD_eCLM=${BUILD_DIR}/eCLM
SCOREP_WRAPPER=off cmake -S ${TSMP2_ROOT}/models/eCLM/src \
-B ${BLD_eCLM} \
-DCMAKE_BUILD_TYPE="PROFILE" \
-DCMAKE_PREFIX_PATH=${INSTALL_DIR} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DCMAKE_C_COMPILER=$CC \
-DCMAKE_Fortran_COMPILER=$FC \
-DCOUP_OAS_PFL=False

cd ${BLD_eCLM}
make SCOREP_WRAPPER_INSTRUMENTER_FLAGS="--thread=pthread --verbose" VERBOSE=1 -j${MAKE_JOBS} install
echo "Successfully built eCLM!"

#
# Copy env file to INSTALL_DIR
#
cp ${TSMP2_ROOT}/env/${ENV_FILE} ${INSTALL_DIR}

echo ""
echo "Successfully built eCLM to ${INSTALL_DIR}"
echo ""
108 changes: 108 additions & 0 deletions build_eclmparflowgpu_profile.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
#!/usr/bin/env bash
#
# Script for manually building OASIS3-MCT, eCLM, and Parflow.
#
set -eo pipefail

if [[ $1 == "nvhpc" ]]; then
COMPILER="NVHPC"
ENV_FILE=jsc.2026.nvhpc.openmpi
else
COMPILER="GNU"
ENV_FILE=jsc.2026.gnu.openmpi
fi

MAKE_JOBS=32
TSMP2_ROOT=`git rev-parse --show-toplevel`

#
# Source env for eCLM and Parflow build
#
cd ${TSMP2_ROOT}
echo "Sourcing ${ENV_FILE} ..."
source env/${ENV_FILE} --parflowgpu --profile

MODEL_ID=${SYSTEMNAME^^}_eCLM-ParflowGPU_S${STAGE}_${COMPILER}-OPENMPI_SCALASCA
BUILD_DIR=${TSMP2_ROOT}/bld_manual/${MODEL_ID}
if [[ -z ${INSTALL_DIR// } ]]; then
INSTALL_DIR=${TSMP2_ROOT}/bin/${MODEL_ID}
else
INSTALL_DIR=${INSTALL_DIR}/${MODEL_ID}
fi
echo "TSMP2 binaries will be installed to ${INSTALL_DIR}"

#
# Build OASIS3-MCT
#
echo ""
echo "Building OASIS3-MCT with Score-P ..."
echo ""
cd ${TSMP2_ROOT}/models/oasis3-mct
git pull origin v5.0_patched && git checkout tsmp-patches-v0.2
export OASIS_ROOT=$(pwd) INSTALL_PREFIX=${INSTALL_DIR}
cd util/make_dir
make realclean -f TopMakefileOasis3
INSTALL_PREFIX=${INSTALL_DIR} make -f TopMakefileOasis3 -j${MAKE_JOBS} static-libs
echo "Successfully built OASIS3-MCT!"

echo ""
echo "Building Parflow with Score-P ..."
echo ""
cd ${TSMP2_ROOT}
BLD_ParFlow=${BUILD_DIR}/ParFlow
rm -rf ${BLD_ParFlow}
SCOREP_WRAPPER=off cmake -S ${TSMP2_ROOT}/models/parflow \
-B ${BLD_ParFlow} \
-DCMAKE_BUILD_TYPE="RELEASE" \
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DCMAKE_PREFIX_PATH=${INSTALL_DIR} \
-DCMAKE_C_COMPILER=$CC \
-DCMAKE_Fortran_COMPILER=$FC \
-DCMAKE_CXX_COMPILER=$CXX \
-DPARFLOW_ENABLE_HYPRE=ON \
-DPARFLOW_ENABLE_NETCDF=ON \
-DPARFLOW_AMPS_SEQUENTIAL_IO=ON \
-DPARFLOW_ENABLE_TIMING=TRUE \
-DPARFLOW_ACCELERATOR_BACKEND="cuda" \
-DMPIEXEC_EXECUTABLE=$(which srun) \
-DMPIEXEC_NUMPROC_FLAG="--ntasks" \
-DPARFLOW_ENABLE_SLURM=TRUE \
-DUMPIRE_ROOT=${EBROOTUMPIRE} \
-DSUNDIALS_ROOT=${EBROOTSUNDIALS} \
-DPARFLOW_AMPS_LAYER=oas3 \
-DOAS3_ROOT=${INSTALL_DIR} \
-DPARFLOW_HAVE_ECLM=ON
cd ${BLD_ParFlow}
make SCOREP_WRAPPER_INSTRUMENTER_FLAGS="--thread=pthread --verbose" VERBOSE=1 -j${MAKE_JOBS} install
echo "Successfully built ParFlowGPU!"

#
# Build eCLM
#
echo ""
echo "Building eCLM with Score-P ..."
echo ""
cd ${TSMP2_ROOT}
rm -rf ${BLD_eCLM}
BLD_eCLM=${BUILD_DIR}/eCLM
SCOREP_WRAPPER=off cmake -S ${TSMP2_ROOT}/models/eCLM/src \
-B ${BLD_eCLM} \
-DCMAKE_BUILD_TYPE="PROFILE" \
-DCMAKE_PREFIX_PATH=${INSTALL_DIR} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DCMAKE_C_COMPILER=$CC \
-DCMAKE_Fortran_COMPILER=$FC \
-DCOUP_OAS_PFL=True

cd ${BLD_eCLM}
make SCOREP_WRAPPER_INSTRUMENTER_FLAGS="--thread=pthread --verbose" VERBOSE=1 -j${MAKE_JOBS} install
echo "Successfully built eCLM!"

#
# Copy env file to INSTALL_DIR
#
cp ${TSMP2_ROOT}/env/${ENV_FILE} ${INSTALL_DIR}

echo ""
echo "Successfully built eCLM-ParflowGPU to ${INSTALL_DIR}"
echo ""
51 changes: 34 additions & 17 deletions build_tsmp2.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ function help_tsmp2() {
echo " --OASIS_SRC Set OASIS3-MCT directory"
echo " --PDAF_SRC Set PDAF_SRC directory"
echo " --no_update Skip component model download"
echo " --build_type Set build configuration: 'DEBUG' 'RELEASE'"
echo " --build_type Set build configuration: 'DEBUG', 'RELEASE', or 'PROFILE'"
echo " --build_dir Set build dir cmake, if not set bld/<SYSTEMNAME>_<model-id> is used. Build artifacts will be generated in this folder."
echo " --install_dir Set install dir cmake, if not set bin/<SYSTEMNAME>_<model-id> is used. Model executables and libraries will be installed here"
echo " --clean_first Delete build_dir if it already exists"
Expand Down Expand Up @@ -208,8 +208,19 @@ if [ "${update_compsrc}" != n ]; then
dwn_compsrc clm35 clm35_src "CLM3.5"
fi

# 2. Determine build configuration
if [[ -z "$build_type" ]];then
build_type="RELEASE"
fi
if [[ ${build_type^^} == "DEBUG" || ${build_type^^} == "RELEASE" || ${build_type^^} == "PROFILE" ]]; then
cmake_build_type="${build_type^^}"
else
echo "ABORT: Unsupported build_type=${build_type}"
exit 1
fi

#
# 2. Source environment file
# 3. Source environment file
#

if [[ ! -z "${env}" ]]; then
Expand All @@ -231,6 +242,7 @@ if [[ ! -f "${env}" ]]; then
exit 1
fi


if [[ -n "${env}" ]]; then
message "Sourcing environment..."

Expand All @@ -242,6 +254,15 @@ if [[ -n "${env}" ]]; then
else
source "${env}"
fi
if [[ ${build_type^^} == "PROFILE" ]];then
env_profile_params="--profile"
if [[ ":${TSMP2_ENV_FILE}:" != *"jsc.2026.gnu.openmpi"* ]]; then
message "ERROR: Profiling builds are only currently supported on jsc.2026.gnu.openmpi."
exit 1
fi
fi
message "SOURCE command: source ${env} ${env_gpu_params} ${env_profile_params}"
source "${env}" ${env_gpu_params} ${env_profile_params}
fi

# TSMP2_ENV_FILE should be set either (1) through --env, (2) as a shell variable,
Expand All @@ -252,9 +273,14 @@ if [[ -z "${TSMP2_ENV_FILE}" ]]; then
fi

#
# 3. Set CMake build directory
# 4. Set CMake build directory
#
BUILD_ID="${SYSTEMNAME^^}_${model_id}"
if [[ ${build_type^^} == "RELEASE" ]]; then
BUILD_ID="${SYSTEMNAME^^}_${model_id}"
else
BUILD_ID="${SYSTEMNAME^^}_${build_type^^}_${model_id}"
fi

if [[ -z "${build_dir}" ]]; then
cmake_build_dir="${cmake_tsmp2_dir}/bld/${BUILD_ID}"
else
Expand Down Expand Up @@ -296,18 +322,9 @@ fi
build_log="$(dirname ${cmake_build_dir})/${BUILD_ID}_$(date +%Y-%m-%d_%H-%M).log"

#
# 4. Set the rest of the CMake options
# 5. Set the rest of the CMake options
#
message "Setting CMAKE options..."
if [[ -z "$build_type" ]];then
build_type="RELEASE"
fi
if [[ ${build_type^^} == "DEBUG" || ${build_type^^} == "RELEASE" ]]; then
cmake_build_type="${build_type^^}"
else
echo "ABORT: Unsupported build_type=${build_type}"
exit 1
fi

if [ -z "${verbose_makefile}" ]; then
cmake_verbose_makefile="OFF"
Expand All @@ -332,7 +349,7 @@ fi
message "Parallel make jobs set to ${parallel_jobs}"

#
# 5. CMake configure
# 6. CMake configure
#
message ""
message "===================="
Expand All @@ -357,7 +374,7 @@ cmake ${cmake_conf} |& tee -a "${build_log}"
message "== CMAKE GENERATE PROJECT finished"

#
# 6. CMake build and install
# 7. CMake build and install
#
message "CMAKE build:"
message "cmake --build ${cmake_build_dir} |& tee -a $build_log"
Expand All @@ -372,7 +389,7 @@ cmake --install ${cmake_build_dir} |& tee -a $build_log
message "== CMAKE INSTALL finished"

#
# 7. Post-installation steps
# 8. Post-installation steps
#
message "Copying build log and environment file to ${cmake_install_dir}..."
if [[ -n "${env}" ]]; then
Expand Down
2 changes: 1 addition & 1 deletion cmake/BuildOASIS3MCT.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ find_package(OpenMP REQUIRED)
if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
set(OPTIM "-g")
set(MCT_DEBUGFLAG "--enable-debugging")
elseif (CMAKE_BUILD_TYPE STREQUAL "RELEASE")
elseif (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "PROFILE")
set(OPTIM "-O2")
set(MCT_DEBUGFLAG "")
else()
Expand Down
16 changes: 15 additions & 1 deletion cmake/BuildParFlow.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,10 @@ endif()
# Set compiler flags
if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
# Flags were based from https://github.com/parflow/parflow/blob/c8aa8d7140db19153194728b8fa9136b95177b6d/.github/workflows/linux.yml#L486
set(PF_CFLAGS "-Wall -Werror -Wno-unused-result -Wno-unused-function -Wno-stringop-overread")
set(PF_CFLAGS "-Wall -Wno-unused-result -Wno-unused-function -Wno-stringop-overread")
if (NOT CMAKE_BUILD_TYPE STREQUAL "PROFILE")
string(APPEND PF_CFLAGS " -Werror") #compile warnings not ignored for DEBUG and RELEASE builds
endif()
if(${PDAF})
# parflow-pdaf fork currently ignores unused variables
string(APPEND PF_CFLAGS " -Wno-unused-variable")
Expand Down Expand Up @@ -88,6 +91,17 @@ if (DEFINED ENV{SUNDIALS_ROOT} AND NOT ${PDAF})
set(PF_SUNDIALS_FLAG "-DSUNDIALS_ROOT=$ENV{SUNDIALS_ROOT}")
endif()

# Profiling build (Experimental)
if (CMAKE_BUILD_TYPE STREQUAL "PROFILE")
set(PF_C_COMPILER $ENV{TSMP2_PROFILE_CC})
set(PF_CXX_COMPILER $ENV{TSMP2_PROFILE_CXX})
set(PF_Fortran_COMPILER $ENV{TSMP2_PROFILE_FC})
else()
set(PF_C_COMPILER ${CMAKE_C_COMPILER})
set(PF_CXX_COMPILER ${CMAKE_CXX_COMPILER})
set(PF_Fortran_COMPILER ${CMAKE_Fortran_COMPILER})
endif()

# Pass options to ParFlow CMake
ExternalProject_Add(ParFlow
PREFIX ParFlow
Expand Down
13 changes: 11 additions & 2 deletions cmake/BuildeCLM.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,21 @@ if(${PDAF})
list(APPEND PDAF_FLAGS -DUSE_PDAF=True)
endif()

# Experimental
if (CMAKE_BUILD_TYPE STREQUAL "PROFILE")
set(eCLM_C_COMPILER $ENV{TSMP2_PROFILE_CC})
set(eCLM_Fortran_COMPILER $ENV{TSMP2_PROFILE_FC})
else()
set(eCLM_C_COMPILER ${CMAKE_C_COMPILER})
set(eCLM_Fortran_COMPILER ${CMAKE_Fortran_COMPILER})
endif()

ExternalProject_Add(eCLM
PREFIX eCLM
SOURCE_DIR ${eCLM_SRC}
SOURCE_SUBDIR src
CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER}
CMAKE_ARGS -DCMAKE_C_COMPILER=${eCLM_C_COMPILER}
-DCMAKE_Fortran_COMPILER=${eCLM_Fortran_COMPILER}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_PREFIX_PATH=${OASIS_ROOT}
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
Expand Down
20 changes: 14 additions & 6 deletions env/jsc.2025.gnu.openmpi
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,20 @@ module load Tcl
module load ecCodes

# Set default MPI compilers
export OMPI_CC=gcc
export OMPI_CXX=g++
export OMPI_FC=gfortran
export CC=mpicc
export FC=mpif90
export CXX=mpicxx
if [[ "$1" == "--profile" || "$2" == "--profile" ]]; then
module load Scalasca
export CC=$(which scorep-mpicc)
export FC=$(which scorep-mpif90)
export CXX=$(which scorep-mpicxx)
unset OMPI_CC OMPI_FC OMPI_CXX
else
export OMPI_CC=gcc
export OMPI_CXX=g++
export OMPI_FC=gfortran
export CC=mpicc
export FC=mpif90
export CXX=mpicxx
fi
export MPI_HOME=$EBROOTOPENMPI
export MPIEXEC_EXECUTABLE=$(which srun)
export TSMP2_PARALLEL_JOBS=16
Expand Down
Loading
Loading