#!/bin/bash # How to set up a software build environment on Rocky 9.5 v 18-03-2025. # # Generally a cluster is set up in three stages: # # 1. make sure hardware is set up and nodes can be provisioned with an OS [from a master/head node] using tools like: Warewulf, xCAT, Bright, Trinity-X, Qlustar or DIY with PXE booting anaconda kickstart. In this stage /home is usually mounted on shared storage [master/head node often] on all nodes and UIDs and GIDs of users are kept in sync on all nodes via YP/NIS or LDAP [set up on master/head node]. # 2. set up SLURM and an [integrated] MPI stack on all nodes of the cluster. Sometimes a basic setup is provided by the provisioning tool used. # 3. set up a build environment on shared storage [/home often] so tools like EasyBuild and Spack can be used to build extra software. For this all nodes should be equipped with Lmod. # # This script is about step 3 and it will: # # - set up a TMPDIR in /home of builder # - set up and test drive EasyBuild to install in i.e. /home/software/easybuild # - set up and test drive Spack to install in i.e. /home/software/spack # # Prerequisites for this script: # # - @development, gfortran, wget, rdma-core-devel, python3-setuptools, python3-pip, python3-devel and Lmod [from EPEL] are installed. # - a local user [builder] account has been created and enough disk space is available [mind any possible quotas]. # - a big enough shared storage location for all nodes [/home/software] is set up owned 755 by the local user [builder]. # # export SOFTWARE=/home/software # set your shared storage location here. we chose /home because it usually is included in backup, size is plenty and it is nfs exported to all nodes # First set up TMPDIR to ~/tmp so build trees do not flood /tmp. mkdir ~/tmp export TMPDIR=~/tmp echo "export TMPDIR=~/tmp" >> ~/.bashrc # set up for future use and remember to clean up once in a while :) # To check if SLURM was installed. export SLURM_VERSION=$([[ -x /usr/bin/sinfo ]] && sinfo --version| cut -d' ' -f 2) export ARCH=$(uname -m) # First always update pip as user. pip install -U --user pip # Then set up Easybuild from https://easybuild.io/ using pip. pip install easybuild export EASYBUILD_INSTALLPATH=$SOFTWARE/easybuild # configure EasyBuild to install software in the right place echo "export EASYBUILD_INSTALLPATH=$SOFTWARE/easybuild" >> ~/.bashrc # set it up for future use mkdir -m 755 -p $EASYBUILD_INSTALLPATH/modules/all # create the right directories and modes echo "module use $EASYBUILD_INSTALLPATH/modules/all/" >> $SOFTWARE/source_this_to_use_software # and create file for users to source echo "source $SOFTWARE/source_this_to_use_software" >> ~/.bashrc # set it up for future use source $SOFTWARE/source_this_to_use_software # and source it if [[ "$SLURM_VERSION" != "" ]] # only if there was a SLURM installed we compile an OpenMPI against it then # To make an OpenMPI built with Easybuild work nice with SLURM. Also read https://docs.easybuild.io/controlling-compiler-optimization-flags/#controlling_compiler_optimization_flags_optarch_caveats_hardcoding and https://docs.easybuild.io/common-toolchains/#more-information-about-toolchains. echo "configopts += '--enable-mpi1-compatibility --with-slurm --with-pmi=/usr --with-pmi-libdir=/usr/lib64/ --disable-static --enable-mpi-cxx --enable-mpi-fortran --with-hwloc=internal --with-libevent=internal --with-pmix=internal '" >> .local/easybuild/easyconfigs/o/OpenMPI/OpenMPI-4.1.5-GCC-12.3.0.eb # patch the easyconfig of OpenMPI first eb -r OpenMPI-4.1.5-GCC-12.3.0.eb # then build and install OpenMPI using -r to get all dependencies fi # Build something with EasyBuild now. eb -r cowsay-3.04.eb module load cowsay cowsay "we have life!" # Now set up Spack from github and configure it to install software in the right place. Read more about it at https://spack.readthedocs.io. git clone https://github.com/spack/spack.git # clone spack git repo source spack/share/spack/setup-env.sh # make spack command available echo "source spack/share/spack/setup-env.sh" >> ~/.bashrc # set it up for future use mkdir -m 755 -p $SOFTWARE/spack/modules/linux-rocky9-$ARCH/Core # create the right directories and modes mkdir ~/.spack # setup basic config so Spack installs in right place cat > ~/.spack/config.yaml << EOD config: # for more info go to https://spack.readthedocs.io/en/latest/config_yaml.html install_tree: root: $SOFTWARE/spack EOD cat > ~/.spack/modules.yaml << EOD modules: # for more info go to https://spack.readthedocs.io/en/latest/module_file_support.html default: roots: lmod: $SOFTWARE/spack/modules EOD echo "module use $SOFTWARE/spack/modules/linux-rocky9-$ARCH/Core" >> $SOFTWARE/source_this_to_use_software # and make sure users can load modules source $SOFTWARE/source_this_to_use_software # and use it our selves # Make Spack use system compilers too. spack compiler find # for more info go to https://spack.readthedocs.io/en/latest/getting_started.html#compiler-configuration if [[ "$SLURM_VERSION" != "" ]] # only if there was a SLURM installed we setup Spack to compile any OpenMPI against it then # Now to make sure Spack also builds OpenMPI against the local SLURM, build it already using the needed options. export GCC_VERSION=$(gcc --version | head -n 1 | cut -d' ' -f 3) cat > ~/.spack/packages.yaml << EOD packages: # for more info go to https://spack.readthedocs.io/en/latest/build_settings.html slurm: externals: - spec: "slurm@$SLURM_VERSION%gcc@$GCC_VERSION arch=linux-rocky9-$ARCH" prefix: /usr buildable: False pmix: externals: - spec: "pmix@3%gcc@$GCC_VERSION arch=linux-rocky9-$ARCH" prefix: /usr buildable: False EOD spack install openmpi +pmi schedulers=slurm fi # Build something with Spack now. spack install cowsay spack module lmod refresh -y # create lmod modules too for all we have built module avail # and show them