| #!/bin/bash
|
| # How to set up a software build environment on Rocky 9.5 v 18-03-2025.
|
| #
|
| # Generally a cluster is set up in three stages:
|
| #
|
| # 1. make sure hardware is set up and nodes can be provisioned with an OS [from a master/head node] using tools like: Warewulf, xCAT, Bright, Trinity-X, Qlustar or DIY with PXE booting anaconda kickstart. In this stage /home is usually mounted on shared storage [master/head node often] on all nodes and UIDs and GIDs of users are kept in sync on all nodes via YP/NIS or LDAP [set up on master/head node].
|
| # 2. set up SLURM and an [integrated] MPI stack on all nodes of the cluster. Sometimes a basic setup is provided by the provisioning tool used.
|
| # 3. set up a build environment on shared storage [/home often] so tools like EasyBuild and Spack can be used to build extra software. For this all nodes should be equipped with Lmod.
|
| #
|
| # This script is about step 3 and it will:
|
| #
|
| # - set up a TMPDIR in /home of builder
|
| # - set up and test drive EasyBuild to install in i.e. /home/software/easybuild
|
| # - set up and test drive Spack to install in i.e. /home/software/spack
|
| #
|
| # Prerequisites for this script:
|
| #
|
| # - @development, gfortran, wget, rdma-core-devel, python3-setuptools, python3-pip, python3-devel and Lmod [from EPEL] are installed.
|
| # - a local user [builder] account has been created and enough disk space is available [mind any possible quotas].
|
| # - a big enough shared storage location for all nodes [/home/software] is set up owned 755 by the local user [builder].
|
| #
|
| #
|
|
|
| export SOFTWARE=/home/software # set your shared storage location here. we chose /home because it usually is included in backup, size is plenty and it is nfs exported to all nodes
|
|
|
| # First set up TMPDIR to ~/tmp so build trees do not flood /tmp.
|
| mkdir ~/tmp
|
| export TMPDIR=~/tmp
|
| echo "export TMPDIR=~/tmp" >> ~/.bashrc # set up for future use and remember to clean up once in a while :)
|
|
|
| # To check if SLURM was installed.
|
| export SLURM_VERSION=$([[ -x /usr/bin/sinfo ]] && sinfo --version| cut -d' ' -f 2)
|
| export ARCH=$(uname -m)
|
|
|
| # First always update pip as user.
|
| pip install -U --user pip
|
|
|
| # Then set up Easybuild from https://easybuild.io/ using pip.
|
| pip install easybuild
|
| export EASYBUILD_INSTALLPATH=$SOFTWARE/easybuild # configure EasyBuild to install software in the right place
|
| echo "export EASYBUILD_INSTALLPATH=$SOFTWARE/easybuild" >> ~/.bashrc # set it up for future use
|
| mkdir -m 755 -p $EASYBUILD_INSTALLPATH/modules/all # create the right directories and modes
|
| echo "module use $EASYBUILD_INSTALLPATH/modules/all/" >> $SOFTWARE/source_this_to_use_software # and create file for users to source
|
| echo "source $SOFTWARE/source_this_to_use_software" >> ~/.bashrc # set it up for future use
|
| source $SOFTWARE/source_this_to_use_software # and source it
|
|
|
| if [[ "$SLURM_VERSION" != "" ]] # only if there was a SLURM installed we compile an OpenMPI against it
|
| then
|
|
|
| # To make an OpenMPI built with Easybuild work nice with SLURM. Also read https://docs.easybuild.io/controlling-compiler-optimization-flags/#controlling_compiler_optimization_flags_optarch_caveats_hardcoding and https://docs.easybuild.io/common-toolchains/#more-information-about-toolchains.
|
| echo "configopts += '--enable-mpi1-compatibility --with-slurm --with-pmi=/usr --with-pmi-libdir=/usr/lib64/ --disable-static --enable-mpi-cxx --enable-mpi-fortran --with-hwloc=internal --with-libevent=internal --with-pmix=internal '" >> .local/easybuild/easyconfigs/o/OpenMPI/OpenMPI-4.1.5-GCC-12.3.0.eb # patch the easyconfig of OpenMPI first
|
| eb -r OpenMPI-4.1.5-GCC-12.3.0.eb # then build and install OpenMPI using -r to get all dependencies
|
|
|
| fi
|
|
|
| # Build something with EasyBuild now.
|
| eb -r cowsay-3.04.eb
|
| module load cowsay
|
| cowsay "we have life!"
|
|
|
|
|
|
|
|
|
| # Now set up Spack from github and configure it to install software in the right place. Read more about it at https://spack.readthedocs.io.
|
| git clone https://github.com/spack/spack.git # clone spack git repo
|
| source spack/share/spack/setup-env.sh # make spack command available
|
| echo "source spack/share/spack/setup-env.sh" >> ~/.bashrc # set it up for future use
|
| mkdir -m 755 -p $SOFTWARE/spack/modules/linux-rocky9-$ARCH/Core # create the right directories and modes
|
| mkdir ~/.spack # setup basic config so Spack installs in right place
|
| cat > ~/.spack/config.yaml << EOD
|
| config: # for more info go to https://spack.readthedocs.io/en/latest/config_yaml.html
|
| install_tree:
|
| root: $SOFTWARE/spack
|
| EOD
|
| cat > ~/.spack/modules.yaml << EOD
|
| modules: # for more info go to https://spack.readthedocs.io/en/latest/module_file_support.html
|
| default:
|
| roots:
|
| lmod: $SOFTWARE/spack/modules
|
| EOD
|
| echo "module use $SOFTWARE/spack/modules/linux-rocky9-$ARCH/Core" >> $SOFTWARE/source_this_to_use_software # and make sure users can load modules
|
| source $SOFTWARE/source_this_to_use_software # and use it our selves
|
|
|
| # Make Spack use system compilers too.
|
| spack compiler find # for more info go to https://spack.readthedocs.io/en/latest/getting_started.html#compiler-configuration
|
|
|
|
|
| if [[ "$SLURM_VERSION" != "" ]] # only if there was a SLURM installed we setup Spack to compile any OpenMPI against it
|
| then
|
|
|
| # Now to make sure Spack also builds OpenMPI against the local SLURM, build it already using the needed options.
|
| export GCC_VERSION=$(gcc --version | head -n 1 | cut -d' ' -f 3)
|
| cat > ~/.spack/packages.yaml << EOD
|
| packages: # for more info go to https://spack.readthedocs.io/en/latest/build_settings.html
|
| slurm:
|
| externals:
|
| - spec: "slurm@$SLURM_VERSION%gcc@$GCC_VERSION arch=linux-rocky9-$ARCH"
|
| prefix: /usr
|
| buildable: False
|
| pmix:
|
| externals:
|
| - spec: "pmix@3%gcc@$GCC_VERSION arch=linux-rocky9-$ARCH"
|
| prefix: /usr
|
| buildable: False
|
| EOD
|
| spack install openmpi +pmi schedulers=slurm
|
|
|
| fi
|
|
|
| # Build something with Spack now.
|
| spack install cowsay
|
| spack module lmod refresh -y # create lmod modules too for all we have built
|
| module avail # and show them
|