diff --git a/scripts/bee-mxqsub b/scripts/bee-mxqsub index a1c28212e..52ad7b34a 100755 --- a/scripts/bee-mxqsub +++ b/scripts/bee-mxqsub @@ -22,6 +22,12 @@ # a single bee-file, where there is nothing to chain, so that the usual # one-package build is unchanged. # +# Packages that are already built need not be part of the chain: --preinstall +# installs their archives into the overlay before the first bee-file is built, +# so that rebuilding, say, mesa against a released LLVM newer than the host's +# does not rebuild LLVM as well. A pre-installed package turns the overlay on +# even for a single bee-file. +# # Note that the build tree lives in --bee-tmpdir, which defaults to the RAM # disk /dev/shm, so --memory has to cover the sources, the objects and the # build root, not just the compilers. The overlay lives there too, and holds a @@ -72,6 +78,8 @@ staged_in= overlay=auto overlay_root= keep_overlay=0 +# Where --preinstall looks up a package given as name-version-revision. +preinstall_from=/src/mariux/beeroot/packages keep_going=0 dry_run=0 run_job=0 @@ -79,6 +87,9 @@ declare -a beesh_args=() declare -a job_env=() declare -a overlay_dirs=() declare -a beefiles=() +# Package archives to install into the overlay before the first build. The +# submit phase resolves them to absolute paths, which the job phase is given. +declare -a preinstalls=() # Parallel to beefiles: the bee-file each entry was copied from, for the log. declare -a staged_from=() @@ -124,6 +135,14 @@ Options: --overlay-dir DIR host directory to overlay, may be repeated (default: /usr /etc) --keep-overlay do not remove the overlay when the job is done + -p, --preinstall PKG install the already built package PKG into the + overlay before the first bee-file is built, may be + repeated; PKG is a package archive or + name-version-revision[.arch], looked up in the + --preinstall-from directory; turns on --overlay + --preinstall-from DIR + where to look up --preinstall packages + (default: ${preinstall_from}) -k, --keep-going build the remaining bee-files after a failure -B, --beesh-arg ARG pass ARG to beesh, may be repeated (default: -c) -E, --env VAR=VALUE set VAR in the job environment, may be repeated @@ -133,6 +152,7 @@ Options: Examples: scripts/bee-mxqsub -m 956G -t 90m -j 256 libreoffice.be0 scripts/bee-mxqsub -t 8h SPIRV-Headers.be0 SPIRV-Tools.be0 glslang.be0 mesa.be0 + scripts/bee-mxqsub -p llvm-23.1.3-0 -p spirv-llvm-translator-23.1.2-0 mesalib.be0 EOF } @@ -176,6 +196,8 @@ while (( $# )); do --overlay-root) overlay_root=$2; shift 2 ;; --overlay-dir) overlay_dirs+=("$2"); shift 2 ;; --keep-overlay) keep_overlay=1; shift ;; + -p|--preinstall) preinstalls+=("$2"); shift 2 ;; + --preinstall-from) preinstall_from=$2; shift 2 ;; -k|--keep-going) keep_going=1; shift ;; -B|--beesh-arg) beesh_args+=("$2"); shift 2 ;; -E|--env) job_env+=("$2"); shift 2 ;; @@ -208,7 +230,14 @@ label=${pkg} (( ${#beefiles[@]} > 1 )) && label=${pkg}+$(( ${#beefiles[@]} - 1 )) if [ "${overlay}" = auto ]; then - if (( ${#beefiles[@]} > 1 )); then overlay=1; else overlay=0; fi + if (( ${#beefiles[@]} > 1 || ${#preinstalls[@]} )); then + overlay=1 + else + overlay=0 + fi +fi +if (( ${#preinstalls[@]} )) && ! (( overlay )); then + die "--preinstall needs the overlay, it cannot go with --no-overlay" fi (( ${#beesh_args[@]} )) || beesh_args=(-c) @@ -366,6 +395,7 @@ teardown_overlay() { # would be lost when it exits. install_into_overlay() { local beefile=$1 fullpkg + local -a archives # beeversion reads the '# BEE_VERSION' line, so the version comes from the # recipe rather than from the file name. @@ -373,27 +403,67 @@ install_into_overlay() { [ -n "${fullpkg}" ] \ || { echo "cannot read the version of ${beefile##*/}" >&2; return 1; } + archives=( "${pkgdir}/${fullpkg}".*.bee.tar.bz2 ) + if [ ! -e "${archives[0]}" ]; then + echo "no package archive ${pkgdir}/${fullpkg}.*.bee.tar.bz2" >&2 + return 1 + fi + if [ ${#archives[@]} -ne 1 ]; then + echo "several archives match ${fullpkg}: ${archives[*]}" >&2 + return 1 + fi + install_archive_into_overlay "${archives[0]}" +} + +# Install the package archive ${1} into the overlay, see install_into_overlay(). +install_archive_into_overlay() { + local archive=$1 + + # Given anything but an existing file, bee would look the word up as a + # package name in the repository and might install something else. + [ -f "${archive}" ] \ + || { echo "package archive ${archive} not found" >&2; return 1; } + "${sandbox[@]}" bash -c ' set -u - pkgdir=$1 fullpkg=$2 - archives=( "${pkgdir}/${fullpkg}".*.bee.tar.bz2 ) - if [ ! -e "${archives[0]}" ]; then - echo "no package archive ${pkgdir}/${fullpkg}.*.bee.tar.bz2" >&2 - exit 1 - fi - if [ ${#archives[@]} -ne 1 ]; then - echo "several archives match ${fullpkg}: ${archives[*]}" >&2 - exit 1 - fi - allpkg=$(beeversion --format="%A" "${archives[0]}") - bee update --force --remove-all "${archives[0]}" || exit 1 + archive=$1 + allpkg=$(beeversion --format="%A" "${archive}") + bee update --force --remove-all "${archive}" || exit 1 # bee-install exits 0 even where it refuses to install and only says so # on stderr, so ask whether the package really arrived. if ! bee list --installed --exact "${allpkg}" | grep -q .; then echo "installing ${allpkg} into the overlay failed" >&2 exit 1 fi - ' bee-mxqsub-install "${pkgdir}" "${fullpkg}" + ' bee-mxqsub-install "${archive}" +} + +# Echo the package archive --preinstall ${1} means: the file itself if it is +# one, else the single archive in ${preinstall_from} whose name-version-revision +# or name-version-revision.arch is ${1}. A prefix is not enough, so that +# llvm-23.1 does not quietly pick llvm-23.1.3-0. +resolve_preinstall() { + local spec=$1 candidate + local -a matches=() + + if [ -f "${spec}" ]; then + readlink -f "${spec}" + return 0 + fi + [[ ${spec} != */* ]] || die "package archive '${spec}' not found" + + for candidate in "${preinstall_from}/${spec}"*.bee.tar.bz2; do + [ -f "${candidate}" ] || continue + case ${spec} in + "$(beeversion --format=%F "${candidate}")"|"$(beeversion --format=%A "${candidate}")") + matches+=("${candidate}") ;; + esac + done + case ${#matches[@]} in + 0) die "no package '${spec}' in ${preinstall_from}, pass name-version-revision or an archive" ;; + 1) echo "${matches[0]}" ;; + *) die "several packages match '${spec}': ${matches[*]##*/}" ;; + esac } # The job phase runs on the cluster node, where the environment is ours to set. @@ -420,6 +490,9 @@ if (( run_job )); then else echo "=== overlay: no" fi + for archive in ${preinstalls[@]+"${preinstalls[@]}"}; do + echo "=== preinstall: ${archive}" + done export BEE_TMP_TMPDIR=${bee_tmpdir} export BEE_TMP_BUILDROOT=${buildroot} @@ -533,6 +606,21 @@ if (( run_job )); then status=0 declare -a failed=() + for archive in ${preinstalls[@]+"${preinstalls[@]}"}; do + echo "=== [preinstall] ${archive##*/}: $(date '+%F %T') starting" + install_archive_into_overlay "${archive}" + rc=$? + echo "=== [preinstall] ${archive##*/}: $(date '+%F %T') install into the overlay exited with ${rc}" + if (( rc )); then + status=${rc} + failed+=("${archive##*/}") + # Every bee-file was meant to build against the pre-installed + # packages, so none is built without them, --keep-going or not. + echo "=== stopping, ${#beefiles[@]} bee-file(s) not built" + beefiles=() + break + fi + done for (( i = 0; i < ${#beefiles[@]}; i++ )); do beefile=${beefiles[i]} echo "=== [$(( i + 1 ))/${#beefiles[@]}] ${beefile##*/}: $(date '+%F %T') starting" @@ -576,6 +664,13 @@ if [[ ${stdout} == */* ]]; then fi [ ! -e "${stdout}" ] || echo "Note: overwriting the existing log file" +# Resolve the packages to install up front, so that a typo fails now and not +# after the job has waited in the queue. +for (( i = 0; i < ${#preinstalls[@]}; i++ )); do + preinstalls[i]=$(resolve_preinstall "${preinstalls[i]}") || exit 1 + echo "Pre-install: ${preinstalls[i]}" >&2 +done + # Check the given working directory before staging, so that a typo does not # leave copies of the bee-files behind. if [ -n "${workdir}" ]; then @@ -637,6 +732,9 @@ if (( overlay )); then mxqsub+=(--overlay-dir="${dir}") done (( keep_overlay )) && mxqsub+=(--keep-overlay) + for archive in ${preinstalls[@]+"${preinstalls[@]}"}; do + mxqsub+=(--preinstall="${archive}") + done else mxqsub+=(--no-overlay) fi