From d6dee9a4bc4d5899168279a7dc894cab162e8a07 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Thu, 15 Oct 2026 06:44:16 +0200 Subject: [PATCH] bee-mxqsub: Add --preinstall to install built packages into the overlay Rebuilding one package against dependencies newer than the host's, for example mesa against the released LLVM 23.1.3, meant putting the whole dependency chain on the command line and rebuilding all of it, although the packages were already built and in /src/mariux/beeroot/packages. `-p, --preinstall PKG` installs such a package into the overlay before the first bee-file is built, with the same `bee update --force --remove-all` and `bee list --installed --exact` check the chain uses. PKG is a package archive, or name-version-revision[.arch], looked up in `--preinstall-from DIR` (default /src/mariux/beeroot/packages). Only an exact match counts, so `llvm-23.1` does not quietly pick llvm-23.1.3-0. The names are resolved when submitting, so a typo fails before the job waits in the queue: $ scripts/bee-mxqsub -d -p llvm-23.1 mesalib.be0 bee-mxqsub: no package 'llvm-23.1' in /src/mariux/beeroot/packages, pass name-version-revision or an archive A pre-installed package turns the overlay on even for a single bee-file, and goes with --no-overlay not at all. If installing one fails, no bee-file is built, --keep-going or not, as they were all meant to build against it: === [preinstall] broken-1.0-0.x86_64.bee.tar.bz2: 2026-10-15 06:43:05 install into the overlay exited with 1 === stopping, 1 bee-file(s) not built === preinstall-probe: failed: broken-1.0-0.x86_64.bee.tar.bz2 === preinstall-probe: 2026-10-15 06:43:05 done, exit 1 Tested in MXQ job 60600434 on platypus: giflib-4.1.6-0 by name, whose host has -1, and whois by archive path, then a throwaway recipe checking that `bee list` in the overlay shows giflib-4.1.6-0 and whois, and not giflib-4.1.6-1: === [preinstall] giflib-4.1.6-0.x86_64.bee.tar.bz2: 2026-10-15 06:42:33 install into the overlay exited with 0 === [preinstall] whois-5.6.6-0.x86_64.bee.tar.bz2: 2026-10-15 06:42:34 install into the overlay exited with 0 PROBE OK === [1/1] preinstall-probe.be0: 2026-10-15 06:42:35 beesh exited with 0 === preinstall-probe: 2026-10-15 06:42:36 done, exit 0 Assisted-by: claude-opus-5.5 (Claude Code) Co-Authored-By: Claude Opus 5.5 --- scripts/bee-mxqsub | 126 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 112 insertions(+), 14 deletions(-) diff --git a/scripts/bee-mxqsub b/scripts/bee-mxqsub index a1c28212e..52ad7b34a 100755 --- a/scripts/bee-mxqsub +++ b/scripts/bee-mxqsub @@ -22,6 +22,12 @@ # a single bee-file, where there is nothing to chain, so that the usual # one-package build is unchanged. # +# Packages that are already built need not be part of the chain: --preinstall +# installs their archives into the overlay before the first bee-file is built, +# so that rebuilding, say, mesa against a released LLVM newer than the host's +# does not rebuild LLVM as well. A pre-installed package turns the overlay on +# even for a single bee-file. +# # Note that the build tree lives in --bee-tmpdir, which defaults to the RAM # disk /dev/shm, so --memory has to cover the sources, the objects and the # build root, not just the compilers. The overlay lives there too, and holds a @@ -72,6 +78,8 @@ staged_in= overlay=auto overlay_root= keep_overlay=0 +# Where --preinstall looks up a package given as name-version-revision. +preinstall_from=/src/mariux/beeroot/packages keep_going=0 dry_run=0 run_job=0 @@ -79,6 +87,9 @@ declare -a beesh_args=() declare -a job_env=() declare -a overlay_dirs=() declare -a beefiles=() +# Package archives to install into the overlay before the first build. The +# submit phase resolves them to absolute paths, which the job phase is given. +declare -a preinstalls=() # Parallel to beefiles: the bee-file each entry was copied from, for the log. declare -a staged_from=() @@ -124,6 +135,14 @@ Options: --overlay-dir DIR host directory to overlay, may be repeated (default: /usr /etc) --keep-overlay do not remove the overlay when the job is done + -p, --preinstall PKG install the already built package PKG into the + overlay before the first bee-file is built, may be + repeated; PKG is a package archive or + name-version-revision[.arch], looked up in the + --preinstall-from directory; turns on --overlay + --preinstall-from DIR + where to look up --preinstall packages + (default: ${preinstall_from}) -k, --keep-going build the remaining bee-files after a failure -B, --beesh-arg ARG pass ARG to beesh, may be repeated (default: -c) -E, --env VAR=VALUE set VAR in the job environment, may be repeated @@ -133,6 +152,7 @@ Options: Examples: scripts/bee-mxqsub -m 956G -t 90m -j 256 libreoffice.be0 scripts/bee-mxqsub -t 8h SPIRV-Headers.be0 SPIRV-Tools.be0 glslang.be0 mesa.be0 + scripts/bee-mxqsub -p llvm-23.1.3-0 -p spirv-llvm-translator-23.1.2-0 mesalib.be0 EOF } @@ -176,6 +196,8 @@ while (( $# )); do --overlay-root) overlay_root=$2; shift 2 ;; --overlay-dir) overlay_dirs+=("$2"); shift 2 ;; --keep-overlay) keep_overlay=1; shift ;; + -p|--preinstall) preinstalls+=("$2"); shift 2 ;; + --preinstall-from) preinstall_from=$2; shift 2 ;; -k|--keep-going) keep_going=1; shift ;; -B|--beesh-arg) beesh_args+=("$2"); shift 2 ;; -E|--env) job_env+=("$2"); shift 2 ;; @@ -208,7 +230,14 @@ label=${pkg} (( ${#beefiles[@]} > 1 )) && label=${pkg}+$(( ${#beefiles[@]} - 1 )) if [ "${overlay}" = auto ]; then - if (( ${#beefiles[@]} > 1 )); then overlay=1; else overlay=0; fi + if (( ${#beefiles[@]} > 1 || ${#preinstalls[@]} )); then + overlay=1 + else + overlay=0 + fi +fi +if (( ${#preinstalls[@]} )) && ! (( overlay )); then + die "--preinstall needs the overlay, it cannot go with --no-overlay" fi (( ${#beesh_args[@]} )) || beesh_args=(-c) @@ -366,6 +395,7 @@ teardown_overlay() { # would be lost when it exits. install_into_overlay() { local beefile=$1 fullpkg + local -a archives # beeversion reads the '# BEE_VERSION' line, so the version comes from the # recipe rather than from the file name. @@ -373,27 +403,67 @@ install_into_overlay() { [ -n "${fullpkg}" ] \ || { echo "cannot read the version of ${beefile##*/}" >&2; return 1; } + archives=( "${pkgdir}/${fullpkg}".*.bee.tar.bz2 ) + if [ ! -e "${archives[0]}" ]; then + echo "no package archive ${pkgdir}/${fullpkg}.*.bee.tar.bz2" >&2 + return 1 + fi + if [ ${#archives[@]} -ne 1 ]; then + echo "several archives match ${fullpkg}: ${archives[*]}" >&2 + return 1 + fi + install_archive_into_overlay "${archives[0]}" +} + +# Install the package archive ${1} into the overlay, see install_into_overlay(). +install_archive_into_overlay() { + local archive=$1 + + # Given anything but an existing file, bee would look the word up as a + # package name in the repository and might install something else. + [ -f "${archive}" ] \ + || { echo "package archive ${archive} not found" >&2; return 1; } + "${sandbox[@]}" bash -c ' set -u - pkgdir=$1 fullpkg=$2 - archives=( "${pkgdir}/${fullpkg}".*.bee.tar.bz2 ) - if [ ! -e "${archives[0]}" ]; then - echo "no package archive ${pkgdir}/${fullpkg}.*.bee.tar.bz2" >&2 - exit 1 - fi - if [ ${#archives[@]} -ne 1 ]; then - echo "several archives match ${fullpkg}: ${archives[*]}" >&2 - exit 1 - fi - allpkg=$(beeversion --format="%A" "${archives[0]}") - bee update --force --remove-all "${archives[0]}" || exit 1 + archive=$1 + allpkg=$(beeversion --format="%A" "${archive}") + bee update --force --remove-all "${archive}" || exit 1 # bee-install exits 0 even where it refuses to install and only says so # on stderr, so ask whether the package really arrived. if ! bee list --installed --exact "${allpkg}" | grep -q .; then echo "installing ${allpkg} into the overlay failed" >&2 exit 1 fi - ' bee-mxqsub-install "${pkgdir}" "${fullpkg}" + ' bee-mxqsub-install "${archive}" +} + +# Echo the package archive --preinstall ${1} means: the file itself if it is +# one, else the single archive in ${preinstall_from} whose name-version-revision +# or name-version-revision.arch is ${1}. A prefix is not enough, so that +# llvm-23.1 does not quietly pick llvm-23.1.3-0. +resolve_preinstall() { + local spec=$1 candidate + local -a matches=() + + if [ -f "${spec}" ]; then + readlink -f "${spec}" + return 0 + fi + [[ ${spec} != */* ]] || die "package archive '${spec}' not found" + + for candidate in "${preinstall_from}/${spec}"*.bee.tar.bz2; do + [ -f "${candidate}" ] || continue + case ${spec} in + "$(beeversion --format=%F "${candidate}")"|"$(beeversion --format=%A "${candidate}")") + matches+=("${candidate}") ;; + esac + done + case ${#matches[@]} in + 0) die "no package '${spec}' in ${preinstall_from}, pass name-version-revision or an archive" ;; + 1) echo "${matches[0]}" ;; + *) die "several packages match '${spec}': ${matches[*]##*/}" ;; + esac } # The job phase runs on the cluster node, where the environment is ours to set. @@ -420,6 +490,9 @@ if (( run_job )); then else echo "=== overlay: no" fi + for archive in ${preinstalls[@]+"${preinstalls[@]}"}; do + echo "=== preinstall: ${archive}" + done export BEE_TMP_TMPDIR=${bee_tmpdir} export BEE_TMP_BUILDROOT=${buildroot} @@ -533,6 +606,21 @@ if (( run_job )); then status=0 declare -a failed=() + for archive in ${preinstalls[@]+"${preinstalls[@]}"}; do + echo "=== [preinstall] ${archive##*/}: $(date '+%F %T') starting" + install_archive_into_overlay "${archive}" + rc=$? + echo "=== [preinstall] ${archive##*/}: $(date '+%F %T') install into the overlay exited with ${rc}" + if (( rc )); then + status=${rc} + failed+=("${archive##*/}") + # Every bee-file was meant to build against the pre-installed + # packages, so none is built without them, --keep-going or not. + echo "=== stopping, ${#beefiles[@]} bee-file(s) not built" + beefiles=() + break + fi + done for (( i = 0; i < ${#beefiles[@]}; i++ )); do beefile=${beefiles[i]} echo "=== [$(( i + 1 ))/${#beefiles[@]}] ${beefile##*/}: $(date '+%F %T') starting" @@ -576,6 +664,13 @@ if [[ ${stdout} == */* ]]; then fi [ ! -e "${stdout}" ] || echo "Note: overwriting the existing log file" +# Resolve the packages to install up front, so that a typo fails now and not +# after the job has waited in the queue. +for (( i = 0; i < ${#preinstalls[@]}; i++ )); do + preinstalls[i]=$(resolve_preinstall "${preinstalls[i]}") || exit 1 + echo "Pre-install: ${preinstalls[i]}" >&2 +done + # Check the given working directory before staging, so that a typo does not # leave copies of the bee-files behind. if [ -n "${workdir}" ]; then @@ -637,6 +732,9 @@ if (( overlay )); then mxqsub+=(--overlay-dir="${dir}") done (( keep_overlay )) && mxqsub+=(--keep-overlay) + for archive in ${preinstalls[@]+"${preinstalls[@]}"}; do + mxqsub+=(--preinstall="${archive}") + done else mxqsub+=(--no-overlay) fi