Skip to content

bee-mxqsub: Add --preinstall to install built packages into the overlay #3484

Open
Paul Menzel (pmenzel) wants to merge 1 commit into
base: master
Choose a base branch
from
+112 −14
Open

bee-mxqsub: Add --preinstall to install built packages into the overlay #3484

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 112 additions & 14 deletions scripts/bee-mxqsub
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@
# a single bee-file, where there is nothing to chain, so that the usual
# one-package build is unchanged.
#
# Packages that are already built need not be part of the chain: --preinstall
# installs their archives into the overlay before the first bee-file is built,
# so that rebuilding, say, mesa against a released LLVM newer than the host's
# does not rebuild LLVM as well. A pre-installed package turns the overlay on
# even for a single bee-file.
#
# Note that the build tree lives in --bee-tmpdir, which defaults to the RAM
# disk /dev/shm, so --memory has to cover the sources, the objects and the
# build root, not just the compilers. The overlay lives there too, and holds a
Expand Down Expand Up @@ -72,13 +78,18 @@ staged_in=
overlay=auto
overlay_root=
keep_overlay=0
# Where --preinstall looks up a package given as name-version-revision.
preinstall_from=/src/mariux/beeroot/packages
keep_going=0
dry_run=0
run_job=0
declare -a beesh_args=()
declare -a job_env=()
declare -a overlay_dirs=()
declare -a beefiles=()
# Package archives to install into the overlay before the first build. The
# submit phase resolves them to absolute paths, which the job phase is given.
declare -a preinstalls=()
# Parallel to beefiles: the bee-file each entry was copied from, for the log.
declare -a staged_from=()

Expand Down Expand Up @@ -124,6 +135,14 @@ Options:
--overlay-dir DIR host directory to overlay, may be repeated
(default: /usr /etc)
--keep-overlay do not remove the overlay when the job is done
-p, --preinstall PKG install the already built package PKG into the
overlay before the first bee-file is built, may be
repeated; PKG is a package archive or
name-version-revision[.arch], looked up in the
--preinstall-from directory; turns on --overlay
--preinstall-from DIR
where to look up --preinstall packages
(default: ${preinstall_from})
-k, --keep-going build the remaining bee-files after a failure
-B, --beesh-arg ARG pass ARG to beesh, may be repeated (default: -c)
-E, --env VAR=VALUE set VAR in the job environment, may be repeated
Expand All @@ -133,6 +152,7 @@ Options:
Examples:
scripts/bee-mxqsub -m 956G -t 90m -j 256 libreoffice.be0
scripts/bee-mxqsub -t 8h SPIRV-Headers.be0 SPIRV-Tools.be0 glslang.be0 mesa.be0
scripts/bee-mxqsub -p llvm-23.1.3-0 -p spirv-llvm-translator-23.1.2-0 mesalib.be0
EOF
}

Expand Down Expand Up @@ -176,6 +196,8 @@ while (( $# )); do
--overlay-root) overlay_root=$2; shift 2 ;;
--overlay-dir) overlay_dirs+=("$2"); shift 2 ;;
--keep-overlay) keep_overlay=1; shift ;;
-p|--preinstall) preinstalls+=("$2"); shift 2 ;;
--preinstall-from) preinstall_from=$2; shift 2 ;;
-k|--keep-going) keep_going=1; shift ;;
-B|--beesh-arg) beesh_args+=("$2"); shift 2 ;;
-E|--env) job_env+=("$2"); shift 2 ;;
Expand Down Expand Up @@ -208,7 +230,14 @@ label=${pkg}
(( ${#beefiles[@]} > 1 )) && label=${pkg}+$(( ${#beefiles[@]} - 1 ))

if [ "${overlay}" = auto ]; then
if (( ${#beefiles[@]} > 1 )); then overlay=1; else overlay=0; fi
if (( ${#beefiles[@]} > 1 || ${#preinstalls[@]} )); then
overlay=1
else
overlay=0
fi
fi
if (( ${#preinstalls[@]} )) && ! (( overlay )); then
die "--preinstall needs the overlay, it cannot go with --no-overlay"
fi

(( ${#beesh_args[@]} )) || beesh_args=(-c)
Expand Down Expand Up @@ -366,34 +395,75 @@ teardown_overlay() {
# would be lost when it exits.
install_into_overlay() {
local beefile=$1 fullpkg
local -a archives

# beeversion reads the '# BEE_VERSION' line, so the version comes from the
# recipe rather than from the file name.
fullpkg=$(beeversion --format='%A' "${beefile}" 2>/dev/null)
[ -n "${fullpkg}" ] \
|| { echo "cannot read the version of ${beefile##*/}" >&2; return 1; }

archives=( "${pkgdir}/${fullpkg}".*.bee.tar.bz2 )
if [ ! -e "${archives[0]}" ]; then
echo "no package archive ${pkgdir}/${fullpkg}.*.bee.tar.bz2" >&2
return 1
fi
if [ ${#archives[@]} -ne 1 ]; then
echo "several archives match ${fullpkg}: ${archives[*]}" >&2
return 1
fi
install_archive_into_overlay "${archives[0]}"
}

# Install the package archive ${1} into the overlay, see install_into_overlay().
install_archive_into_overlay() {
local archive=$1

# Given anything but an existing file, bee would look the word up as a
# package name in the repository and might install something else.
[ -f "${archive}" ] \
|| { echo "package archive ${archive} not found" >&2; return 1; }

"${sandbox[@]}" bash -c '
set -u
pkgdir=$1 fullpkg=$2
archives=( "${pkgdir}/${fullpkg}".*.bee.tar.bz2 )
if [ ! -e "${archives[0]}" ]; then
echo "no package archive ${pkgdir}/${fullpkg}.*.bee.tar.bz2" >&2
exit 1
fi
if [ ${#archives[@]} -ne 1 ]; then
echo "several archives match ${fullpkg}: ${archives[*]}" >&2
exit 1
fi
allpkg=$(beeversion --format="%A" "${archives[0]}")
bee update --force --remove-all "${archives[0]}" || exit 1
archive=$1
allpkg=$(beeversion --format="%A" "${archive}")
bee update --force --remove-all "${archive}" || exit 1
# bee-install exits 0 even where it refuses to install and only says so
# on stderr, so ask whether the package really arrived.
if ! bee list --installed --exact "${allpkg}" | grep -q .; then
echo "installing ${allpkg} into the overlay failed" >&2
exit 1
fi
' bee-mxqsub-install "${pkgdir}" "${fullpkg}"
' bee-mxqsub-install "${archive}"
}

# Echo the package archive --preinstall ${1} means: the file itself if it is
# one, else the single archive in ${preinstall_from} whose name-version-revision
# or name-version-revision.arch is ${1}. A prefix is not enough, so that
# llvm-23.1 does not quietly pick llvm-23.1.3-0.
resolve_preinstall() {
local spec=$1 candidate
local -a matches=()

if [ -f "${spec}" ]; then
readlink -f "${spec}"
return 0
fi
[[ ${spec} != */* ]] || die "package archive '${spec}' not found"

for candidate in "${preinstall_from}/${spec}"*.bee.tar.bz2; do
[ -f "${candidate}" ] || continue
case ${spec} in
"$(beeversion --format=%F "${candidate}")"|"$(beeversion --format=%A "${candidate}")")
matches+=("${candidate}") ;;
esac
done
case ${#matches[@]} in
0) die "no package '${spec}' in ${preinstall_from}, pass name-version-revision or an archive" ;;
1) echo "${matches[0]}" ;;
*) die "several packages match '${spec}': ${matches[*]##*/}" ;;
esac
}

# The job phase runs on the cluster node, where the environment is ours to set.
Expand All @@ -420,6 +490,9 @@ if (( run_job )); then
else
echo "=== overlay: no"
fi
for archive in ${preinstalls[@]+"${preinstalls[@]}"}; do
echo "=== preinstall: ${archive}"
done

export BEE_TMP_TMPDIR=${bee_tmpdir}
export BEE_TMP_BUILDROOT=${buildroot}
Expand Down Expand Up @@ -533,6 +606,21 @@ if (( run_job )); then

status=0
declare -a failed=()
for archive in ${preinstalls[@]+"${preinstalls[@]}"}; do
echo "=== [preinstall] ${archive##*/}: $(date '+%F %T') starting"
install_archive_into_overlay "${archive}"
rc=$?
echo "=== [preinstall] ${archive##*/}: $(date '+%F %T') install into the overlay exited with ${rc}"
if (( rc )); then
status=${rc}
failed+=("${archive##*/}")
# Every bee-file was meant to build against the pre-installed
# packages, so none is built without them, --keep-going or not.
echo "=== stopping, ${#beefiles[@]} bee-file(s) not built"
beefiles=()
break
fi
done
for (( i = 0; i < ${#beefiles[@]}; i++ )); do
beefile=${beefiles[i]}
echo "=== [$(( i + 1 ))/${#beefiles[@]}] ${beefile##*/}: $(date '+%F %T') starting"
Expand Down Expand Up @@ -576,6 +664,13 @@ if [[ ${stdout} == */* ]]; then
fi
[ ! -e "${stdout}" ] || echo "Note: overwriting the existing log file"

# Resolve the packages to install up front, so that a typo fails now and not
# after the job has waited in the queue.
for (( i = 0; i < ${#preinstalls[@]}; i++ )); do
preinstalls[i]=$(resolve_preinstall "${preinstalls[i]}") || exit 1
echo "Pre-install: ${preinstalls[i]}" >&2
done

# Check the given working directory before staging, so that a typo does not
# leave copies of the bee-files behind.
if [ -n "${workdir}" ]; then
Expand Down Expand Up @@ -637,6 +732,9 @@ if (( overlay )); then
mxqsub+=(--overlay-dir="${dir}")
done
(( keep_overlay )) && mxqsub+=(--keep-overlay)
for archive in ${preinstalls[@]+"${preinstalls[@]}"}; do
mxqsub+=(--preinstall="${archive}")
done
else
mxqsub+=(--no-overlay)
fi
Expand Down