run_sepia_stitching.sh

#!/bin/bash -l
set -e

. ./ssh_tools.sh

# Parse command line args
debug=false
rerun=false
dry_run=false
keepfiles=false
for i in "$@"
do
    case $i in
        --uuid=*)
            uuid="${i#*=}"
            shift
            ;;
        --rerun)
            rerun=true
            shift
            ;;
        --hostname=*)
            hostname="${i#*=}"
            shift
            ;;
        --sshcmd=*)
            sshcmd="${i#*=}"
            shift
            ;;
        --debug)
            debug=true
            shift
            ;;
        --dry-run)
            dry_run=true
            shift
            ;;
        --keepfiles)
            keepfiles=true
            shift
            ;;
        *)
            sepia="$i"
            shift
            ;;
    esac
done

if [[ -z ${sepia+x} ]]; then
    echo "sepia must be specified"
    exit 1
fi

if [[ $debug = true ]]; then
    set -x
fi

# Read config file
if [[ ! -f "$sepia.cfg" ]]; then
    echo "No config file"
    exit 1
fi
while IFS="=" read -r name value; do
    declare "$name=$value"
done < "$sepia.cfg"

# Parse local config file
while IFS="=" read -r name value; do
    declare "$name=$value"
done < "$HOME/.cuttleline.cfg"

# Check if hostname or sshcmd was specified
if [[ -z ${hostname+x} ]] && [[ -z ${sshcmd+x} ]]; then
    echo "hostname or sshcmd must be specified"
    exit 1
fi
if [[ -z ${sshcmd+x} ]]; then
    sshcmd="ssh $hostname"
fi

# Parse host config file
while IFS="=" read -r name value; do
    declare "$name=$value"
done < <(retry_ssh "cat ~/.cuttleline.cfg")

if [[ -z ${code_dir+x} ]]; then
    echo "code_dir must be specified"
    exit 1
fi
if [[ -z ${working_dir+x} ]]; then
    echo "working_dir must be specified"
    exit 1
fi
if [[ -z ${mpicmd+x} ]]; then
    echo "mpicmd must be specified"
    exit 1
fi

# Generate path for output files
if [[ -z ${outdir+x} ]]; then
    echo "outdir must be specified"
    exit 1
fi

# Make working dir on remote
if [[ -z ${uuid+x} ]]; then
    uuid=$(uuidgen)
fi
echo "Job Id: $uuid"
working_dir="$working_dir/$uuid"
retry_ssh "mkdir -p $working_dir"

basename=$(basename "$sepia")

# Determine files to be transferred
syncfiles="$sepia.syncfiles"
rm -f "$syncfiles"
sepia_dirs=($(ls -d "$outdir/$basename"*"/"))
for sepia_dir in "${sepia_dirs[@]}"; do
    basename=$(basename "$sepia_dir")
    echo "+ $basename" >> "$syncfiles"
    queenframe="$basename/${basename#*-}".queenframe
    echo "+ $queenframe" >> "$syncfiles"
done
echo "+ $(basename $sepia).stitching" >> "$syncfiles"
echo "+ $(basename $sepia).stitching.log" >> "$syncfiles"
echo "- *" >> "$syncfiles"

echo "Sending files"
retry_rsync -ap \
    --include-from="$syncfiles" \
    "$outdir"/ :"$working_dir" \
    --delete
echo "Submitting job"
cmd="$code_dir/run_sepia_stitching_remote.sh \
    $(if [[ ! -z ${partition+x} ]]; then
        echo --partition="\"$partition\""
    fi) \
    $(if [[ $debug = true ]]; then
        echo "--debug"
    fi) \
    $(if [[ $rerun = true ]]; then
        echo "--rerun-stitching"
    fi) \
    --mpicmd=\"$mpicmd\" \
    --stitching-slurm=\"$stitching_slurm\" \
    --stitching-args=\"$stitching_args\" \
    \"$working_dir/$(basename $sepia)\""
jobid=$(retry_ssh "exec /bin/sh -s" <<< $cmd \
    | sed -n 's/.*STITCHING_JOBID=\(\w*\).*/\1/p'; \
    if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
echo "Waiting for job"
while true; do
    queued_jobs=($(retry_ssh "squeue -o '%i' --noheader"))
    job_running=false
    for queued_job in "${queued_jobs[@]}"; do
        if [[ "$queued_job" = "$jobid" ]]; then
            job_running=true
        fi
    done
    if [[ "$job_running" = true ]]; then
        retry_rsync -ap \
            --include-from="$syncfiles" \
            :"$working_dir"/ "$outdir" \
            --delete
        sleep 900
    else
        break
    fi
done
echo "Syncing back files ..."
retry_rsync -ap \
    --include-from="$syncfiles" \
    :"$working_dir"/ "$outdir" \
    --delete
rm -r $syncfiles

# Remove working dir
echo "Cleaning up"
if [[ $dry_run = false ]] && [[ $keepfiles = false ]]; then
    retry_ssh "rm -rf $working_dir"
fi