run_pipeline_remote.sh

#!/bin/bash -l
set -e

function submit_job {
    while [[ 1 ]]; do
        num_jobs=$(squeue -u $USER --noheader | wc -l)
        if [[ $num_jobs -gt $max_queued ]]; then
            sleep $wait_for_queue
            continue
        fi
        sbatch "$@"
        return $?
    done
}

code_dir=$(dirname "$0")
sleep_between_submits=0.25
wait_for_queue=900

# Parse command line args
debug=false
rerun_registration=false
rerun_stitching=false
rerun_average=false
rerun_colourlabels=false
rerun_queenframe=false
rerun_cleanqueen=false
skip_chunks=false
skip_stitching=false
max_queued=299
for i in "$@"
do
    case $i in
        --debug)
            debug=true
            shift
            ;;
        --rerun-registration)
            rerun_registration=true
            shift
            ;;
        --rerun-stitching)
            rerun_stitching=true
            shift
            ;;
        --rerun-average)
            rerun_average=true
            shift
            ;;
        --rerun-colourlabels)
            rerun_colourlabels=true
            shift
            ;;
        --rerun-queenframe)
            rerun_queenframe=true
            shift
            ;;
        --rerun-cleanqueen)
            rerun_cleanqueen=true
            shift
            ;;
        --skip-chunks)
            skip_chunks=true
            shift
            ;;
        --skip-stitching)
            skip_stitching=true
            shift
            ;;
        --background=*)
            background="${i#*=}"
            shift
            ;;
        --foreground=*)
            foreground="${i#*=}"
            shift
            ;;
        --partition=*)
            partition="${i#*=}"
            shift
            ;;
        --mpicmd=*)
            mpicmd="${i#*=}"
            shift
            ;;
        --segmentation-slurm=*)
            segmentation_slurm="${i#*=}"
            shift
            ;;
        --segmentation-args=*)
            segmentation_args="${i#*=}"
            shift
            ;;
        --registration-slurm=*)
            registration_slurm="${i#*=}"
            shift
            ;;
        --registration-args=*)
            registration_args="${i#*=}"
            shift
            ;;
        --masterframe-slurm=*)
            masterframe_slurm="${i#*=}"
            shift
            ;;
        --masterframe-args=*)
            masterframe_args="${i#*=}"
            shift
            ;;
        --chunkaverage-slurm=*)
            chunkaverage_slurm="${i#*=}"
            shift
            ;;
        --chunkaverage-args=*)
            chunkaverage_args="${i#*=}"
            shift
            ;;
        --chunk-stitching-slurm=*)
            chunk_stitching_slurm="${i#*=}"
            shift
            ;;
        --chunk-stitching-args=*)
            chunk_stitching_args="${i#*=}"
            shift
            ;;
        --stitching-slurm=*)
            stitching_slurm="${i#*=}"
            shift
            ;;
        --stitching-args=*)
            stitching_args="${i#*=}"
            shift
            ;;
        --queenframe-slurm=*)
            queenframe_slurm="${i#*=}"
            shift
            ;;
        --queenframe-args=*)
            queenframe_args="${i#*=}"
            shift
            ;;
        --average-slurm=*)
            average_slurm="${i#*=}"
            shift
            ;;
        --average-args=*)
            average_args="${i#*=}"
            shift
            ;;
        --colourlabels-slurm=*)
            colourlabels_slurm="${i#*=}"
            shift
            ;;
        --colourlabels-args=*)
            colourlabels_args="${i#*=}"
            shift
            ;;
        --cleanqueen-slurm=*)
            cleanqueen_slurm="${i#*=}"
            shift
            ;;
        --cleanqueen-args=*)
            cleanqueen_args="${i#*=}"
            shift
            ;;
        --areas-slurm=*)
            areas_slurm="${i#*=}"
            shift
            ;;
        --areas-args=*)
            areas_args="${i#*=}"
            shift
            ;;
        --max-queued=*)
            max_queued="${i#*=}"
            shift
            ;;
        *)
            prefix=$i
            shift
            ;;
    esac
done

if [[ $debug = true ]]; then
    set -x
fi

# Setup file names
dirname=$(dirname "$prefix")
chunktimes="$prefix.chunktimes"
chunktimes_log="$prefix.chunktimes.log"
stitching="$prefix.stitching"
stitching_log="$prefix.stitching.log"
queenframe="$prefix.queenframe"
queenframe_log="$prefix.queenframe.log"
average="$prefix.average"
average_log="$prefix.average.log"
colourlabels="$prefix.colourlabels"
colourlabels_log="$prefix.colourlabels.log"
cleanqueen="$prefix.cleanqueen"
cleanqueen_log="$prefix.cleanqueen.log"
areas="$prefix.areas"
areas_log="$prefix.areas.log"
classifier="$dirname/classifier.clf"
colourica="$dirname/colourica.ica"
stitching_deps=""
chunk_stitching_deps=""
average_deps=""
chunkidx=1

#Sort chunks by filesize
if [[ $skip_chunks = false ]];then
sorted_chunktimes=$(while read -r start end video_start start_secs end_secs; do
    chunk_size=$(stat --printf="%s" $prefix-$start-$end.mp4)
    echo "$chunk_size $start $end $video_start $start_secs $end_secs"
done < $chunktimes | sort -k1 -rn)
else
sorted_chunktimes=$(while read -r start end video_start start_secs end_secs; do
    echo "0 $start $end $video_start $start_secs $end_secs"
done < $chunktimes)
fi

#Get all chunk filenames
while read -r chunk_size start end video_start start_secs end_secs; do
    # Setup filenames
    chunk[$chunkidx]="$prefix-$start-$end.mp4"
    chunk_video_start[$chunkidx]="$video_start"
    chunk_start_secs[$chunkidx]="$start_secs"
    chunk_end_secs[$chunkidx]="$end_secs"
    segmentation[$chunkidx]="$prefix-$start-$end.seg"
    segmentation_log[$chunkidx]="$prefix-$start-$end.seg.log"
    registration[$chunkidx]="$prefix-$start-$end.reg"
    registration_log[$chunkidx]="$prefix-$start-$end.reg.log"
    masterframe[$chunkidx]="$prefix-$start-$end.masterframe"
    masterframe_log[$chunkidx]="$prefix-$start-$end.masterframe.log"
    chunk_stitching[$chunkidx]="$prefix-$start-$end.cstitching"
    chunk_stitching_log[$chunkidx]="$prefix-$start-$end.cstitching.log"
    chunkaverage[$chunkidx]="$prefix-$start-$end.chunkaverage"
    chunkaverage_log[$chunkidx]="$prefix-$start-$end.chunkaverage.log"
    let chunkidx=chunkidx+1
done <<< "$sorted_chunktimes"
nchunks=${#chunk[@]}

if [[ $skip_chunks = false ]];then
    #Submit all segmentation jobs for the chunks
    for chunkidx in `seq 1 $nchunks`;
    do
        check_segmentation_jobid[$chunkidx]=$(submit_job \
            $(if [[ ! -z ${partition+x} ]]; then
                echo --partition="$partition"
            fi) \
            --job-name=check-segmentation \
            -o"$dirname/%j.out" \
            $code_dir/check_filedeps.sh \
            "$classifier" "${chunk[$chunkidx]}" \
            --output="${segmentation[$chunkidx]}" \
            | awk '{print $4}'; \
            if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
        segmentation_jobid[$chunkidx]=$(submit_job \
            $(if [[ ! -z ${partition+x} ]]; then
                echo --partition="$partition"
            fi) \
            $segmentation_slurm \
            --dependency=afterok:"${check_segmentation_jobid[$chunkidx]}" \
            --kill-on-invalid-dep=yes \
            --job-name=segmentation \
            -o"${segmentation_log[$chunkidx]}" \
            $code_dir/run_segmentation.sh \
            --mpicmd="$mpicmd" \
            --code-dir="$code_dir" \
            --classifier="$classifier" \
            --video-start="${chunk_video_start[$chunkidx]}" \
            --start="${chunk_start_secs[$chunkidx]}" \
            --end="${chunk_end_secs[$chunkidx]}" \
            --chunk="${chunk[$chunkidx]}" \
            --segmentation="${segmentation[$chunkidx]}" \
            --segmentation-args="$segmentation_args" \
            | awk '{print $4}';
            if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
        sleep $sleep_between_submits
    done

    #Submit all regisration jobs for the chunks
    for chunkidx in `seq 1 $nchunks`;
    do
        check_registration_jobid[$chunkidx]=$(submit_job \
            $(if [[ ! -z ${partition+x} ]]; then
                echo --partition="$partition"
            fi) \
            --dependency=afterany:"${segmentation_jobid[$chunkidx]}" \
            --kill-on-invalid-dep=yes \
            --job-name=check-registration \
            -o"$dirname/%j.out" \
            $code_dir/check_filedeps.sh \
            "${segmentation[$chunkidx]}" \
            --output="${registration[$chunkidx]}" \
            $(if [[ $rerun_registration = true ]]; then
                echo "--skip-mtime"
            fi) \
            | awk '{print $4}';
            if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
        registration_jobid[$chunkidx]=$(submit_job \
            $(if [[ ! -z ${partition+x} ]]; then
                echo --partition="$partition"
            fi) \
            $registration_slurm \
            --dependency=afterok:"${check_registration_jobid[$chunkidx]}" \
            --kill-on-invalid-dep=yes \
            --job-name=registration \
            -o"${registration_log[$chunkidx]}" \
            $code_dir/run_registration.sh \
            --mpicmd="$mpicmd" \
            --code-dir="$code_dir" \
            --segmentation="${segmentation[$chunkidx]}" \
            --background="$background" \
            --registration="${registration[$chunkidx]}" \
            --registration-args="$registration_args" \
            | awk '{print $4}';
            if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
        sleep $sleep_between_submits
    done

    #Submit all masterframe jobs for the chunks
    for chunkidx in `seq 1 $nchunks`;
    do
        check_masterframe_jobid[$chunkidx]=$(submit_job \
            $(if [[ ! -z ${partition+x} ]]; then
                echo --partition="$partition"
            fi) \
            --dependency=afterany:"${registration_jobid[$chunkidx]}":"${segmentation_jobid[$chunkidx]}" \
            --kill-on-invalid-dep=yes \
            --job-name=check-masterframe \
            -o"$dirname/%j.out" \
            $code_dir/check_filedeps.sh \
            "${registration[$chunkidx]}" \
            "${segmentation[$chunkidx]}" \
            --output="${masterframe[$chunkidx]}" \
            | awk '{print $4}';
            if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
        masterframe_jobid[$chunkidx]=$(submit_job \
            $(if [[ ! -z ${partition+x} ]]; then
                echo --partition="$partition"
            fi) \
            $masterframe_slurm \
            --dependency=afterok:"${check_masterframe_jobid[$chunkidx]}" \
            --kill-on-invalid-dep=yes \
            --job-name=masterframe \
            -o"${masterframe_log[$chunkidx]}" \
             $code_dir/run_masterframe.sh \
            --mpicmd="$mpicmd" \
            --code-dir="$code_dir" \
            --registration="${registration[$chunkidx]}" \
            --background="$background" \
            --masterframe="${masterframe[$chunkidx]}" \
            --masterframe-args="$masterframe_args" \
            | awk '{print $4}';
            if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
        sleep $sleep_between_submits
    done

    #Submit all chunkaverage jobs for the chunks
    for chunkidx in `seq 1 $nchunks`;
    do
        check_chunkaverage_jobid[$chunkidx]=$(submit_job \
            $(if [[ ! -z ${partition+x} ]]; then
                echo --partition="$partition"
            fi) \
            --dependency=afterany:"${registration_jobid[$chunkidx]}":"${segmentation_jobid[$chunkidx]}" \
            --kill-on-invalid-dep=yes \
            --job-name=check-chunkaverage \
            -o"$dirname/%j.out" \
            $code_dir/check_filedeps.sh \
            "${registration[$chunkidx]}" \
            "${segmentation[$chunkidx]}" \
            --output="${chunkaverage[$chunkidx]}" \
            | awk '{print $4}';
            if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
        chunkaverage_jobid[$chunkidx]=$(submit_job \
            $(if [[ ! -z ${partition+x} ]]; then
                echo --partition="$partition"
            fi) \
            $chunkaverage_slurm \
            --dependency=afterok:"${check_chunkaverage_jobid[$chunkidx]}" \
            --kill-on-invalid-dep=yes \
            --job-name=chunkaverage \
            -o"${chunkaverage_log[$chunkidx]}" \
             $code_dir/run_chunkaverage.sh \
            --mpicmd="$mpicmd" \
            --code-dir="$code_dir" \
            --registration="${registration[$chunkidx]}" \
            --chunkaverage="${chunkaverage[$chunkidx]}" \
            --chunkaverage-args="$chunkaverage_args" \
            | awk '{print $4}';
            if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
        sleep $sleep_between_submits
        chunk_stitching_deps+=":${masterframe_jobid[$chunkidx]}:${chunkaverage_jobid[$chunkidx]}"
        average_deps+=":${chunkaverage_jobid[$chunkidx]}"
    done

else
    stitching_deps+=""
    chunk_stitching_deps+=""
    average_deps+=""
fi

sleep $sleep_between_submits


#run stitching on individual chunks
if [[ $skip_stitching = false ]];then
    #Submit all stitching jobs for the chunks
    for chunkidx in `seq 1 $nchunks`;
    do
         check_chunk_stitching_jobid[$chunkidx]=$(submit_job \
            $(if [[ ! -z ${partition+x} ]]; then
                echo --partition="$partition"
            fi) \
            $(if [[ ! -z "$chunk_stitching_deps" ]]; then
                echo --dependency=afterany"$chunk_stitching_deps"
            fi) \
            --kill-on-invalid-dep=yes \
            --job-name=check-chunk-stitching \
            -o"$dirname/%j.out" \
            $code_dir/check_filedeps.sh \
            "${masterframe[$chunkidx]}" \
            --output="${chunk_stitching[$chunkidx]}" \
            $(if [[ $rerun_stitching = true ]]; then
                echo "--skip-mtime"
            fi) \
            | awk '{print $4}';
            if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
        chunk_stitching_jobid[$chunkidx]=$(submit_job \
            $(if [[ ! -z ${partition+x} ]]; then
                echo --partition="$partition"
            fi) \
            $chunk_stitching_slurm \
            --dependency=afterok:"${check_chunk_stitching_jobid[$chunkidx]}" \
            --kill-on-invalid-dep=yes \
            --job-name=chunk_stitching \
            -o"${chunk_stitching_log[$chunkidx]}" \
            $code_dir/run_chunk_stitching.sh \
            --mpicmd="$mpicmd" \
            --code-dir="$code_dir" \
            --src-chunk="${masterframe[$chunkidx]}" \
            --trg-chunks="${masterframe[*]}" \
            --chunk-stitching="${chunk_stitching[$chunkidx]}" \
            --chunk-stitching-args="$chunk_stitching_args" \
            | awk '{print $4}';
            if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
        sleep $sleep_between_submits
        stitching_deps+=":${chunk_stitching_jobid[$chunkidx]}"
    done

    check_stitching_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        $(if [[ ! -z "$stitching_deps" ]]; then
            echo --dependency=afterany"$stitching_deps"
        fi) \
        --kill-on-invalid-dep=yes \
        --job-name=check-stitching \
        -o"$dirname/%j.out" \
        $code_dir/check_filedeps.sh \
        --optional \
        "${chunk_stitching[*]}" \
        --output="$stitching" \
        $(if [[ $rerun_stitching = true ]]; then
            echo "--skip-mtime"
        fi) \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    stitching_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        $stitching_slurm \
        --dependency=afterok:$check_stitching_jobid \
        --kill-on-invalid-dep=yes \
        --job-name=stitching \
        -o"$stitching_log" \
        $code_dir/run_stitching.sh \
        --mpicmd="$mpicmd" \
        --code-dir="$code_dir" \
        --chunks="${chunk_stitching[*]}" \
        --stitching="$stitching" \
        --stitching-args="$stitching_args" \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    sleep $sleep_between_submits

    check_queenframe_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        --dependency=afterany:"$stitching_jobid" \
        --kill-on-invalid-dep=yes \
        --job-name=check-queenframe \
        -o"$dirname/%j.out" \
        $code_dir/check_filedeps.sh \
        "$stitching" \
        --output="$queenframe" \
        $(if [[ $rerun_queenframe = true ]]; then
            echo "--skip-mtime"
        fi) \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    queenframe_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        $queenframe_slurm \
        --dependency=afterok:"$check_queenframe_jobid" \
        --kill-on-invalid-dep=yes \
        --job-name=cleanqueen \
        -o"$queenframe_log" \
        $code_dir/run_queenframe.sh \
        --mpicmd="$mpicmd" \
        --code-dir="$code_dir" \
        --stitching="$stitching" \
        --queenframe="$queenframe" \
        --queenframe-args="$queenframe_args" \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    sleep $sleep_between_submits

    check_average_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        $(if [[ ! -z "$average_deps" ]]; then
            echo --dependency=afterany"$average_deps":"$stitching_jobid"
            else
            echo --dependency=afterany:"$stitching_jobid"
        fi) \
        --kill-on-invalid-dep=yes \
        --job-name=check-average \
        -o"$dirname/%j.out" \
        $code_dir/check_filedeps.sh \
        "$stitching" \
        --optional \
        "${chunkaverage[*]}" \
        --output="$average" \
        $(if [[ $rerun_average = true ]]; then
            echo "--skip-mtime"
        fi) \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    average_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        $average_slurm \
        --dependency=afterok:"$check_average_jobid" \
        --kill-on-invalid-dep=yes \
        --job-name=average \
        -o"$average_log" \
        $code_dir/run_average.sh \
        --mpicmd="$mpicmd" \
        --code-dir="$code_dir" \
        --stitching="$stitching" \
        --average="$average" \
        --average-args="$average_args" \
        --chunkaverage-ext="chunkaverage" \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    sleep $sleep_between_submits

    check_colourlabels_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        --dependency=afterany:"$average_jobid" \
        --kill-on-invalid-dep=yes \
        --job-name=check-colourlabels \
        -o"$dirname/%j.out" \
        $code_dir/check_filedeps.sh \
        "$average" \
        "$colourica" \
        --output="$colourlabels" \
        $(if [[ $rerun_colourlabels = true ]]; then
            echo "--skip-mtime"
        fi) \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    colourlabels_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        $colourlabels_slurm \
        --dependency=afterok:"$check_colourlabels_jobid" \
        --kill-on-invalid-dep=yes \
        --job-name=colourlabels \
        -o"$colourlabels_log" \
        $code_dir/run_colourlabels.sh \
        --mpicmd="$mpicmd" \
        --code-dir="$code_dir" \
        --average="$average" \
        --colourica="$colourica" \
        --colourlabels="$colourlabels" \
        --colourlabels-args="$colourlabels_args" \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    sleep $sleep_between_submits

    check_cleanqueen_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        --dependency=afterany:"$colourlabels_jobid" \
        --kill-on-invalid-dep=yes \
        --job-name=check-cleanqueen \
        -o"$dirname/%j.out" \
        $code_dir/check_filedeps.sh \
        "$queenframe" \
        --output="$cleanqueen" \
        $(if [[ $rerun_cleanqueen = true ]]; then
            echo "--skip-mtime"
        fi) \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    cleanqueen_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        $cleanqueen_slurm \
        --dependency=afterok:"$check_cleanqueen_jobid" \
        --kill-on-invalid-dep=yes \
        --job-name=cleanqueen \
        -o"$cleanqueen_log" \
        $code_dir/run_cleanqueen.sh \
        --mpicmd="$mpicmd" \
        --code-dir="$code_dir" \
        --queenframe="$queenframe" \
        --cleanqueen="$cleanqueen" \
        --cleanqueen-args="$cleanqueen_args" \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    sleep $sleep_between_submits

    check_areas_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        --dependency=afterany:"$cleanqueen_jobid" \
        --kill-on-invalid-dep=yes \
        --job-name=check-areas \
        -o"$dirname/%j.out" \
        $code_dir/check_filedeps.sh \
        "$cleanqueen" \
        --output="$areas" \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    areas_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        $areas_slurm \
        --dependency=afterok:"$check_areas_jobid" \
        --kill-on-invalid-dep=yes \
        --job-name=areas \
        -o"$areas_log" \
        $code_dir/run_areas.sh \
        --mpicmd="$mpicmd" \
        --code-dir="$code_dir" \
        --foreground="$foreground" \
        --cleanqueen="$cleanqueen" \
        --areas="$areas" \
        --areas-args="$areas_args" \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)

    final_cmd='#!/bin/bash'
    final_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        --dependency=afterok:"$areas_jobid":"$colourlabels_jobid" \
        --kill-on-invalid-dep=yes \
        --job-name=final \
        <<< $final_cmd \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    echo "FINAL_JOBID=$final_jobid"
    
else
    final_cmd='#!/bin/bash'
    final_jobid=$(submit_job \
        $(if [[ ! -z ${partition+x} ]]; then
            echo --partition="$partition"
        fi) \
        --dependency=afterok:"${chunkaverage_jobid[$chunkidx]}"\
        --kill-on-invalid-dep=yes \
        --job-name=final \
        <<< $final_cmd \
        | awk '{print $4}';
        if [[ "${PIPESTATUS[0]}" -eq "0" ]]; then true; else false; fi)
    echo "FINAL_JOBID=$final_jobid"
fi