predictworker.sh

#!/bin/bash

script_name=$(basename "$0")
script_dir=$(dirname "$0")
# shellcheck source=commonfunctions.sh
source "${script_dir}/commonfunctions.sh"

version="???"
waitinterval="1"

if [ -f "$script_dir/VERSION" ] ; then
    version=$(cat "$script_dir"/VERSION)
fi

gpu="all"

function usage()
{
    echo "usage: $script_name [-h] [--gpu GPU] [--waitinterval WAIT]
                      predictdir

              Version: $version

              Runs caffepredict.sh in a serial fashion
              waiting for the next available package
              to process. This script uses predict.config 
              file to obtain location of trained model 
              and image data

positional arguments:
  predictdir           Predict directory generated by
                       runprediction.sh

optional arguments:
  -h, --help           show this help message and exit
  --gpu                Which GPU to use, can be a number ie 0 or 1 or
                       all to use all GPUs (default $gpu)
  --waitinterval       Number of seconds to wait between checking
                       for number of completed packages 
                       (default $waitinterval)

    " 1>&2;
    exit 1;
}

TEMP=$(getopt -o h --long "help,waitinterval:,gpu:" -n "$0" -- "$@")
eval set -- "$TEMP"

while true ; do
    case "$1" in
        -h ) usage ;;
        --help ) usage ;;
        --gpu ) gpu=$2 ; shift 2 ;;
        --waitinterval ) waitinterval=$2 ; shift 2 ;;
        --) shift ; break ;;
    esac
done

if [ $# -ne 1 ] ; then
  usage
fi

out_dir=$1

echo ""

predict_config="$out_dir/predict.config"

parse_predict_config "$predict_config"

if [ $? != 0 ] ; then
    fatal_error "$out_dir" "ERROR parsing $predict_config" 2
fi

echo "Running Prediction"
echo ""

echo "Trained Model Dir: $trained_model_dir"
echo "Image Dir: $img_dir"
echo "Models: $model_list"
echo "Speed: $aug_speed"
echo "GPU: $gpu"
echo ""

package_proc_info="$out_dir/augimages/package_processing_info.txt"

if [ ! -s "$package_proc_info" ] ; then
    fatal_error "$out_dir" "ERROR $package_proc_info not found" 7
fi

parse_package_processing_info "$package_proc_info"

space_sep_models=$(get_models_as_space_separated_list "$model_list")

for model_name in $(echo "$space_sep_models") ; do
  
    (( cntr=1 ))
    for CUR_PKG in $(seq -w 001 "$num_pkgs") ; do
        for CUR_Z in $(seq -w 01 "$num_zstacks") ; do
            package_name=$(get_package_name "$CUR_PKG" "$CUR_Z")
            Z="$out_dir/augimages/$model_name/$package_name"
            out_pkg="$out_dir/$model_name/$package_name"
            
            # caffepredict.sh creates PREDICTDONE when done
            # so check for that file before running
            if [ -f "$out_pkg/PREDICTDONE" ] ; then
                echo "  Found $out_pkg/PREDICTDONE Prediction completed. Skipping..."
                continue
            fi
   
            # StartPostprocessing.py creates DONE when done
            # so check for that file before running
            if [ -f "$out_pkg/DONE" ] ; then
                echo "  Found $out_pkg/DONE Postprocessing completed. Skipping..."
                continue
            fi
            echo "For model $model_name preprocessing $package_name $cntr of $tot_pkgs"
            res=$(wait_for_preprocess_to_finish_on_package "$out_dir" "$Z" "$waitinterval")
            if [ "$res" == "killed" ] ; then
                echo "KILL.REQUEST file found. Exiting"
                exit 1
            fi
            echo "Running prediction on $model_name $package_name"
            /usr/bin/time -p "${script_dir}"/caffepredict.sh --gpu "$gpu" "$trained_model_dir/$model_name/trainedmodel" "$Z" "$out_pkg"
            ecode=$?
            if [ $ecode != 0 ] ; then
                fatal_error "$out_dir" "ERROR, a non-zero exit code ($ecode) was received from: caffepredict.sh" 4
            fi
            (( cntr+=1 ))
        done
    done
done

echo ""
echo "Prediction has completed."
echo ""