#!/bin/bash

# $Id$

#  This script is meant to be the "front end" for the user running LANDER
#+ jobs.  Typically it is run from user's crontab(5).

#  Usage:
#+     lander_dispatcher.sh <input_queue> <processor>
#+ note that the user must specify a full path to myscript and it should be
#+ under /home/samfs-00.
#+ The last argument signifies if the user is getting host anonymized data
#+ or fully anonymized data.  If the user is getting both, two dispatchers
#+ can be scheduled.



LANDER_CONFIG=~plander/lander.conf
#read config
if [ -e $LANDER_CONFIG ]; then
    . $LANDER_CONFIG
else
    echo "ERROR: Config file $CONFIG does not exist" >&2
    exit 1
fi

#arguments
max_jobs=$LANDER_USERJOB_MAXJOBS
max_wait=0
verbose=
use_failqueue=
opt_distribute=
while getopts "hDFi:n:uw:j:r:" opt; do
    case $opt in
        ( h ) usage; exit 0 ;;
        ( D ) opt_distribute="-D" ;;
        ( F ) use_failqueue="-F" ;;
        ( i ) INDIR=$OPTARG ;;
        ( j ) jobspercore=$OPTARG ;;
        ( n ) max_jobs=$OPTARG ;;
        ( p ) INPROCESSDIR=$OPTARG ;;
        ( r ) FILE_REGEX=$OPTARG ;;
        ( s ) SKIPOOO="yes" ;;
        ( u ) UNORDERED="yes" ;;
        ( v ) verbose="yes" ;;
        ( w ) max_wait=$OPTARG ;;
        ( h|* ) usage; exit 0 ;;
    esac
done
shift $((OPTIND-1))

THISSCRIPT=$0
INPUTQUEUE=$1            # arg1: anonymization type := (raw|full|host|flow)
USERSCRIPT=$2            # arg2: user script (executable)


MUTEX=~/.user_dispatcher-mutex-$ANONTYPE
if [ -e "$MUTEX" ]; then
    read pid < $MUTEX
    if [ -n "$pid" ] && kill -0 $pid >&/dev/null; then
        echo "WARN: another user_dispatcher is running"
        exit 1
    fi
fi
echo $$ >$MUTEX

#xxx should be in config
file_regex=${FILE_REGEX:-'[0-9]{8}-[0-9]{6}-[0-9]+[^/]*$'}

usage () {
    cat >&2 <<EOF
    USAGE: $THISSCRIPT <OPTIONS> <input_queue> <user_script>
    where:
           <input_queue>        user's input queue
           <user_script>        path the the processor

           <OPTIONS>            can be:
                 -D             distribute result to connected queues
                 -F             use FAIL queue associated with this <input_queue>
                                move files there if processing fails; otherwise files
                                will stay in the "reserved" state.
                 -j <jobs>      Jobs per core times 100; i.e. 150 will run 1.5*ncores
                 -n N           Run no more than N concurrently 
                                (defaults to $LANDER_USERJOB_MAXJOBS)
                 -r <regex>     Regular expression for incoming files
                 -s             Skip out of order files (only useful with -n 1)
                 -u             Unordered: no sequence numbers
                 -v             verbose: do additional sanity checks and log the results
                 -w X           Require that the modtime at the head of the queue be 
                                at least X seconds in the past (defaults to 0)
EOF
}

if [ -z "$LOGNAME" ]; then
    echo "LOGNAME isn't set" >&2
    exit 1
fi

if [ -z "$INPUTQUEUE" ]; then
    echo "input queue is not set" >&2
    exit 1
fi

if [ -z "$USERSCRIPT" ]; then
    echo "userscript arguement is missing" >&2
    usage
    exit 1
fi

if [ ! -x "$USERSCRIPT" ] && [ "$(type -t $USERSCRIPT)" != "function" ]; then
    echo "userscript $USERSCRIPT is not executable" >&2
    exit 1
fi


if [ "$max_jobs" == "1" ]; then
    jobspercore="" #one job means only one process!
fi

USERSCRIPT_base=${USERSCRIPT##*/}

LOGFILE=${LANDER_USERDIR}/${USERSCRIPT_base}-d.log #write to this log in user's home

exec >> $LOGFILE 2>&1

lander_set_DATE

if [ -n "$verbose" ]; then
    lander_queue_is_active $INPUTQUEUE || {
        echo "WARNING: queue $INPUTQUEUE is not active"
        echo "         use user_lander.sh to enable it"
    } >&2
fi

lander_set_JOBS_RUNNING $LOGNAME "$LANDER_PROCESS_WORKER" "$USERSCRIPT"
CURRENTRUNNING=$JOBS_RUNNING

# Piping to 'head' here cuts down on find's exec time
#+for very large directories by breaking the pipe early.
lander_queue_size $INPUTQUEUE || exit 0

NUMFILES=$_lander_qsize

echo "$DATE files:$NUMFILES workersbefore:$CURRENTRUNNING workersafter:$JOBS_RUNNING/max:$max_jobs"

[ "$NUMFILES" == "0" ] && exit 0

if [ $CURRENTRUNNING -lt $max_jobs ]; then
    #USER_WORKER_SCRIPT will loop, executing USERSCRIPT on all files
    #+it will be responsibility of the user/USERSCRIPT to clean up.

    [ -n "$SKIPOOO" ] && args="-s"
    
    [ -n "$jobspercore" ] && args="$args -j $jobspercore"

    [ -n "$UNORDERED" ] && args="$args -u"

    [ -n "$FILE_REGEX" ] && args="$args -r '$FILE_REGEX'"

    args="$args -w $max_wait $use_failqueue $opt_distribute $INPUTQUEUE $USERSCRIPT"

    lander_create_job "$LANDER_PROCESS_WORKER" "$args" $LANDER_USERJOB_NICE

    lander_set_JOBS_RUNNING $LOGNAME "$LANDER_PROCESS_WORKER" "$USERSCRIPT"

    echo "$DATE files:$NUMFILES workersbefore:$CURRENTRUNNING workersafter:$JOBS_RUNNING/max:$max_jobs"
fi

rm -f $MUTEX
