#!/bin/bash -e

DEFAULT_WINDOW=43200  #seconds
TRACEMERGE=${TRACEMERGE:-/usr/local/bin/tracemerge} #path to tracemerge
input_format=erf
output="erf:/dev/stdout"
holddir=/tmp
MUTEXFILE="$HOME/$(basename $0).lock" #need mutual exclusion

hash lockfile 2>/dev/null || {
    echo "ERROR: Need 'lockfile' utility in the PATH for execution."
    echo "ERROR: it can be installed via procmail package"
    exit 1
} >&2
 
usage() {
    cat >&2 <<EOF
    USAGE: `basename $0` [OPTIONS] dir1 [dir2 [...]]
 Merge same "windows" of erf files from dir1 dir2, etc and write the
 result to stdout (or file specified in -o option)
	where:
            <OPTIONS>           can be:
            -H <holddir>        move files in-processing to holddir/d1, hoddir/d2, etc.,
                                where d1, d2, ... are basenames of dir1, dir2
                                defaults to $holddir, but should be ideally on the same
                                partition as dir1, etc.
            -I <fmt>            input format default ($input_format), used for URIs
            -L <lockfile>       use lockfile for mutual exclusion (default: $MUTEXFILE)
            -l                  use localtime instead of default utc
            -o <uri>            write merged streams to the file (default $output)
                                (use e.g. pcapfile:/dev/stdout for pcap format output)
            -r <regex>          regex for input files (should start with YYYYMMDD-HHMMSS)
            -t <tracemerge_opt> pass these options to tracemerge
            -w <int>            window size (in seconds) default: $DEFAULT_WINDOW
            -O <int>            offset to the real window boundary (default to 0)
                                if this is 60, then windowing will be performed at
                                1m, 12h1m, etc.
 WARNING: this script's lowest granularity is a file.  It's NOT going to split a file
 into parts if a window boundary is in the middle of the file.  The WHOLE file will be
 in the first window.  If this is undesirable, don't use this script.
EOF
}

tracemerge_opts=""

WINDOW=$DEFAULT_WINDOW
OFFSET=0
use_utc="-u" #option to date
find_regex='.*/[0-9]{8}-[0-9]{6}-[0-9]+[^/]*$'
while getopts "hH:I:L:lo:r:w:O:" opt; do
    case $opt in
	( I ) input_format=$OPTARG; ;;
	( L ) MUTEXFILE=$OPTARG; ;;
	( H ) holddir=$OPTARG; ;;
	( l ) use_utc=""; ;; #use localtime
	( o ) output=$OPTARG; ;;
	( r ) find_regex= $OPTARG; ;;
	( t ) tracemerge_opts=$OPTARG; ;;
	( w ) WINDOW=$OPTARG; ;;
	( O ) OFFSET=$OPTARG; ;;

	( h|* ) usage; exit 0; ;;
    esac
done
shift $((OPTIND-1));

[ $# == "0" ] && { usage; exit 1; }

DIRS=$@
trap "rm -f $MUTEXFILE" EXIT HUP

#translate timestamp of a file, e.g. 
#+   name:   YYYYMMDD-HHMMSS-
#+   hexoff: 0   4 6  9 b d
#+and set to FILE_WINDOW to WINDOW index
file2FILE_WINDOW() {
	local _fn=$(basename $1)
	local _ep=$(date +%s $use_utc --date="${_fn:0:4}-${_fn:4:2}-${_fn:6:2} ${_fn:9:2}:${_fn:11:2}:${_fn:13:2}")
	FILE_WINDOW=$(( (_ep-OFFSET)/WINDOW ))
}


#first acquire mutex, or else exit
while true; do
    if lockfile -r5 -! "$MUTEXFILE" 2>/dev/null ; then
        #mutex file exists, check it is not stale
	    read pid < $MUTEXFILE
	    echo "WARN: mutex lock exists pid:$pid" >&2	
	    if [ -n "$pid" ] && kill -0 "$pid"; then
	        if [ $(ps -p $pid -ocomm --no-headers) == $(basename $0) ]; then
		        echo "WARN: another monitor is running, exitting" >&2
		        exit 1
	        fi
	    fi
	    echo "WARN: removing stale lock" >&2
	    rm -f $MUTEXFILE
	    #try again
    else
	    break
    fi
done
chmod +w $MUTEXFILE
echo $$ >$MUTEXFILE

#Fill 3 arrays with elements per directory:
list_dn=() #directory names
list_fs=() #file names from each directory
list_wi=() #current window numbers per directory
i=0
for d in $DIRS; do
    files=$(find $d -regextype posix-extended \
	             -regex $find_regex -print | sort)
    cnt=$(echo $files | wc -w)
    (( cnt < 2 )) && exit 0 #need at least 2 files

    #pick files for processing from each directory

    #get first and last files (in time-stamp order)
    read f1 fN < <(echo "$files" | sed -n '1h; ${ H; x; s/\n/ /g; p}')

    file2FILE_WINDOW $f1; f1_win=$FILE_WINDOW
    file2FILE_WINDOW $fN; fN_win=$FILE_WINDOW

    (( f1_win < fN_win )) || {
	    echo "INFO: not enough data in $d (all files fall in same window); wait then run again" >&2
	    exit 0
    }

    #save dir, window index, and files from f1...
    list_dn[$i]=$d
    list_wi[$i]=$f1_win
    nfiles=0
    for fi in $files; do
	    file2FILE_WINDOW $fi
	    [ $FILE_WINDOW != $f1_win ] && break
	    list_fs[$i]+="$fi "
	    let '++nfiles'
    done

    (( f1_win + 1 < fN_win )) && {
	    if [ "$nfiles" == 1 ]; then
	        echo "ERROR: one file spans more than one window; increase window or make files smaller" >&2 
	        exit 1;
	    else
	        echo "WARN: more than one window of data in $d/, proccessing only the first one" >&2
	    fi
    }
    let '++i'
done #for d in $DIRS

N=$i

#if we got here, it means we have enough data in all DIRS.
#+Check if these data belong to the same window (good) or different (bad)
for ((i=1; i<N; ++i)); do
    if [ "${list_wi[$i]}" != "${list_wi[0]}" ]; then
	    echo "ERROR: mismatched windows in directories ${list_dn[$i]}/ and ${list_dn[0]}/" >&2
	    echo "       probably need to remove old data" >&2
	    #XXX not sure what the best way to handle this is. probably remove old data (lowest window) and restart itself
	    exit 1
    fi
done

#OK, all data is from the same window, move files into holddir first, then merge and write output
[ -d "$holddir" ] || {
    if ! mkdir "$holddir"; then
	    echo "ERROR: holddir '$holddir' does not exist and cannot be created" >&2
	    exit 1
    fi
}

ARGS=
list_hfs=() #list of files in holddir, relative to holddir
for ((i=0; i<N; ++i)); do
    bd=$(basename "${list_dn[$i]}")
    [ -d $holddir/$bd ] || {
	    if ! mkdir $holddir/$bd; then
	        echo "ERROR: holddir subdir '$holddir/$bd' does not exist and cannot be created" >&2
	        exit 1
	    fi
    }
    for f in ${list_fs[$i]}; do
	    bf=$(basename $f)
	    hf=$holddir/$bd/$bf
	    mv $f $hf || { echo "ERROR: can't move file $f to $hf" >&2; exit 1; }
	    list_hfs[$i]+="$bf "
    done
	
    #build lists of arguments
    #xxx need to be very careful with escapes because of later eval
    ARGS+="$input_format:<(for f in \${list_hfs[$i]}; do cat $holddir/$bd/\$f; done) "
done

#release mutex and merge
#trap ":" EXIT HUP
rm -f $MUTEXFILE

eval $TRACEMERGE $tracemerge_opts $output $ARGS

#remove files if successful
if [ "$?" == 0 ]; then
    for ((i=0; i<N; ++i)); do
	    bd=$(basename "${list_dn[$i]}")
	    ( cd $holddir/$bd && rm -f ${list_hfs[$i]} )
    done
    exit 0
else
    echo "TRACEMERGE pipeline failed, exiting, leaving unmerged files in holddir '$holddir'" >&2
    exit 1
fi
