
#Global variables
#cache of the current queue
_cached_queue=""
#current queue length
_qsize=0
#disambiguator: different hosts/processes
_salt="$(hostname -s).$$"

#XXX these should be in lander.conf
RAWFSTYPE=hdfs #hdfs/fs to lander.conf
RAWDIR_HDFS=/user/slander/lander/raw
RAWDIR_FS=$RAWDIR #xxx
RESERVED_HDFS=/user/slander/lander/tmp
RESERVED_FS=$TMPDIR #xxx

# Set _qsize to the number of files waiting in the queue
# Caches the current queue state.
lander_queue_size() {
    if [ -z "$_cached_queue" ]; then
	lander_queue_size_$RAWFSTYPE
    fi
    _qsize=$(echo "$_cached_queue" | wc -l)
    return 0
}
lander_queue_size_hdfs() {
    local files=$( $HDFS -lsr $RAWDIR_HDFS | awk '{print $8}' |  egrep '\/[0-9]{8}-[0-9]{6}-[0-9]+$' )
    [ -z "$files" ] && return 0
    # sort files by name (but ignore the directories)
    local fnkey=`echo "/x$files" 2>/dev/null | head -1 | tr '/' ' ' | wc -w` #should be num of '/' + 1
    _cached_queue=`echo "$files" | sort -t '/' -k $fnkey`
    return 0
}
lander_queue_size_fs() {
    local files=$(find $RAWDIR_FS -regextype posix-extended \
	-regex '.*\/[0-9]{8}-[0-9]{6}-[0-9]+$' -print 2>/dev/null)
    # sort files by name (but ignore the directories)
    local fnkey=$(echo "/x$files" 2>/dev/null | \
	head -1 | tr '/' ' ' | wc -w) #should be num of '/' + 1
    _cached_queue=$(echo "$files" | sort -t '/' -k $fnkey)
}

#clear cached state of the queue
lander_queue_flush_cache() {
    _cached_queue=""
}


# Pop one file from the head of the queue, cache the current queue state.
# Sets 'reserved_local' to the file name on the local FS.
# Returns true if success or false if the queue is empty. 
#
# we want to be extremely careful not to read/write to shared fs if it
# can be helped.  So all our processing components read/write to /tmp
# and then copy results to shared fs, while retaining the result
# locally.
lander_queue_reserve_top() {
    local localtmp=$1
    lander_queue_reserve_top_$RAWFSTYPE $localtmp
}
lander_queue_reserve_top_hdfs() {
    local localtmp=$1
    while true; do
	lander_queue_size_hdfs || \
	    { reserved_local=""; return 1; }
        local firstinq=$( echo "$_cached_queue" | sed -n '1p;q' )
        local restofq=$( echo "$_cached_queue" | sed 1d )
        _cached_queue=$restofq

        local file=$(basename $firstinq)
        local lander=$(basename $(dirname $fistinq))
	local reservation="$RESERVED_HDFS/$lander/$file-$_salt"
	#check if next seq number has been written
	local seqnum=${file##*-}
	local nextseqnum=`printf "%08d" $(((10#$seqnum+1)%100000000))`
	if ! $HDFS -test -e $RAWDIR_HDFS/$lander/*-$nextseqnum 2>/dev/null; then
	    while true; do
		local now_s=$(date +%s)
		local modtime_ms=$($HDFS -stat %Y $fistinq)
		if ((now_s*1000 - modtime_ms >= 300000)); then
	            #file at the head hasn't been modified for a while
		    break;
		else
		    sleep 60s
		fi
	    done
	    if ! $HDFS -test -e $RAWDIR_HDFS/$lander/*-$nextseqnum 2>/dev/null; then
		echo "WARN: working on $firstinq, while $nextseqnum hasn't been delivered" >&2
	    fi
	fi
        $HDFS -mv $fistinq $reservation 2>/dev/null \
            || { sleep 10; continue; }
	$HDFS -test -e $reservation 2>/dev/null \
	    || { sleep 10; continue; }
        mkdir -p $localtmp/$lander 2>/dev/null
        reserved_local="$localtmp/$lander/$file"
        $HDFS -get $reservation $reserved_local || {
	    echo "ERR: error hdfs -get $reservation $reserved_local" >&2
	    continue
	}
        return 0
    done
}
lander_queue_reserve_top_fs() {
    local localtmp=$1
    while true; do
	lander_queue_size_fs || { reserved_local=""; return 1; }
        local firstinq=$( echo "$_cached_queue" | sed -n '1p;q' )
        local restofq=$( echo "$_cached_queue" | sed 1d )
        _cached_queue=$restofq

        local file=$(basename $firstinq)
        local lander=$(basename $(dirname $fistinq))
	local reservation="$RESERVED_FS/$lander/$file-$_salt"
	#check if next seq number has been written
	local seqnum=${file##*-}
	local nextseqnum=`printf "%08d" $(((10#$seqnum+1)%100000000))`
	if ! [ -e $RAWDIR_HDFS/$lander/*-$nextseqnum 2>/dev/null ]; then
	    while true; do
		local now=$(date +%s)
		local modtime=$(/usr/bin/stat --format %Y $fistinq 2>/dev/null)
		if ((now - modtime >= 300)); then
	            #file at the head hasn't been modified for a while
		    break;
		else
		    sleep 60s
		fi
	    done
	    if ! [ -e $RAWDIR_HDFS/$lander/*-$nextseqnum 2>/dev/null ]; then
		echo "WARN: working on $firstinq, while $nextseqnum hasn't been delivered" >&2
	    fi
	fi
        mv $fistinq $reservation 2>/dev/null \
            || continue
	# sometimes mv exit status is zero, but move in fact
	#+fails (probably because of NFS attribute caching), 
	#+so we have to check for existence of the target.
	[ -e "$reservation" ] \
	    || continue
	mkdir -p $localtmp/$lander 2>/dev/null
	reserved_local="$localtmp/$lander/$file"
	cp -p $reservation $reserved_local \
	    || { echo "Error copying $reservation to $reserved_local"; continue; }
        return 0
    done
}


# Put previously reserved with reserve_top_hdfs() file back into the queue.
# This can be useful e.g. if the file is to be processed by a different agent.
lander_queue_unreserve() {
    local file=$1
    lander_queue_unreserve_$RAWFSTYPE $file
}
lander_queue_unreserve_hdfs() {
    local file=$1
    local lander=$(basename $(dirname $file))
    local reservation="$RESERVED_HDFS/$lander/$file-$_salt"

    $HDFS -mv $reservation $RAWDIR_HDFS/$lander || {
	echo "Cannot move $reservation back to $RAWDIR_HDFS/$lander" >&2
	return 1
    }
    rm -f $file
    return 0
}
lander_queue_unreserve_fs() {
    local file=$1
    local lander=$(basename $(dirname $file))
    local reservation="$RESERVED_FS/$lander/$file-$_salt"
    mv $reservation $RAWDIR_FS/$lander || {
	echo "Cannot move $reservation back to $RAWDIR_FS/$lander" >&2
	return 1
    }
    rm -f $file
}

# The file has been entirely processed, remove it from HDFS, as well as from
# local tmp.
lander_queue_release() {
    local file=$1
    lander_queue_release_$RAWFSTYPE $file
}
lander_queue_release_hdfs() {
    local file=$1
    local lander=$(basename $(dirname $file))
    local reservation="$RESERVED_HDFS/$lander/$file-$_salt"
    rm -f $file
    $HDFS -rm $reservation || {
        echo "Could not remove reservation $reservation" >&2
	return 1
    }
    return 0
}
lander_queue_release_fs() {
    local file=$1
    local lander=$(basename $(dirname $file))
    local reservation="$RESERVED_FS/$lander/$file-$_salt"
    rm -rf $file $reservation
    return 0
}
