#!/bin/bash

#This script watches for new files written in a local raw dir and
#moves them to HDFS.

# LANDER_RAWDIR         : where dagtalker writes data (HDFS: local, NFS: NFS)
# LANDER_QUEUE_RAW_HDFS : where to store stuff in HDFS
# LANDER_QUEUE_TYPE     : exit if this is not set to 'hdfs'

usage() {
    cat >&2 <<EOF
    USAGE: `basename $0` [OPTIONS]
       Move friles from LANDER_RAWDIR to LANDERQUEUE_RAW_HDFS
    OPTIONS:
       -D     Break files into subdirs by YEAR/MONTH/DATE
EOF
}

datesubdirs=
while getopts "hD" opt; do
    case $opt in
        ( D ) datesubdirs=y; break;;
        ( h|* ) usage; exit 0;;
    esac
done
shift $((OPTIND-1));

(( $# != "0" )) || usage


LANDER_CONFIG=~plander/lander.conf

trap 'exit_now=true; echo "waiting for things to settle..."; wait' EXIT HUP STOP QUIT TSTP

#read config
if [ -e $LANDER_CONFIG ]; then
    . $LANDER_CONFIG
else
    echo "ERROR: Config file $LANDER_CONFIG does not exist" >&2
    exit 1
fi

[ "$LANDER_QUEUE_TYPE" == "hdfs" ] || {
    echo "LANDER_QUEUE_TYPE=$LANDER_QUEUE_TYPE, nothing to do"
    exit 0 #not hdfs, nothing to do
}

HDFSMOVERLOG=~plander/hdfsmover.log
HDFSMAXTRIES=5
LANDER_HDFSMOVER_MAXPARALLEL=5

exec >> $HDFSMOVERLOG 2>&1

LOGERROR=1
LOGWARN=2
LOGINFO=3

LOGLEVEL=$LOGINFO
ADMIN=yuri@isi.edu

log() {
    log="$(date --rfc-3339=s)\t$@"
    echo -e "$log"
}

logmail() {
    log "$@"
    mail -s "hdfsmover error" $ADMIN <<EOF

hdfsmover error on $(hostname): 
    $(echo -e "$log")

EOF
}

info() {
    [ $LOGLEVEL -ge $LOGINFO ] && log "INFO\t$@"
}
warn() {
    [ $LOGLEVEL -ge $LOGWARN ] && log "WARN\t$@"
}
error() {
    [ $LOGLEVEL -ge $LOGERROR ] && logmail "ERROR\t$@"
}

#Allow starting this script from cron job, ensure only one
#+instance is running
HDFSMOVER_PID=$HOME/hdfsmover.pid
if [ -e $HDFSMOVER_PID ]; then
    read pid rest < $HDFSMOVER_PID
    if [ -n "$pid" ] && kill -0 $pid >& /dev/null; then
	    info "hdfsmover is already running as $pid"
	    exit 0
    fi
fi
echo $$ > $HDFSMOVER_PID

#remove HDFS file
hdfs_rename_file() {
    local _s=$1 _d=$2
    local _try
    for _try in {1..$HDFSMAXTRIES}; do
	    if $LANDER_HDFS -mv $_s $_d; then
	        info "moved $_s to $_d OK"
	        return 0
	    fi
	    $LANDER_HDFS -rm $_d >& /dev/null
    done
    error "Failed to rename $_s to $_d after $HDFSMAXTRIES"
    return 1
}

#puts file into hdfs, tries several times
hdfs_copyin_file() {
    local _s=$1 _d=$2
    local _try
    for _try in {1..$HDFSMAXTRIES}; do
	    if $LANDER_HDFS -put $_s $_d; then
	        info "copied $_s to $_d OK"
	        return 0
	    fi
	    #remove dst and retry
	    $LANDER_HDFS -rm $_d 2>/dev/null
    done
    warn "Failed to copyin $_s to $_d after $_try attempts"
    return 1
}

#files do not need to be renamed by pid, since single controller
#hands them out to threads
exclusive_put() {
    local _s=$1 _d=$2
    local _sd=$(dirname  $_s)
    local _sb=$(basename $_s)
    mv "$_s" "$_sd/_copyout.$_sb"
    #local _dd=$(dirname  $_d)
    #local _db=$(basename $_d)
    #hdfs_copyin_file "$_sd/_copyout.$_sb" "$_dd/_copyin.$_db" || {
	#    mv "$_sd/_copyout.$_sb" "$_s"
	#    return 1
    #}
    #hdfs_rename_file "$_dd/_copyin.$_db" "$_d" || {
	#    #copyin file is ok, but for some reason cannot be renamed
	#    return 2
    #}
    distributeLocal "$_sd/_copyout.$_sb"
    rm -f "$_sd/_copyout.$_sb"
    return 0
}

#forks at most $LANDER_HDFSMOVER_MAXPARALLEL hdfs -put's
#if so many are already running, sleeps, polling.  Uses job control.
hdfs_moverjob() {
    local _s=$1 _d=$2
    while [ $(jobs -pr | wc -l) -ge "$LANDER_HDFSMOVER_MAXPARALLEL" ]; do
	    sleep 3
    done
    {
	    exclusive_put "$_s" "$_d" 
	    #sleep 5
    } &
}

#re-create hdfs dirs/permissions/own
recreate_hdfs_dir() {
    local dir=$1
    local p= topd=
    for d in $(echo $dir | sed 's%/% %g'); do
	    [ -z "$p" ] && topd="/$d"
	    p="$p/$d"
	    $LANDER_HDFS -test -d $p || {
	        $LANDER_HDFS -mkdir $p || {
		        error "Failed to create $p"
		        return 1
	        }
	        echo "Re-created $p"
	    }
    done
    for s in $LANDER_SOURCES; do
	    $LANDER_HDFS -test -d "$dir/$s" || {
	        $LANDER_HDFS -mkdir "$dir/$s" || { 
		        error "Failed to create $dir/$s"
		        return 1
	        } 
	        echo "Re-created $dir/$s"
	    }
    done
	
    $LANDER_HDFS -chgrp -R ${LANDER_GROUPS[raw]} $topd || {
	    error "Failed to chgrp $topd"
	    return 1
    }
    $LANDER_HDFS -chmod -R go-rx $topd || {
	    error "Failed to chmod $topd"
	    return 1
    }
}

#main
[ -z "$LANDER_HDFS" ] && {
    error "LANDER_HDFS is not set, cannot move"
    exit 1
}

recreate_hdfs_dir $LANDER_QUEUE_RAW_HDFS || exit 1
recreate_hdfs_dir $LANDER_QUEUE_RESERVED_HDFS || exit 1

#remove any stale .copyin files
$LANDER_HDFS -rm $LANDER_QUEUE_RAW_HDFS/*/_copyin.* >& /dev/null
if [ -n "$datesubdirs" ]; then
    $LANDER_HDFS -rm $LANDER_QUEUE_RAW_HDFS/*/*/*/*/_copyin.* >& /dev/null
fi

RUNFOR_MIN=120
time_start=$(date +"%s")
time_end=$(( RUNFOR_MIN*60 + time_start))

#rename copyouts
for f in $(find $LANDER_QUEUE_RAW_FS/ -regextype posix-extended \
    -regex '.*\/_copyout.[0-9]{8}-[0-9]{6}-[0-9]+.*'); do
    mv $f ${f#_copyout.}
done

while [ $(date +"%s") -lt $time_end ]; do    
    [ -z "$exit_now" ] || break
    #start moving new files
    files=$(find $LANDER_QUEUE_RAW_FS/ -regextype posix-extended \
	    -regex '.*\/[0-9]{8}-[0-9]{6}-[0-9]+.*' -mmin +5 -print 2>/dev/null)
    [ -z "$files" ] && { 
	    sleep 30s
	    continue
    }
    
    # sort files by basename
    fnkey=$(echo "/x$files" 2>/dev/null | head -1 | tr '/' ' ' | wc -w) #should be num of '/' + 1
    sortedfiles=$(echo "$files" | sort -t '/' -k $fnkey)

    info "Have " $(echo "$sortedfiles" | wc -l) " to move"
    
    for FULLFILE in $sortedfiles; do
	    lander=$(basename $(dirname $FULLFILE))
	    file=$(basename $FULLFILE)
        if [ -n "$datesubdirs" ]; then
            date_y=${file:0:4}
            date_m=${file:4:2}
            date_d=${file:6:2}
            $LANDER_HDFS -mkdir -p "$LANDER_QUEUE_RAW_HDFS/$lander/$date_y/$date_m/$date_d" >&/dev/null
            hFULLFILE=$LANDER_QUEUE_RAW_HDFS/$lander/$date_y/$date_m/$date_d/$file
        else
	        hFULLFILE=$LANDER_QUEUE_RAW_HDFS/$lander/$file
        fi
	    [ -z "$exit_now" ] || break 2
        
	    hdfs_moverjob $FULLFILE $hFULLFILE #this parallelizes hdfs -put
	    
    done #for FULLFILE in $sortedfiles
done #until time_end

wait

chmod a+r $HDFSMOVERLOG
mv $HDFSMOVERLOG $LANDER_LOGDIR/

exit 0
