#!/bin/bash 

# $Id$

# This script checks queue size and disk usage as well as per-user quota

MUTEXFILE=~plander/$(basename "$0").lock
LANDER_CONFIG=~plander/lander.conf
#read config
if [ -e $LANDER_CONFIG ]; then
    . $LANDER_CONFIG
else
    echo "ERROR: Config file $CONFIG does not exist" >&2
    exit 1
fi

TMPFILE="/tmp/landerquotas"

SENDMAIL=/bin/mail

#redirect all output to the logfile
exec >> $LANDER_MONITOR_LOGFILE 2>&1


echo
lander_set_DATE
echo "$DATE"

jobname=$(basename $0)

#function to remove suffix, usage: convert_to_bytes var
convert_to_bytes() {
    local var=$1;
    local val
    eval "val=\$$var"
    val=${val/T/*1024G}
    val=${val/G/*1024M}
    val=${val/M/*1024K}
    val=${val/K/*1024}
    val=${val/B/}
    eval $var=$(($val))
}

#function to add suffix
convert_to_suffix() {
    local var=$1;
    local val u v m tv tu
    eval "val=\$$var"
    m=1; v=0; u="B"
    for tu in B K M G T; do
	    let 'tv=val/m'
	    [ $tv -eq 0 ] && break
	    let 'm=m*1024'
	    v=$tv
	    u=$tu
    done
    
    val="${v}${u}"
    eval $var=\$val
}
   
#suspend data collection using watchfile-mechanism
suspend_dagtalkers() {
    #find $LANDER_QUEUE_RAW_FS -maxdepth 1 -type d -exec /bin/rm -f '{}'/$WATCHFILE \;
    lander_remove_watch $LANDER_WATCHFILE
}
#resume data collection using watchfile-mechanism
resume_dagtalkers() {
    #find $LANDER_QUEUE_RAW_FS -maxdepth 1 -type d -exec /bin/touch '{}'/$WATCHFILE \;
    lander_create_watch $LANDER_WATCHFILE
}

convert_to_bytes LANDER_MINFREESPACE #pass var by name
#Raise alarm when available user space drops below LANDER_MINFREESPACE
lander_diskfree $LANDER_TOPDIR_U
sz=$lander_diskfree_dused; convert_to_suffix sz
echo "Current user disk usage: $sz"
if [ $lander_diskfree_avail -le $LANDER_MINFREESPACE ]; then
    # email for now
    cat <<EOF | $SENDMAIL -s "LANDER USERS ARE OUT OF SPACE" $LANDER_ADMIN
    User disk usage is out of control: $sz
    $(df -h $LANDER_TOPDIR_U)

EOF
    echo "sent mail - (users out of space)"
fi

#Raise alarm and shut down landers when available raw space drops below MINFREESPACE
lander_diskfree $LANDER_QUEUE_RAW_FS
sz=$lander_diskfree_dused; convert_to_suffix sz
echo "Current raw disk usage: $sz"
if [ $lander_diskfree_avail -le $LANDER_MINFREESPACE ]; then
    # notify landers to wait until suspend_dagtalkers
    # email for now
    cat <<EOF  | $SENDMAIL -s "LANDER IS OUT OF SPACE" $LANDER_ADMIN

  Suspending dagtalkers
  Disk usage is out of control: $sz
  $(df -h $LANDER_QUEUE_RAW_FS)

EOF
    echo "sent mail - (rawdir out of space, suspending dagtalkers)"
    suspend_dagtalkers
fi

#check for other monitor running
if [ -e "$MUTEXFILE" ]; then
    echo "WARNING: mutex lock exists"
    read pid < $MUTEXFILE
    if kill -0 $pid 2>/dev/null; then
	    #the following is probably unnecessary
	    #xxx careful here because solaris needs to run this
	    jname=$(ps -o args -p $pid | sed -n '2 { s%.*/%%g;  p; }')
	    if [ "$jname" == "$jobname" ]; then
	        echo "WARNING: old monitor appears to be running, exitting"
	        exit 1
	    else
	        echo "WARNING: process is running but unexpected jobname ($jname)"
	    fi
    fi
    echo "WARNING: Removing stale lock"
    /bin/rm -f "$MUTEXFILE"
fi

echo "$$" > "$MUTEXFILE.$$"
mv -i "$MUTEXFILE.$$" "$MUTEXFILE" || {
    echo "WARNING: failed to overwrite MUTEXFILE (another monitor started?)"
    rm -f "$MUTEXFILE.$$"
    exit 1
}

[ -z "$LANDER_QUEUE_RAW_FS" ] && {
    echo "ERROR: LANDER_QUEUE_RAW_FS isn't set" >&2
    exit 1;
}
#XXX almaak does not understand posix-extended
NOFILES=$(find $LANDER_QUEUE_RAW_FS/ -xdev -name '[0-9]*-[0-9]*-[0-9]*' -print | wc -l)

if [ $NOFILES -gt $LANDER_MAXFILES ]; then 
    echo "WARNING: Pending files limit exceeded: $NOFILES > $LANDER_MAXFILES"
fi

lander_set_JOBS_ALL $LOGNAME

echo "$LOGNAME jobs: $JOBS_ALL Pending Files: $NOFILES"
if [ "$JOBS_ALL" -gt $LANDER_MAXJOBS ]; then
    echo "WARNING: QueueSize limit exceeded: $JOBS_ALL > $LANDER_MAXJOBS"
fi


lander_set_LOAD
if [ -n "$LOAD" ]; then
    echo "Current load: $LOAD"
    diff=`echo $LANDER_MAXLOAD_WARN - $LOAD | bc -l`
    if [ -z "${diff/-*/}" ]; then
	echo "WARNING: load limit exceeded: $LOAD > $LANDER_MAXLOAD_WARN"
    fi
fi

#read previous LANDER_QUOTAFILE and set variables overquota_USER
while read line; do
    set -- $line
    if [ -n "${1%%#*}" ]; then
	    #this is not a comment
	    set -- $line
	    n=$1 #user name
	    q=$2 #quota
	    u=$3 #usage
	    eval "oldusage_$n=$u"

	    convert_to_bytes q
	    convert_to_bytes u

	    if [ $u -gt $q ]; then
	        eval "overquota_$n=yes"
	    fi
    fi
done < <(lander_catfile $LANDER_QUOTAFILE)

#make TMPFILE the next LANDER_QUOTAFILE
cat >$TMPFILE << EOF 
# File: $LANDER_QUOTAFILE  
# This file is automatically generated by $jobname
# DO NOT EDIT BY HAND
EOF

chgrp ${LANDER_GROUPS[main]} $TMPFILE
chmod 640 $TMPFILE

#read in the USERSFILE and generate new quota
while read line; do
    set -- $line
    if [ -n "${1%%#*}" ]; then
	    #this is not a comment
	    set -- $line
	    user_name=$1
	    user_email=$2
	    user_quota=${3:-$LANDER_MINQUOTA}
	    
	    convert_to_bytes user_quota
	    
	    # if user is not taking data, leave their old entry if any
	    eval oldusage=\$oldusage_$user_name
	    if [ -n "$oldusage" ] && ! lander_user_subscribed $user_name \* \*; then
	        # no need to recompute quota: no delivery and have old entry
	        echo -e "${user_name}\t\t${user_quota}\t\t${oldusage}" >>$TMPFILE
	        continue
	    fi
        
        # check disk usage for this user
	    lander_set_USER_DISK_USAGE $user_name
        
	    if [ -n "$USER_DISK_USAGE" ]; then
	        echo -e "${user_name}\t\t${user_quota}\t\t${USER_DISK_USAGE}" >>$TMPFILE
	        # currently overquota?
	        overquotanow=""
	        if [ $USER_DISK_USAGE -gt $user_quota ]; then
		        overquotanow="yes"
	        fi
	        if [ -n "$user_email" ]; then
	            # was overquota before?
		        eval overquotaold=\$overquota_$user_name
		        if [ -z "$overquotaold" -a -n "$overquotanow" ]; then
		            # just went overquota
		            echo "WARNING: $user_name is using ${USER_DISK_USAGE}B of space. Shutting down data delivery."
		            subj="LANDER Monitor: User disk quota exceeded (${USER_DISK_USAGE}B/${q}B)"
		            echo "LANDER trace delivery shut down." | $SENDMAIL -s "$subj" "$email"
		        fi
		        if [ -n "$overquotaold" -a -z "$overquotanow" ]; then
		            # back into quota
		            echo "WARNING: $user_name now satisfies quota requirements.  Resuming data delivery."
		            subj="LANDER Monitor: User disk quota OK (${USER_DISK_USAGE}B/${q}B)"
		            echo "LANDER trace delivery enabled." | $SENDMAIL -s "$subj" "$email"
		        fi
	        else
		        echo "WARNING: no email for user $user_name"
	        fi
	    else
	        echo "WARNING: lander_set_USER_DISK_USAGE failed on $LANDER_TOPDIR_U/$user_name"
	    fi
    fi
done < <(lander_catfile $LANDER_USERSFILE)

#update LANDER_QUOTAFILE
lander_file_copylocal "$TMPFILE" "$LANDER_QUOTAFILE"

#if dagsnap_lander is not running, may restart
if [ -n "$LANDER_MONITOR_RESTARTS_DAGTALKER" -a -x /etc/init.d/dagtalker ]; then
    lander_check_dagsnap_lander || { 
	    sudo /etc/init.d/dagtalker $LANDER_MONITOR_RESTARTS_DAGTALKER && \
	        echo "restarted dagsnap_lander as $LANDER_MONITOR_RESTARTS_DAGTALKER"
    }
fi

#check if dagtalkers should resume
#+ resume only if the following conditions are met:
#+ 1. Available disk space is more than 100G
#+ 2. $LANDER_QUEUE_RAW_FS directory is empty
#+ 3. IF running on mini-LANDER: cpu load is not too high 
#+    (this would indicate running user scripts)
convert_to_bytes LANDER_MINSPACE_THRESHOLD
if (( lander_diskfree_avail > LANDER_MINFREESPACE + LANDER_MINSPACE_THRESHOLD && \
      NOFILES  < LANDER_MAXFILES_THRESHOLD )); then
    #get current load values
    lander_set_LOAD
    if [ -n "$LOAD" ]; then
	    diff=`echo $LOAD - $LANDER_LOAD_RESUME | bc -l`
	    if [ -z "${diff/-*/}" ]; then
	        echo "Continuing data collection"
	        resume_dagtalkers
	    fi
    else
	    #this is the case of HPCC
	    echo "Continuing data collection"
	    resume_dagtalkers
    fi
fi


#finally remove lock and exit
/bin/rm -f "$MUTEXFILE"
exit 0
