#!/bin/bash

# Script to check hpcc qbalance on the hpcc
# Assumes can ssh to plander@$HPCC

# $Id$

MIN_NWCH=100			# send mail if nwch drops below this level
NOTIFY="lander@isi.edu"		# whom to notify
ADMIN="yuri@isi.edu"		# whom to send verbose (error) messages

SSH=/usr/bin/ssh
RM=/bin/rm
MAIL=/bin/mail
HPCC=hpc
QBALANCE=qbalance

#if argument-file exists and has non-zero content, send it to admin
remove_output() {
    out="$1"
    if [ -s "$out" ]; then
	cat "$out" | $MAIL -s "messages from qbalance" $ADMIN
    fi
    $RM -rf "$out"
}

#redirect everything to a temp file
output=`mktemp /tmp/qbalance.XXXX`;
exec >$output 2>&1

#check for other monitor running
MUTEXFILE=/tmp/qbalance.mutex
if lockfile -r0 -! "$MUTEXFILE" 2>/dev/null ; then
    #mutex file exists, check it's not stale
    echo "WARNING: mutex lock exists"
    read pid < $MUTEXFILE
    pname=`cat  /proc/$pid/cmdline 2>/dev/null`
    jname=`basename "$pname"`
    if [ "$jname" = "qbalance.sh" ]; then
	echo "WARNING: old qbalance.sh appears to be running, exitting"
	remove_output $output
	exit 1
    fi
    echo "WARNING: Removing stale lock"
    rm -f "$MUTEXFILE"
fi

#save my pid to the mutex file
[ -e "$MUTEXFILE" ] && chmod +w "$MUTEXFILE"
echo "$$" > "$MUTEXFILE"

#get the remainging balance
set -- `ssh plander@$HPCC $QBALANCE -h | grep ${LANDER_GROUPS[main]}`; nwch=$2

#remove the decimal
nwch=${nwch%.*}

if [ $nwch -lt $MIN_NWCH ]; then
    msg="Plander (${LANDER_GROUPS[main]}) HPCC balance is too low: $nwch < $MIN_NWCH"
    echo "$msg" | $MAIL -s "WARNING: LANDER HPCC balance is too low" $NOTIFY
fi

remove_output $output
#cleanup the mutex
$RM -f "$MUTEXFILE"

exit 0

