#!/bin/bash

#
# plumb
#
# Copyright (C) 2020-2021 by University of Southern California
#
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License,
# version 2, as published by the Free Software Foundation.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
#

#set -x
# from: https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -o pipefail

die () {
        echo "$@" 1>&2
        exit 1
}

# get cwd
wd=$(pwd)

function finish {
	# Your cleanup code here
        cd "$wd" || exit 1
}
trap finish EXIT

#### Error Codes ####
PLUMB_HOME_NOT_SET=1
PLUMB_HOME_PATH_NOT_A_DIR=2
#TOO_MANY_ARGS=3
INVALID_ARGS=4
#HDFS_FILE_NOT_EXISTS=5
INSUFFICIENT_ARGS=6
UNEXPECTED_ERR=7

DEBUG=false
VERBOSE=false



#####################

function main
{

        # make sure PLUMB_HOME is set
        if [ -z "$PLUMB_HOME" ]; then
                SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"

                if [ !  -f   "$SCRIPTPATH"/plumb ];then
                        echo "Unable to find plumb script."
                        exit $PLUMB_HOME_NOT_SET
                fi
                PLUMB_HOME=$SCRIPTPATH
                export PLUMB_HOME=$PLUMB_HOME
        fi
        cd "$PLUMB_HOME"

        if [ ! -d "$PLUMB_HOME" ]; then
                echo "Value provide in PLUMB_HOME is not a directory."
                exit $PLUMB_HOME_PATH_NOT_A_DIR
        fi

        source "$PLUMB_HOME"/call_lander_api_function_via_rpc.sh

        # OPTIND and OPTARG are used
        while getopts dv ch
        do
                case $ch in
                        d) DEBUG=true;;
                        v) VERBOSE=true;;
                        *) usage;;
                esac
        done
        shift `expr $OPTIND - 1`

        if [[ $# -lt 1 ]]; then
                error  $INSUFFICIENT_ARGS
        fi


        case "$1" in
        put)
                fileuri="$2"
                qname="$3"
                if [ -z "$fileuri" ] || [ -z "$qname" ] ; then
                        echo "put command needs two arguments: fileuri and qname"
                        exit $INSUFFICIENT_ARGS
                fi
                put "$fileuri"  "$qname"
                ;;
        
        ls-q)
                ls-q
                ;;

        ls-f)
                qname="$2"
                if [ -z "$qname" ] ; then
                        echo "ls-f command needs one argument: qname."
                        exit $INSUFFICIENT_ARGS
                fi
                ls-f   "$qname"
                ;;

        rm)
                qname="$2"
                fileuri="$3"
                if [ -z "$fileuri" ] || [ -z "$qname" ] ; then
                        echo "rm-f command needs two arguments: qname and fileuri"
                        exit $INSUFFICIENT_ARGS
                fi
                rm-f   "$qname"    "$fileuri"
                ;;


        queue-entry-checksum)
                fileuri="$2"
                if [ -z "$fileuri" ] ; then
                        echo "hdfs-checksum command needs one argument: fileuri"
                        exit $INSUFFICIENT_ARGS
                fi
                queue-entry-checksum  "$fileuri"
                ;;

        # ToDo: following hdfs-put is discontinued.  Needs to remove from here as well.
        hdfs-put)
                src="$2"
                dst="$3"
                if [ -z "$src" ] || [ -z "$dst" ] ; then
                        echo "hdfs-put command needs two arguments: src and dst"
                        exit $INSUFFICIENT_ARGS
                fi
                hdfs-put "$src" "$dst"
                ;;

        queue-entry-cat)
                fileuri="$2"
                if [ -z "$fileuri" ] ; then
                        echo "hdfs-cat command needs one argument: fileuri"
                        exit $INSUFFICIENT_ARGS
                fi
                queue-entry-cat  "$fileuri"
                ;;

        mk-q)
                new_queue_name="$2"
                queue_owner="$3"
                queue_data_type="$4"
                if [ -z "$new_queue_name" ] ; then
                        echo "mk-q command needs one mandatory argument (and two optional parameters: First queue_owner \
                        (defaults to admin) and data type of the queue (defaults to type raw): \
                        mk-q new_queue_name <who_will_be_owner_of_queue> <queue_data_tpe>"
                        exit $INSUFFICIENT_ARGS
                fi
                if [ -z "$queue_owner" ] ; then
                        queue_data_type="admin"
                fi
                if [ -z "$queue_data_type" ] ; then
                        queue_data_type="raw"
                fi
                mk-q  "$new_queue_name" "$queue_owner" "$queue_data_type"
                ;;

        rm-q)
                queue_name="$2"
                if [ -z "$queue_name" ] ; then
                        echo "rm-q command needs one mandatory argument: queue_name_to_be_deleted"
                        exit $INSUFFICIENT_ARGS
                fi

                rm-q  "$queue_name"
                ;;

        ls-faulty)
                queue_name="$2"
                if [ -z "$queue_name" ] ; then
                        echo "ls-faulty command needs one mandatory argument: queue_name"
                        exit $INSUFFICIENT_ARGS
                fi

                ls-faulty  "$queue_name"
                ;;


        queue-entry-fault)
                queue_name=$2
                fileuri="$3"

                if [ -z "$queue_name" ] || [ -z "$fileuri" ]; then
                        echo "queue-entry-fault command needs two arguments: queue_name and file_uri"
                        exit $INSUFFICIENT_ARGS
                fi
                
                queue-entry-fault  "$queue_name" "$fileuri"
                ;;

        queue-entry-restore)
                queue_name=$2
                fileuri="$3"

                if [ -z "$queue_name" ] || [ -z "$fileuri" ]; then
                        echo "queue-entry-restore command needs two arguments: queue_name and file_uri_as_returned_by_ls-faulty"
                        exit $INSUFFICIENT_ARGS
                fi

                queue-entry-restore  "$queue_name" "$fileuri"
                ;;

        # Following only works if Plumb has filled-in required info to queue (currently sitting with HLE) server.
        ls-windowed-q)
                ls-windowed-q
                ;;

        # get the list of all possible statuses of a window
        ls-all-window-statuses)
                ls-all-window-statuses
                ;;
        # get the list of all windows in a queue. Returns <window id>     <status>
        ls-win)
                queue_name="$2"
                if [ -z "$queue_name" ] ; then
                        echo "ls-win command needs one mandatory argument: queue_name_that_has_window_on_it"
                        exit $INSUFFICIENT_ARGS
                fi
                ls-win "$queue_name"
                ;;
        ls-win-in-status)
                queue_name=$2
                status="$3"

                if [ -z "$queue_name" ] || [ -z "$status" ]; then
                        echo "ls-win-status command needs two arguments: queue_name_having_window and current_Status_of_window"
                        exit $INSUFFICIENT_ARGS
                fi

                ls-win-in-status  "$queue_name" "$status"
                ;;

        set-win-status)
                queue_name=$2
                window_no=$3
                status="$4"

                if [ -z "$queue_name" ] || [ -z "$window_no" ] || [ -z "$status" ]; then
                        echo "set-win-status command needs three arguments: queue_name_having_window, window_no, and status_to_set"
                        exit $INSUFFICIENT_ARGS
                fi

                set-win-status  "$queue_name" "$window_no" "$status"
                ;;


        ls-f-in-win)
                queue_name=$2
                window_no="$3"

                if [ -z "$queue_name" ] || [ -z "$window_no" ]; then
                        echo "ls-f-in-win command needs two arguments: queue_name_having_window and window_no"
                        exit $INSUFFICIENT_ARGS
                fi

                ls-f-in-win  "$queue_name" "$window_no"
                ;;

        mv-f-in-win)
                queue_name=$2
                window_no=$3
                file_uri="$4"

                if [ -z "$queue_name" ] || [ -z "$window_no" ] || [ -z "$file_uri" ]; then
                        echo "mv-f-in-win command needs three arguments: queue_name_having_window, window_no, and file_uri"
                        exit $INSUFFICIENT_ARGS
                fi

                mv-f-in-win  "$queue_name" "$window_no" "$file_uri"
                ;;


        ls-missing-files-in-a-window)
                queue_name=$2

                if [ -z "$queue_name" ] ; then
                        echo "ls-missing-files-in-a-window command needs one argument: queue_name_having_window"
                        exit $INSUFFICIENT_ARGS
                fi

                ls-missing-files-in-a-window "$queue_name"
                ;;

        ls-site-info-in-a-window)
                queue_name=$2

                if [ -z "$queue_name" ] ; then
                        echo "ls-site-info-in-a-window command needs one argument: queue_name_having_window"
                        exit $INSUFFICIENT_ARGS
                fi

                ls-site-info-in-a-window "$queue_name"
                ;;

        
        *)
                error $INVALID_ARGS
                ;;

        esac
}





function error
{
        ecmd="Not enough args or wrong args."
        ecmd="$ecmd Supported commands are: put, ls-q, ls-f, rm, queue-entry-checksum, queue-entry-cat mk-q,"
        ecmd="$ecmd rm-q, ls-faulty, queue-entry-fault, queue-entry-restore "
        ecmd="$ecmd ls-windowed-q, ls-all-window-statuses, ls-win, ls-win-in-status, set-win-status"
        ecmd="$ecmd ls-f-in-win, mv-f-in-win, ls-missing-files-in-a-window, ls-site-info-in-a-window"

        echo "$ecmd"
        exit "$1"

}

PARAMS=()
fileuri=""
qname=""
src=""
dst=""


##########################
function put {

        f=$1
        q=$2
        loc="remote"
        prop="enqueue"

        echo "put is called with filuri= $1   and qname= $2"
        
        # step 1: local or not?
        if [ -f "$1" ]; then
                loc="local"
        fi

        # step 2: use enqueu or distribute
        
        # find owner of the queue
        # ToDo: currently done for admin user.  Need to fix for ordinary user
        # ToFix: Currently admin must provide user name. Probably system should
        # find it itself when its admin.  Currently using dummy in following call.
        qOwner=$(lander_queue_queue2user_hle "$q" "dummy")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to find owner of the queue."
                exit $retCode
        fi

        count=$(lander_queue_list_derived_queues_hle  "$q" "$qOwner" | wc -l)
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to decide between enqueu or distribute internal functions."
                exit $retCode
        fi

        if [[ "$count" -gt 1 ]]; then
                prop="distribute"
        fi

        # step 3: do the actual work

        whattodo="${loc}_${prop}"

        case "$whattodo" in
                local_enqueue)
                        echo "use enqueue_local"
                        lander_queue_enqueue_local_hle "$q" "$f"  "$qOwner"
                        retCode=$?
                        if [[ $retCode != 0 ]]; then
                                echo "[Internal error]Failed to do enqueue_local."
                                exit $retCode
                        fi
                        rm "$f"
                        ;;

                local_distribute)
                        echo "use distribute_local"
                        lander_queue_distribute_local_hle "$q" "$f"  "$qOwner"
                        retCode=$?
                        if [[ $retCode != 0 ]]; then
                                echo "[Internal error]Failed to do distribute_local."
                                exit $retCode
                        fi
                        rm "$f"
                        ;;

                remote_enqueue)
                        echo "use enqueue_remote"
                        enqURI=$(lander_getEnqueueURI_hle "$q"  "$f" "$qOwner")
                        retCode=$?
                        if [[ $retCode != 0 ]]; then
                                echo "[Internal error]Failed to get enque URI."
                                exit $retCode
                        fi

                        callString="import top.sup; from top.sup import hdfs as x; x.e(x.mvHDFSFile('$f', '$enqURI'))"
                        retString=$( (python3 -c"$callString")2>&1 1>/dev/null)
                        retCode=$?
                        if [[ $retCode != 0 ]]; then
                                echo "[Internal error]Failed to move src file ($f) to dst ($enqURI)."
                                exit $retCode
                        fi

                        lander_queue_enqueue_remote_hle "$q" "$enqURI"  "$qOwner"
                        retCode=$?
                        if [[ $retCode != 0 ]]; then
                                echo "[Internal error]Failed to do remote enqueue."
                                exit $retCode
                        fi
                        ;;

                remote_distribute)
                        echo "use distribute_remote"

                        distURI=$(lander_getDistributeURI_hle "$q"  "$f" "$qOwner")
                        retCode=$?
                        if [[ $retCode != 0 ]]; then
                                echo "[Internal error]Failed to get distribute URI."
                                exit $retCode
                        fi

                        callString="import top.sup; from top.sup import hdfs as x; x.e(x.mvHDFSFile('$f', '$distURI'))"
                        retString=$( (python3 -c"$callString")2>&1 1>/dev/null)
                        retCode=$?
                        if [[ $retCode != 0 ]]; then
                                echo "[Internal error]Failed to move src file ($f) to dst ($distURI)."
                                exit $retCode
                        fi

                        lander_distribute_hle "$q" "$distURI"  "$qOwner"
                        retCode=$?
                        if  [[ $retCode != 0 ]]; then
                                echo "[Internal error]Failed to do remote distribute."
                                exit $retCode
                        fi
                        ;;

                *)
                        error  $UNEXPECTED_ERR
                        ;;
        esac
}

#########################

##########################
function hdfs-put {

        local src=$1
        local dst=$2


        err=$(lander_copyFromLocalToHDFS_hle "$src" "$dst")
        echo "$err"
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to copy a local file ($src) into hdfs ($dst)."
                exit $retCode
        fi




}
###########################

##########################
function queue-entry-checksum {

        local fileuri=$1

        checksum=$(/usr/local/bin/hdfs -Ddfs.checksum.combine.mode=COMPOSITE_CRC -checksum  "$fileuri" | awk '{print $NF}')


        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to get checksum of file ($fileuri)."
                exit $retCode
        fi

        echo "$checksum"


}
###########################
##########################
function ls-q {


        qList=$(lander_queue_list_hle "dummyQ" "" "dummyUser")        


        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to get list of all queues in the system."
                exit $retCode
        fi

        if [ -n "$qList" ]; then
                echo  "$qList"
        fi


}
###########################
##########################
# ToFix: If qname like pcap.sz is used, nothing will show up because
#        actually files are in dervied queues (and each dervied_q can
#        have a different files currently because some already procesed etc.).

function ls-f {
        local q=$1
        qOwner=$(lander_queue_queue2user_hle "$q" "dummy")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to find owner of the queue."
                exit $retCode
        fi



        fList=$(lander_queue_files_hle   "$q"  "$qOwner")


        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to get list of files in the queue ($q)."
                exit $retCode
        fi

        if [ -n "$fList" ]; then
                echo  "$fList"
        fi



}
###########################
##########################

function rm-f {
        local q=$1
        local f=$2
        qOwner=$(lander_queue_queue2user_hle "$q" "dummy")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to find owner of the queue."
                exit $retCode
        fi



        lander_queue_release_unconditional_hle  "$q"  "$f"   "$qOwner"


        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to delete file ($f) in the queue ($q)."
                exit $retCode
        fi



}
###########################
##########################
function queue-entry-cat {
        # ToFix: I am unable to use my hdfs.py fast function here because I am unable to
        # redirect output to stdout.

        local fileuri=$1

        /usr/local/bin/hdfs -cat  "$fileuri"  


        retCode=$?
        if [[ $retCode != 0 ]]; then
                (>&2 echo "[Internal error]Failed to cat file ($fileuri).")
                exit $retCode
        fi



}
###########################
##########################
function mk-q {
  local new_queue_name="$1"
  local queue_owner="$2"
  local queue_data_type="$3"


        lander_queue_create_hle "$new_queue_name" "$queue_data_type" "$queue_owner"
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to create a queue with error code: $retCode"
                exit $retCode
        fi

        # activate newly created queue.
        lander_queue_activate_hle "$new_queue_name" "$queue_owner"
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to activate newly created queue with error code: $retCode"
        fi
        
        exit $retCode
}

###########################
##########################
function rm-q {
        local queue_name="$1"

        qOwner=$(lander_queue_queue2user_hle "$queue_name" "dummy")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to find owner of the queue."
                exit $retCode
        fi

        lander_queue_delete_hle "$queue_name" "$qOwner"
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to delete queue with error code: $retCode"
        fi

        exit $retCode
}
###########################
##########################
function ls-faulty {
        local queue_name="$1"
        
        qOwner=$(lander_queue_queue2user_hle "$queue_name" "dummy")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to find owner of the queue."
                exit $retCode
        fi
        
        fList=$(lander_queue_files_in_state_x_hle  "$queue_name" "FAULTY" "$qOwner")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to delete queue with error code: $retCode"
        fi
        
        if [ -n "$fList" ]; then
                echo  "$fList"
        fi
        
        exit $retCode
        
        
}
###########################
##########################
function queue-entry-fault {
        local queue_name="$1"
        local file_uri="$2"
        
        qOwner=$(lander_queue_queue2user_hle "$queue_name" "dummy")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to find owner of the queue."
                exit $retCode
        fi
        
        
        lander_mark_as_faulty_hle "$queue_name" "$file_uri" "$qOwner"
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to mark file as faulty with error code: $retCode"
        fi
        
        exit $retCode
}
###########################
##########################
function queue-entry-restore {
        local queue_name="$1"
        local file_uri="$2"
        
        qOwner=$(lander_queue_queue2user_hle "$queue_name" "dummy")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to find owner of the queue."
                exit $retCode
        fi
        
        
        lander_mark_as_healthy_hle "$queue_name" "$file_uri" "$qOwner"
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to mark file as healthy with error code: $retCode"
        fi
        
        exit $retCode
}
###########################
##########################
# List will be empty if Plumb haven't set state in HLE
function ls-windowed-q {
        qList=$(ls_windowed_queues "dummyUser")
        
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to get list of all windowed queues in the system."
                exit $retCode
        fi
        
        if [ -n "$qList" ]; then
                echo  "$qList"
        fi
        
        
}
###########################
##########################
function ls-all-window-statuses {
        statuses=$(ls_all_window_statuses "dummyUser")
        
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to get list of all statuses of a window."
                exit $retCode
        fi
        
        if [ -n "$statuses" ]; then
                echo  "$statuses"
        fi
        
        
}
###########################
##########################

function ls-win {
        local q=$1
        statuses=$(ls_win  "$q"   "dummyUser")
        
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to get list of windows in the queue ($q). Error code is $retCode"
                exit $retCode
        fi
        
        if [ -n "$statuses" ]; then
                echo  "$statuses"
        fi
        
}
###########################
##########################
function ls-win-in-status {
        local queue_name="$1"
        local status="$2"
        
        windows=$(ls_win_in_status "$queue_name" "$status" "dummyUser")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to get windows from queue ($queue_name) having status ($status) and the error code is: $retCode"
                exit $retCode
        fi
        
        if [ -n "$windows" ]; then
                echo  "$windows"
        fi
}
###########################
##########################
function set-win-status {
        local queue_name="$1"
        local window_no="$2"
        local status="$3"
        
        set_win_status "$queue_name" "$window_no" "$status" "dummyUser"
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to set window status with error code: $retCode"
        fi
        
        exit $retCode
}
###########################
##########################
function ls-f-in-win {
        local queue_name="$1"
        local window_no="$2"
        
        files=$(ls_f_in_win "$queue_name" "$window_no" "dummyUser")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to get files in window ($window_no) from queue ($queue_name) and the error code is: $retCode"
                exit $retCode
        fi
        
        if [ -n "$files" ]; then
                echo  "$files"
        fi
}
###########################
##########################
function mv-f-in-win {
        local queue_name="$1"
        local window_no="$2"
        local file_uri="$3"
        
        mv_f_in_win "$queue_name" "$window_no" "$file_uri" "dummyUser"
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to change window no of the file ($file_uri) with error code: $retCode"
        fi
        
        exit $retCode
}
###########################
##########################
function ls-missing-files-in-a-window {
        local queue_name="$1"
        
        files=$(ls_missing_files_in_a_window "$queue_name" "dummyUser")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to get missing files in queue ($queue_name) and the error code is: $retCode"
                exit $retCode
        fi
        
        if [ -n "$files" ]; then
                echo  "$files"
        fi
}
###########################
##########################
function ls-site-info-in-a-window {
        local queue_name="$1"
        
        info=$(ls_site_info_in_a_window "$queue_name" "dummyUser")
        retCode=$?
        if [[ $retCode != 0 ]]; then
                echo "[Internal error]Failed to get site info in queue ($queue_name) and the error code is: $retCode"
                exit $retCode
        fi
        
        if [ -n "$info" ]; then
                echo  "$info"
        fi
}
###########################

main "$@"
