#!/bin/bash

# STORHAUG: High-Availability Storage Server with Pacemaker
#
# Copyright (c) 2015 Red Hat Inc.
# Copyright (c) 2015 Jose A. Rivera <jarrpa@redhat.com>
#   All Rights Reserved.
#
# License: GPLv2 or any later version.
#   See the file LICENSE or http://www.gnu.org/licenses/gpl-2.0.en.html#SEC1
#

HA_NUM_SERVERS=0
HA_SERVERS=""
SYS_CONFDIR="/etc/sysconfig"
HA_CONF="${SYS_CONFDIR}/storhaug.conf"
HA_CONF_INCDIR="${SYS_CONFDIR}/storhaug.d"
HA_MNT_DIR="/run/storhaug"
HA_SMB_MNT_DIR="lock"
HA_NFS_MNT_DIR="state"
STORAGE_SERVERS=""
STORAGE_NUM_SERVERS=0
DETERMINISTIC_FAILOVER=false
SERVICE_MAN="DISTRO_NOT_FOUND"
SECRET_PEM="${HA_CONF_INCDIR}/secret.pem"

### Utility functions

_hostname=$(hostname)

usage()
{
    echo -e "Usage: `basename "$0"` [<OPTIONS>] <COMMAND> [<ARGUMENTS>]"
    echo -e "Manage a storhaug high-availability (HA) storage cluster."
    echo -e "\nGlobal OPTIONS:"
    echo -e "  -h, --help            Output this useful help message"
    echo -e "\nCOMMANDS:"
    echo -e "  status                Check the status of the cluster"
    echo -e "  setup                 Setup a new cluster"
    echo -e "  teardown              Teardown an existing cluster"
    echo -e "  cleanup               Cleanup local cluster config"
    echo -e "  cleanup-all           Cleanup cluster config on all nodes"
    echo -e "  add                   Add a node to the cluster"
    echo -e "  delete, remove        Remove a node from the cluster"
    echo -e "\nCommand ARGUMENTS:"
    echo -e "  add <NODE>            Add hostname NODE to the cluster"
    echo -e "  remove <NODE>         Remove hostname NODE from the cluster"
    echo -e "  delete <NODE>         Synonym for 'remove'"
    echo -e "\n\nConfiguration is read from the following locations:"
    echo -e "  ${HA_CONF}"
    echo -e "  ${HA_CONF_INCDIR}/*.conf"
}

parsebool()
{
    case $(eval echo \${${1}}) in
        TRUE | True | true | YES | Yes | yes) declare "${1}"=true ;;
        FALSE | False | false | NO | No | no) declare "${1}"=false ;;
        *) storlog "ERR" "Couldn't parse boolean: ${1}=$(eval echo \${${1}})" ;;
    esac
}

storlog()
{
    LEVEL=$1; shift
    case $LEVEL in
        ERR|ERROR)
        echo "ERROR: $1" >&2
        logger --tag="storhaug" -p "daemon.err" "$1"
        rm -rf $HA_CONF_secdir
        exit 1
        ;;
        WARN|WARNING)
        echo "WARNING: $1"
        logger --tag="storhaug" -p "daemon.warn" "$1"
        ;;
        INFO)
        echo "$1"
        logger --tag="storhaug" -p "daemon.info" "$1"
        ;;
        DEBUG)
        logger --tag="storhaug" -p "daemon.debug" "$1"
        ;;
    esac
}

sshdo()
{
    if [ "${_hostname}" == "${1}" ] || \
       [ "${_hostname%%.*}" == "${1%%.*}" ] || \
       [ "localhost" == "${1}" ]; then
        ${2}
    else
        ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i ${SECRET_PEM} root@${1} "${2}"
    fi
    local _ret=$?
    if [ $_ret -ne 0 ]; then
        storlog "WARN" "Command failed on ${1}: ${2}"
    fi
    return $_ret
}

scpdo()
{
    # avoid prompting for password, even with password-less scp
    # scp $host1:$file $host2:$file prompts for the password
    scp -3 -r -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i ${SECRET_PEM} ${1} ${2}
    local _ret=$?
    if [ $_ret -ne 0 ]; then
        storlog "WARN" "SCP failed from ${1} to ${2}"
    fi
    return $_ret
}

# Check that a symlink exists, create it otherwise.
# Usage: ensure_ln <TARGET> <LINK>
ensure_ln ()
{
    if [ ! -L "${2}" ] ; then
        rm -rf "${2}"
    else
        _t=$(readlink "${2}")
        if [ "$_t" != "${1}" ] ; then
            rm -f "${2}"
        fi
    fi
    # This is not an "else".  It also re-creates the link if it was
    # removed above!
    if [ ! -e "${2}" ]; then
        ln -sf "${1}" "${2}"
    fi
}

# Check that a directory exists, create it otherwise.
# Only use on paths guaranteed to be directories.
ensure_dir ()
{
    if [ ! -d "${1}" ] ; then
        mkdir -p "${1}"
    fi
}

# Check that a file exists, touch it otherwise.
# Only use on paths guaranteed to be regular files.
ensure_file ()
{
    if [ ! -e "${1}" ]; then
        touch "${1}"
    fi
}

### General cluster functions

check_cluster_exists()
{
    local name=${1}

    if [ -e /var/run/corosync.pid ]; then
        local cluster_name=$(pcs status | grep "Cluster name:" | cut -d ' ' -f 3)
        if [ ${cluster_name} -a ${cluster_name} = ${name} ]; then
            storlog "ERR" "Cluster $name already exists, exiting"
        fi
    fi
}

determine_servers()
{
    local cmd=${1}
    local num_servers=0
    local tmp_ifs=${IFS}

    if [[ "X${cmd}X" != "XsetupX" ]]; then
        local ha_servers=$(pcs status | grep "Online:" | grep -o '\[.*\]' | sed -e 's/\[//' | sed -e 's/\]//')
        IFS=$' '
        for server in ${ha_servers} ; do
            num_servers=$(expr ${num_servers} + 1)
        done
        IFS=${tmp_ifs}
        HA_NUM_SERVERS=${num_servers}
        HA_SERVERS="${ha_servers}"
        # TODO: Determine storage and vip servers from pcs status
        if [[ "x${STORAGE_NODES}" != "x" ]]; then
            STORAGE_SERVERS="${STORAGE_NODES//,/ }"
            STORAGE_NUM_SERVERS=$(wc -w <<< "${STORAGE_SERVERS}")
        else
            STORAGE_SERVERS=${HA_SERVERS}
            STORAGE_NUM_SERVERS=${HA_NUM_SERVERS}
        fi
        if [[ "x${HA_VIP_NODES}" != "x" ]]; then
            VIP_SERVERS="${HA_VIP_NODES//,/ }"
        fi
    else
        IFS=$','
        for server in ${HA_CLUSTER_NODES} ; do
            num_servers=$(expr ${num_servers} + 1)
        done
        IFS=${tmp_ifs}
        HA_NUM_SERVERS=${num_servers}
        HA_SERVERS="${HA_CLUSTER_NODES//,/ }"
        if [[ "x${STORAGE_NODES}" != "x" ]]; then
            STORAGE_SERVERS="${STORAGE_NODES//,/ }"
            STORAGE_NUM_SERVERS=$(wc -w <<< "${STORAGE_SERVERS}")
        else
            STORAGE_SERVERS=${HA_SERVERS}
            STORAGE_NUM_SERVERS=${HA_NUM_SERVERS}
        fi
        if [[ "x${HA_VIP_NODES}" != "x" ]]; then
            VIP_SERVERS="${HA_VIP_NODES//,/ }"
        fi
    fi
}

copy_config()
{
    while [[ ${1} ]]; do
        if [ ${_hostname%%.*} != ${1%%.*} ]; then
            scpdo ${HA_CONF} ${1}:$(dirname ${HA_CONF})/
            scpdo ${HA_CONF_INCDIR} ${1}:$(dirname ${HA_CONF_INCDIR})/
        fi
        shift
    done
}

# TODO: Move to RA
copy_nfs_config()
{
    if [ ${_hostname%%.*} != ${1%%.*} ]; then
        scpdo ${HA_VOL_SERVER}:${HA_NFS_CONF} ${1}:${HA_NFS_CONF}
        scpdo ${HA_VOL_SERVER}:${HA_NFS_EXPDIR} ${1}:${HA_NFS_EXPDIR}
    fi
}

### General resource functions

clear_virt_ip_constraints()
{
    local cibfile=${1}; shift
    pcs -f ${cibfile} constraint remove *vip* || \
      storlog "WARN" "Failed: pcs constraint remove *vip*"
}

do_create_virt_ip_constraints()
{
    local cibfile=${1}; shift
    local ipcount=${1}; shift
    local primary=${1}; shift
    local weight="1000"

    # A set of location constraints to set the prefered order
    # for where a VIP should move
    while [[ ${1} ]]; do
        pcs -f ${cibfile} constraint location vip${ipcount} prefers ${1}=${weight} || \
          storlog "WARN" "Failed: pcs constraint location vip${ipcount} prefers ${1}=${weight}"
        weight=$(expr ${weight} + 1000)
        shift
    done

    # Set the highest preference for the VIP to its primary node
    pcs -f ${cibfile} constraint location vip${ipcount} prefers ${primary}=${weight} || \
      storlog "WARN" "Failed: pcs constraint location vip${ipcount} prefers ${primary}=${weight}"
}

create_virt_ip_constraints()
{
    local cibfile=${1}; shift
    local ipcount=${1}; shift
    local srvcount=${ipcount}
    local primary=""
    local head=""
    local tail=""

    # build a list of failover peers, e.g. for a four node cluster, for node1,
    # the result is "node2 node3 node4"; for node2, "node3 node4 node1"
    # and so on.
    read -r -a servers <<< "${VIP_SERVERS:-STORAGE_SERVERS}"
    while [ ${srvcount} -gt ${STORAGE_NUM_SERVERS} ]; do
        srvcount=$((srvcount - STORAGE_NUM_SERVERS))
    done
    primary=${servers[${srvcount}-1]}
    if [ ${STORAGE_NUM_SERVERS} -gt 1 ]; then
        head=${servers[@]:${srvcount}-${STORAGE_NUM_SERVERS}-1}
        tail=${servers[@]:${srvcount}}
    fi

    do_create_virt_ip_constraints ${cibfile} ${ipcount} ${primary} ${tail} ${head}
}

create_virt_ip()
{
    local cibfile=${1}; shift
    local ipcount=${1}; shift
    local ip=${1}; shift

    pcs -f ${cibfile} resource create vip${ipcount} ocf:heartbeat:IPaddr2 \
        params \
            ip=${ip} \
            flush_routes="true" \
        op monitor interval=60s \
        meta resource-stickiness="0"

    pcs -f ${cibfile} constraint location vip${ipcount} rule resource-discovery=exclusive score=0 role eq storage

    pcs -f ${cibfile} resource create vip${ipcount}_trigger ocf:heartbeat:ganesha_trigger \
        meta resource-stickiness="0"

    pcs -f ${cibfile} constraint colocation add vip${ipcount}_trigger with vip${ipcount} INFINITY
    pcs -f ${cibfile} constraint order vip${ipcount} then vip${ipcount}_trigger
}

### Setup functions

setup_cluster()
{
    local unclean=""

    storlog "INFO" "Setting up cluster ${HA_NAME} on the following servers: ${servers}"

    pcs cluster auth ${HA_SERVERS} -u hacluster -p ${HA_PASSWORD} --force
    pcs cluster setup --force --name ${HA_NAME} ${HA_SERVERS} || storlog "ERR" "Failed to setup cluster ${HA_NAME}"
    pcs cluster start --all || storlog "ERR" "Failed to start cluster ${HA_NAME}"

    sleep 3
    unclean=$(pcs status | grep -u "UNCLEAN")
    while [[ "${unclean}X" = "UNCLEANX" ]]; do
         sleep 1
         unclean=$(pcs status | grep -u "UNCLEAN")
    done
    sleep 1

    local tmp_ifs=${IFS}
    IFS=$' '
    for server in ${STORAGE_SERVERS:-$HA_SERVERS} ; do
        pcs property set --node $server role=storage || \
          storlog "WARN" "Failed: pcs property set --node $server role=storage"
    done
    IFS=${tmp_ifs}

    if [ ${HA_NUM_SERVERS} -lt 3 ]; then
        pcs property set no-quorum-policy=ignore || \
          storlog "WARN" "Failed: pcs property set no-quorum-policy=ignore"
    fi
    pcs property set stonith-enabled=false || storlog "WARN" "Failed: pcs property set stonith-enabled=false"
}

setup_create_resources()
{
    local cibfile=$(mktemp --tmpdir=$HA_CONF_secdir)

    pcs cluster cib ${cibfile}

    # Shared volumes
    mkdir -p "${HA_MNT_DIR}/${HA_NFS_MNT_DIR}"
    pcs -f ${cibfile} resource create ganesha_state ocf:heartbeat:Filesystem \
        params \
            device="localhost:/${HA_NFS_VOL}" \
            directory="${HA_MNT_DIR}/${HA_NFS_MNT_DIR}" \
            fstype="glusterfs" \
            options="_netdev,defaults,direct-io-mode=enable,transport=tcp,xlator-option=*client*.ping-timeout=10" \
        --clone ganesha_state-clone ganesha_state meta interleave="true" clone-max="${STORAGE_NUM_SERVERS}"

    pcs -f ${cibfile} constraint location ganesha_state-clone rule resource-discovery=exclusive score=0 role eq storage

    pcs cluster cib-push ${cibfile} || storlog "ERR" "Failed to create filesystem resources."

    # CTDB
    pcs -f ${cibfile} resource create ctdb ocf:heartbeat:CTDB \
        params \
            ctdb_socket="/var/run/ctdb/ctdbd.socket" \
            ctdb_manages_winbind="no" \
            ctdb_manages_samba="no" \
            ctdb_logfile="/var/log/log.ctdb" \
        op monitor interval="10" timeout="30" \
        op monitor interval="11" timeout="30" role="Master" \
        op start interval="0" timeout="90" \
        op stop interval="0" timeout="100" \
        op promote interval="0" timeout="30" \
        op demote interval="0" timeout="30" \
        --master meta interleave="true" globally-unique="false" notify="true" clone-max="${STORAGE_NUM_SERVERS}"

    # Samba
    pcs -f ${cibfile} resource create nmb systemd:nmb \
        op start timeout="60" interval="0" \
        op stop timeout="60" interval="0" \
        op monitor interval="60" timeout="60"
    pcs -f ${cibfile} resource create smb systemd:smb \
        op start timeout="60" interval="0" \
        op stop timeout="60" interval="0" \
        op monitor interval="60" timeout="60"
    pcs -f ${cibfile} resource group add samba-group nmb smb
    pcs -f ${cibfile} resource clone samba-group meta interleave="true" clone-max="${STORAGE_NUM_SERVERS}"

    # Samba: We need CTDB
    pcs -f ${cibfile} constraint colocation add samba-group-clone with ctdb-master INFINITY
    pcs -f ${cibfile} constraint order ctdb-master then samba-group-clone INFINITY

    # Ganesha
    pcs -f ${cibfile} resource create nfs-ganesha ocf:heartbeat:ganesha \
        params \
            config="${HA_NFS_CONF}" \
            state_fs="${HA_NFS_TYPE}" \
            state_mnt="${HA_MNT_DIR}/${HA_NFS_MNT_DIR}" \
        --clone nfs-ganesha-clone ganesha meta interleave="true" \
                                           globally-unique="false" \
                                           notify="true"

    # Ganesha: We need our shared state FS
    pcs -f ${cibfile} constraint colocation add nfs-ganesha-clone with ganesha_state-clone INFINITY
    pcs -f ${cibfile} constraint order ganesha_state-clone then nfs-ganesha-clone INFINITY

    pcs cluster cib-push ${cibfile} || storlog "ERR" "Failed to create service resources."

    # Virtual IPs
    local ipcount=0
    for ip in ${HA_VIPS}; do
        ((ipcount++))
        create_virt_ip ${cibfile} ${ipcount} ${ip}
    done

    if [[ ${DETERMINISTIC_FAILOVER} == true ]]; then
        for ((i=1;i<=${ipcount};i++)); do
            create_virt_ip_constraints ${cibfile} ${i}
        done
    fi

    pcs cluster cib-push ${cibfile} || storlog "ERR" "Failed to create virtual IP resources."

    rm -f ${cibfile}
}

# TODO: Move to RA
setup_state_volume()
{
    local mnt=$(mktemp -d --tmpdir=$HA_CONF_secdir)
    local dname=""
    local dirname=""
    local staterefs="${mnt}/nfs-ganesha/.noderefs"

    mount -t glusterfs ${HA_SERVER}:/${HA_NFS_VOL} ${mnt}
    ensure_dir ${staterefs}

    dname=${_hostname#*.}

    for srv in ${STORAGE_SERVERS:-HA_SERVERS}; do

        if [[ "${srv}" == *${dname} ]]; then
            dirname=${srv}
        else
            dirname=${srv}.${dname}
        fi

        ensure_dir ${mnt}/nfs-ganesha/${dirname}/ganesha/v4recov
        ensure_dir ${mnt}/nfs-ganesha/${dirname}/ganesha/v4old
        ensure_dir ${mnt}/nfs-ganesha/${dirname}/statd/sm
        ensure_dir ${mnt}/nfs-ganesha/${dirname}/statd/sm.bak
        ensure_file ${mnt}/nfs-ganesha/${dirname}/state
        ensure_file ${mnt}/nfs-ganesha/${dirname}/statd/state
        ensure_ln ${HA_MNT_DIR}/${HA_NFS_MNT_DIR}/nfs-ganesha/${dirname} ${staterefs}/${dirname}

        for server in ${HA_SERVERS} ; do
            if [[ "${server}" == *${dname} ]]; then
                server=${server}
            else
                server=${server}.${dname}
            fi
            if [ ${server} != ${dirname} ]; then
                ensure_ln ${HA_MNT_DIR}/${HA_NFS_MNT_DIR}/nfs-ganesha/.noderefs/${server}/ganesha ${mnt}/nfs-ganesha/${dirname}/ganesha/${server}
                ensure_ln ${HA_MNT_DIR}/${HA_NFS_MNT_DIR}/nfs-ganesha/.noderefs/${server}/statd ${mnt}/nfs-ganesha/${dirname}/statd/${server}
            fi
        done
        shift
    done

    umount ${mnt}
    rmdir ${mnt}
}

### Teardown functions

teardown_cluster()
{
    for server in ${HA_SERVERS} ; do
        if [[ ${HA_CLUSTER_NODES} != *${server}* ]]; then
            storlog "INFO" "${server} is not in config, removing"
            pcs cluster stop ${server} || storlog "WARN" "Failed: pcs cluster stop ${server}"
            pcs cluster node remove ${server} || storlog "WARN" "Failed: pcs cluster node remove ${server}"
        fi
    done

# BZ 1193433 - pcs doesn't reload cluster.conf after modification
# after teardown completes, a subsequent setup will appear to have
# 'remembered' the deleted node. You can work around this by
# issuing another
#   pcs cluster node remove $node
#  or
#   crm_node -f -R $server
#  or
#   cibadmin --delete --xml-text '<node id="$server" uname="$server"/>

    pcs cluster stop --all || storlog "WARN" "Failed to stop cluster ${name}"

    pcs cluster destroy || storlog "ERR" "Failed to destroy cluster ${name}"
}

### Cleanup functions

cleanup_config()
{
    local _cmd='eval "rm -rf ${SYS_CONFDIR}/cluster/cluster.conf*; \
rm -rf /var/lib/pacemaker/cib/*; \
rm -rf ${HA_NFS_EXPDIR}/*.conf; \
sed -r -i -e '"'"'/^%include[[:space:]]+\".+\\.conf\"$/d'"'"' ${HA_NFS_CONF}"'
    sshdo "${1}" "${_cmd}"
}

### AddNode functions

addnode()
{
    local node=${1}; shift
    local vip=${1}; shift
    local role=${1}; shift

    storlog "INFO" "Adding node ${node} to ${HA_NAME}"

    HA_CLUSTER_NODES="$HA_CLUSTER_NODES,$node"
    if [ "${role}" == *storage* ]; then
        STORAGE_NODES="$STORAGE_NODES,$node"
    fi
    if [ "x${vip}" != "x" ]; then
        HA_VIPS="${HA_VIPS} ${vip}"
        if [[ ${DETERMINISTIC_FAILOVER} == true && "x${HA_VIP_NODES}" != "x" ]]; then
            HA_VIP_NODES="${HA_VIP_NODES},${node}"
        fi
    fi
    determine_servers "add"

    pcs cluster node add ${node} || storlog "WARN" "Failed: pcs cluster node add ${node}"
    pcs cluster start ${node} || storlog "ERR" "Failed: pcs cluster start ${node}"

    if [ "${role}" == *storage* ]; then
        pcs property set --node ${node} role=storage || \
          storlog "WARN" "Failed: pcs property set --node ${node} role=storage"
    fi

    if [ "x${vip}" != "x" ]; then
        local cibfile=$(mktemp --tmpdir=$HA_CONF_secdir)
        pcs cluster cib ${cibfile}

        local ipcount=$(wc -w <<< "${HA_VIPS}")
        create_virt_ip ${cibfile} ${ipcount} ${vip}
        if [[ ${DETERMINISTIC_FAILOVER} == true ]]; then
            clear_virt_ip_constraints ${cibfile}
            for ((i=1;i<=${ipcount};i++)); do
                create_virt_ip_constraints ${cibfile} ${i}
            done
        fi
        pcs cluster cib-push ${cibfile} || \
          storlog "ERR" "Failed to add virtual IP resources."
    fi

    sed -i "s/\\(HA_CLUSTER_NODES=\\).*/\\1\"${HA_CLUSTER_NODES}\"/" ${HA_CONF}
    if [ "${role}" == *storage* ]; then
        if grep -q STORAGE_NODES ${HA_CONF}; then
            sed -i "s/\\(STORAGE_NODES=\\).*/\\1\"${STORAGE_NODES}\"/" ${HA_CONF}
        else
            echo "STORAGE_NODES=\"${STORAGE_NODES}\"" >> ${HA_CONF}
        fi
    fi
    if [ "x${vip}" != "x" ]; then
        sed -i "s/\\(HA_VIPS=\\).*/\\1\"${HA_VIPS}\"/" ${HA_CONF}
        if [[ ${DETERMINISTIC_FAILOVER} == true && "x${HA_VIP_NODES}" != "x" ]]; then
            sed -i "s/\\(HA_VIP_NODES=\\).*/\\1\"${HA_VIP_NODES}\"/" ${HA_CONF}
        fi
    fi
}

### DeleteNode functions

deletenode()
{
    local node=${1}; shift
    local vip=${1}; shift

    storlog "INFO" "Deleting node ${node} from ${HA_NAME}"

    HA_CLUSTER_NODES="${HA_CLUSTER_NODES//$node}"
    if [[ ${DETERMINISTIC_FAILOVER} == true && "x${HA_VIP_NODES}" != "x" ]]; then
        HA_VIP_NODES="${HA_VIP_NODES//$node}"
    fi
    if [[ "x${STORAGE_NODES}" != "x" ]]; then
        STORAGE_NODES="${STORAGE_NODES//$node}"
    fi
    determine_servers "delete"

    pcs cluster node remove ${node} || storlog "ERR" "Failed: pcs cluster node remove ${node}"

    if [[ ${DETERMINISTIC_FAILOVER} == true ]]; then
        local cibfile=$(mktemp --tmpdir=$HA_CONF_secdir)
        pcs cluster cib ${cibfile}

        local ipcount=$(wc -w <<< "${HA_VIPS}")
        clear_virt_ip_constraints ${cibfile}
        # TODO: delete_virt_ip ${cibfile} ${ipcount} ${vip}
        for ((i=1;i<=${ipcount};i++)); do
            create_virt_ip_constraints ${cibfile} ${i}
        done
        pcs cluster cib-push ${cibfile} || storlog "ERR" "Failed to refresh deterministic failover."
    fi

    sed -i "s/\\(HA_CLUSTER_NODES=\\).*/\\1\"${HA_CLUSTER_NODES}\"/" ${HA_CONF}
    if [[ ${DETERMINISTIC_FAILOVER} == true && "x${HA_VIP_NODES}" != "x" ]]; then
        sed -i "s/\\(HA_VIP_NODES=\\).*/\\1\"${HA_VIP_NODES}\"/" ${HA_CONF}
    fi
    if grep -q STORAGE_NODES ${HA_CONF}; then
        sed -i "s/\\(STORAGE_NODES=\\).*/\\1\"${STORAGE_NODES}\"/" ${HA_CONF}
    fi
}

### Refresh functions

# TODO: Move to RA
refresh_nfs_config()
{
        local VOL=${1}; shift
        local HA_CONFDIR=${1}; shift
        local tganesha_vol_conf=$(mktemp)

        cp ${HA_NFS_EXPDIR}/export.$VOL.conf ${tganesha_vol_conf}
        local get_id="cat $HA_NFS_EXPDIR/export.$VOL.conf | grep Export_Id | cut -d \" \" -f8"

        if [ -e ${SECRET_PEM} ]; then
        while [[ ${1} ]]; do
            _id=$(sshdo ${current_host} "${get_id}")
            _out=$(sshdo ${current_host} "dbus-send --print-reply \
--system --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \
org.ganesha.nfsd.exportmgr.RemoveExport uint16:$_id 2>&1")
            _ret=$?
            storlog "DEBUG" "${_out}"
            if [ ${_ret} -ne 0 ]; then
                storlog "ERR" "RemoveExport failed on ${current_host}."
            fi

            sleep 1
            sed -i s/Export_Id.*/"Export_Id= $_id ;"/ ${tganesha_vol_conf}
            if [ ${_hostname%%.*} != ${1%%.*} ]; then
                scpdo ${tganesha_vol_conf} \
                  ${current_host}:${HA_NFS_EXPDIR}/export.$VOL.conf
            fi

            _out=$(sshdo ${current_host} "dbus-send  --system \
--dest=org.ganesha.nfsd  /org/ganesha/nfsd/ExportMgr \
org.ganesha.nfsd.exportmgr.AddExport string:$HA_NFS_EXPDIR/export.$VOL.conf \
string:\"EXPORT(Path=/$VOL)\"")
            _ret=$?
            storlog "DEBUG" "${_out}"
            if [ ${_ret} -ne 0 ]; then
                storlog "ERR" "AddExport failed on ${current_host}."
            fi
            storlog "DEBUG" "refresh-config completed for ${current_host}."
            shift
        done
        else
            storlog "ERR" "refresh-config failed: Passwordless ssh is not enabled."
        fi
        rm -f ${tganesha_vol_conf}
}

### Mainline

cmd=${1}; shift
if [[ ${cmd} == *help || ${cmd} == "-h" ]]; then
    usage
    exit 0
elif [[ ${cmd} == *status ]]; then
    exec pcs status
    exit 0
fi

HA_CONF_secdir=$(mktemp -d --tmpdir "$(basename $0).XXXXXXXXXX")
HA_CONF_sec="$HA_CONF_secdir/sec.conf"

# Filter all config files into secure format
egrep '^#|^[^ ]*=[^;&]*'  "$HA_CONF" > "$HA_CONF_sec"
for conffile in `ls $HA_CONF_INCDIR/*.conf 2>/dev/null`; do
    egrep '^#|^[^ ]*=[^;&]*'  "$conffile" >> "$HA_CONF_sec"
done

# Source/load the config
. $HA_CONF_sec

parsebool "DETERMINISTIC_FAILOVER"

if [ -z "$HA_NFS_CONF" ]; then
    # Try loading the NFS-Ganesha config from various distro-specific locations
    if [ -f /etc/sysconfig/ganesha ]; then
        GANSYSCONF="/etc/sysconfig/ganesha"
    elif [ -f /etc/conf.d/ganesha ]; then
        GANSYSCONF="/etc/conf.d/ganesha"
    elif [ -f /etc/default/ganesha ]; then
        GANSYSCONF="/etc/default/ganesha"
    fi

    if [ -z "$GANSYSCONF" ]; then
        GANSYSOPTS=$(grep -s "OPTIONS" "$GANSYSCONF")
        if [ -n "$GANSYSOPTS" ] && grep -qs "-f" < "${GANSYSOPTS}"; then
            HA_NFS_CONF=$(sed -ne 's/^.*-f[= ]*([^\s]*)*/\1/p')
        fi
    fi
fi
HA_NFS_CONF="${HA_NFS_CONF:-/etc/ganesha/ganesha.conf}"
HA_NFS_EXPDIR="${HA_NFS_EXPDIR:-$(dirname ${HA_NFS_CONF})/exports}"

case "${cmd}" in
    setup | --setup)
        storlog "INFO" "Setting up ${HA_NAME}"
        check_cluster_exists ${HA_NAME}
        determine_servers "setup"

        if [ ${HA_NUM_SERVERS} -gt 1 ]; then
            setup_state_volume
            setup_cluster
            setup_create_resources
            copy_config ${HA_SERVERS}
        else
            storlog "ERR" "Insufficient servers for HA, aborting"
        fi
        ;;
    teardown | --teardown)
        storlog "INFO" "Tearing down ${HA_NAME}"
        determine_servers "teardown"
        teardown_cluster
        ;;
    cleanup | --cleanup)
        cleanup_config $_host
        ;;
    cleanup-all | --cleanup-all)
        for server in ${HA_SERVERS}; do
            cleanup_config $server
        done
        ;;
    add | --add)
        node=${1}; shift
        vip=${1}; shift
        role=${1}; shift
        copy_nfs_config ${node}
        addnode ${node} ${vip} ${role}
        copy_config ${HA_SERVERS}
        ;;
    delete | --delete | remove | --remove)
        node=${1}; shift
        deletenode ${node}
        copy_config ${HA_SERVERS}
        ;;
    *)
        storlog "ERR" "Unknown argument: ${cmd}"
        ;;
esac

rm -rf $HA_CONF_secdir
