#!/bin/bash
####################################################################
# (C)  Copyright 2017 Hewlett Packard Enterprise Development LP.
# @(#) Serviceguard Support for VMware SRM configuration script.
# @(#) This script should be used as post power on script which 
# @(#) will be executed on protected virtual machines.
# @(#) Product Name                :  HP Serviceguard
# @(#) Product Version             :  A.12.10.00
# @(#) Patch Name                  :  
#
# *** Note: This file MUST NOT be edited. *****
#
# Any changes made to it will be overwritten when you upgrade to the
# next release of HP Serviceguard.
#
# Changing this file may lead to unrecoverable package startup
# and/or shutdown problems.
#
# NOTE: Using this script in non-VMware SRM environment is not
#       supported and it will return wrong values.
###################################################################

###########################
# Globals
###########################

# Get the VMware SRM Environment Variables.

SRM_GUESTNAME=$VMware_VM_GuestName
SRM_VCENTER=$VMware_VC_Host
SRM_RECOVERY_MODE=$VMware_RecoveryMode
SRM_RECOVERY_PLAN_NAME=$VMware_RecoveryName
SRM_VMNAME=$VMware_VM_Name

if [ -f /etc/redhat-release ]
then
    LOGGER="/usr/bin/logger"
else
    LOGGER="/bin/logger"
fi

SYSLOGGER_CMD="$LOGGER -i -t cmsrmconfig"
SYSLOGGER_CMD_BEGIN="$SYSLOGGER_CMD -- --- HPE Serviceguard post power on script logs for VMware SRM recovery ---"
SYSLOGGER_CMD_END="$SYSLOGGER_CMD -- --- End of logs for HPE Serviceguard post power on script for VMware SRM recovery ---"

###########################
# Source utility functions.
###########################
. /etc/cmcluster.conf
SG_UTILS=${SGCONF}/scripts/mscripts/utils.sh
if [[ -f ${SG_UTILS} ]]; then
    . ${SG_UTILS}
    if (( $? != 0 ))
    then
        sg_log 0 "ERROR: Unable to source package utility functions file: ${SG_UTILS}"
        exit 1
    fi
else
    sg_log 0 "ERROR: Unable to find package utility functions file: ${SG_UTILS}"
    exit 1
fi

####################################################################
## Source the VMFS utility functions:
####################################################################
VMFS_UTIL=${SGCONF}/scripts/sg/vmfs_util.sh
if [[ -f ${VMFS_UTIL} ]]; then
    . ${VMFS_UTIL}
    if (( $? != 0 ))
    then
        sg_log 0 "ERROR: Unable to source package vmfs utility functions file: ${VMFS_UTIL}"
        exit 1
    fi
else
    sg_log 0 "ERROR: Unable to find package vmfs utility functions file: ${VMFS_UTIL}"
    exit 1
fi

VMFS_COMMON_UTIL=${SGCONF}/scripts/sg/vmfs_common_util.sh
if [[ -f ${VMFS_COMMON_UTIL} ]]; then
    . ${VMFS_COMMON_UTIL}
    if (( $? != 0 ))
    then
        sg_log 0 "ERROR: Unable to source package vmfs utility functions file: ${VMFS_COMMON_UTIL}"
        exit 1
    fi
else
    sg_log 0 "ERROR: Unable to find package vmfs utility functions file: ${VMFS_COMMON_UTIL}"
    exit 1
fi

# Get new vcenter credentials
function set_vcenter_credentials()
{
    typeset ret
    typeset vcen_name=$3
    typeset username=$1
    typeset pswd=$2
    typeset epswd
    
    echo "HostName=$vcen_name" > $SRM_LOG_DIR/vcenter_credential
    echo "UserName=$username" >> $SRM_LOG_DIR/vcenter_credential

    epswd=`java -cp $SGLIB/serviceguard_ds.jar:$SGLIB/serviceguard_pe.jar com.hpe.serviceguard.vmfs.srm.pswdEncrypt $pswd`
    if [ -z $epswd ]; then
        sg_log 0 "ERROR: Password encryption for vcenter $vcen_name failed"
        rm -rf $SRM_LOG_DIR/vcenter_credential
        return 1;
    fi
    echo "Password=$epswd" >> $SRM_LOG_DIR/vcenter_credential

    $SGSBIN/cmvmusermgmt -U -F $SRM_LOG_DIR/vcenter_credential -z
    ret=$?
    rm -rf $SRM_LOG_DIR/vcenter_credential
    if [ $ret -ne 0 ]; then
        sg_log 0 "ERROR: cmvmusermgmt failed to update $vcen_name credentials"	
        return 1;
    fi

    return 0
}

# This function will reverse the second and third field 
# of the mapping file.
function reverse_mapping_file
{
    reversed_file=/var/tmp/mapping_reverse
     if [[ -f $reversed_file ]]; then
         rm -f $reversed_file
     fi
     awk -F '|'  '{ print $1 " | " $3 " | " $2}' $mapping_file > $reversed_file
     if [[ $? -ne 0 ]]; then
         $SYSLOGGER_CMD_BEGIN
         $SYSLOGGER_CMD "ERROR:Failed to reverse SRM map file."
         $SYSLOGGER_CMD_END
         srm_cleanup_before_exit 1
     fi
     sed -r "s/[[:space:]]+\|[[:space:]]+\|/ /g" $reversed_file > $mapping_file
     if [[ $? -ne 0 ]]; then
         $SYSLOGGER_CMD_BEGIN
         $SYSLOGGER_CMD "ERROR:Failed to reverse SRM map file."
         $SYSLOGGER_CMD_END
         srm_cleanup_before_exit 1
     fi
     rm -f $reversed_file
}


# This function will convert the existing cluster configuration with new vCenter
# name. New vCenter will be passed as input parameter.
function do_convert_vcenter ()
{
   typeset converting_vcenter=$1
    if [ -f $SGCONF/cmclconfig ] ; then
        # Make backup copy of the  Serviceguard configuration file.
        cp -f $SGCONF/cmclconfig $SRM_LOG_DIR/cmclconfig.srm_save
        # Invoke the conversion utility
        $SGSBIN/convert -v $converting_vcenter
        ret=$?
        if [ $ret -ne 0 ]; then
            sg_log 0 "ERROR: Could not update vCenter $converting_vcenter information for"
            sg_log 0 "recovery site in HPE Serviceguard configuration file."
            mv -f $SRM_LOG_DIR/cmclconfig.srm_save $SGCONF/cmclconfig
            return 1
        fi
    else
        sg_log 0 "ERROR: HPE serviceguard configuration file is not present."
        return 1
    fi

    return 0
}

# This function will convert the existing cluster configuration database(CDB)
# with new network configurations read from a mapping file.
# The network configuration mapping will be updated by the user and will be
# present in the location $SGRUN/srm with file name srm-ip-config
function do_convert_networks()
{
    typeset ip_file=$1

    typeset convert_errors=$SGRUN/srm/srm-convert-errors

    sg_log 0 "Calling the convert utility to make the changes for"
    sg_log 0 "network configuration in CDB as per $ip_file"
    if [ -f $SGCONF/cmclconfig ] ; then
        # Make backup copy of the old configuration file.
        cp -f $SGCONF/cmclconfig $SRM_LOG_DIR/cmclconfig.srm_net_save

        # Invoke the conversion utility
        $SGSBIN/convert -i $ip_file 2> $convert_errors
        ret=$?
        if [ $ret -ne 0 ]; then
            $SYSLOGGER_CMD_BEGIN
            $SYSLOGGER_CMD "ERROR: Conversion failed to update the CDB for the network information."
            $SYSLOGGER_CMD -f $convert_errors
            rm -rf $convert_errors
 
            mv -f $SRM_LOG_DIR/cmclconfig.srm_net_save $SGCONF/cmclconfig
            return 1
        fi
    else
        sg_log 0 "ERROR: HPE serviceguard configuration file is not present."
        return 1
    fi

    rm -rf $convert_errors
    sg_log 0 "Subnet change in CDB done"
    return 0
}

mapping_file=$SGRUN/srm/srm-ip-config
function update_etc_hosts
{
    sed -i "s/^[ \t]*//" $mapping_file
    pri_dns_name=`cat $mapping_file | grep -E '^DNS_NAME' | awk -F '|' '{print $2}' | tr -d '[[:space:]]'`
    sec_dns_name=`cat $mapping_file | grep -E '^DNS_NAME' | awk -F '|' '{print $3}' | tr -d '[[:space:]]'`
    host_name=$(host=`hostname`; echo ${host%%.*})

    sed -i "s/^[ \t]*//" /etc/hosts
    while read line
    do
        if [[ $line =~ ^IP[[:space:]]+ ]] || [[ $line =~ ^IPV6_ ]]; then
            echo $line | grep "MASK" > /dev/null 2>&1
            if [[ $? -ne 0 ]]; then
                pri_ip=`echo $line | awk -F '|' '{print $2}' | tr -d '[[:space:]]'`
                sec_ip=`echo $line | awk -F '|' '{print $3}' | tr -d '[[:space:]]'`

                sed -i s/"^$pri_ip\(.*\)$pri_dns_name\(.*\)"/"$sec_ip\1$sec_dns_name\2"/ /etc/hosts
            fi
        fi
    done <<< "$(cat $mapping_file)"
    while read line
    do
        if [[ $line =~ ^NODE_NAME ]] ; then
            cur_node=`echo $line | awk -F '|' '{print $2}' | tr -d '[[:space:]]'`
            if [[ "$host_name" = "$cur_node" ]]; then
                flag=1
            else
                flag=0
            fi
        fi
        if [[ $flag -eq 1 ]]; then
            if [[ $line =~ ^IP[[:space:]]+ ]] || [[ $line =~ ^IPV6_ ]]; then
                echo $line | grep "MASK" > /dev/null 2>&1
                if [[ $? -ne 0 ]]; then
                    ip=`echo $line | awk -F '|' '{print $3}' | tr -d '[[:space:]]'`
                    grep -E '^::1' /etc/hosts > /dev/null 2>&1
                    if [[ $? -eq 0 ]]; then
                        sed -i -e "/^::1/a $ip\t$cur_node\.$sec_dns_name\t$cur_node" /etc/hosts
                    else
                        sed -i -e "/^127.0.0.1/a $ip\t$cur_node\.$sec_dns_name\t$cur_node" /etc/hosts
                    fi
                fi
            fi
        fi
    done <<< "$(cat $mapping_file)"

}

function srm_cleanup_before_exit
{
    typeset -i _rc=$1

    # Cleanup all temporary files
    # For this process all the files will have the prefix similar to below
    # Log_30564_10_39_02_579991896
    # ^^^^^^^^  ^^^^^^^^^^^^^^^^^^
    # Log_$$    $(date +"%H_%M_%S_%N")

    # Cleanup the saved cmclconfig file(if any)
    # It will be under SRM_LOG_DIR=${SGRUN}/srm
    # Before deleting copy bring back the old cmclconfig
    if [[ $_rc -eq 1 ]]; then
        if [ -f $SRM_LOG_DIR/cmclconfig.srm_save ] ; then
            mv -f $SRM_LOG_DIR/cmclconfig.srm_save $SGCONF/cmclconfig > /dev/null 2>&1
        else
            mv -f $SRM_LOG_DIR/cmclconfig.srm_net_save $SGCONF/cmclconfig > /dev/null 2>&1
        fi
    fi        
    rm -f $SRM_LOG_DIR/cmclconfig.srm_save > /dev/null 2>&1
    rm -f $SRM_LOG_DIR/cmclconfig.srm_net_save > /dev/null 2>&1
    rm -f $_input_file > /dev/null 2>&1

    # Exit to indicate success/failure to VMware SRM plan.
    #
    cleanup_before_exit $_rc
}

#MAIN
typeset script_name="cmsrmconfig"

# File Handling
srm_date_prefix=$(date +"%H_%M_%S_%N")
srm_log_date_prefix=$(date +"%d_%m_%y_%H_%M_%S_%N")
srm_str_prefix="Log_"$$_
srm_log_prefix=$srm_str_prefix$srm_date_prefix
log_prefix_for_srm=$script_name"_post-power_on_script_"$srm_log_date_prefix

# Validate default SRM log directory exists
# If log directory does not exist create it.
#
if [[ ! -d ${SGRUN}/srm ]]
then
    mkdir ${SGRUN}/srm
    if (( $? != 0 ))
    then
       sg_log 0 "ERROR: Creating directory ${SGRUN}/srm"
       exit 1
    fi
    chmod 555 ${SGRUN}/srm
fi

# This script execution logs will be stored under $SGRUN/srm directory.
# For SLES log directory: /usr/local/cmcluster/run/srm/<log_file_name>
# For RHEL log directory: /opt/cmcluster/run/srm/<log_file_name>
# Example Log file name will be like below:
#  cmsrmconfig_post-power_on_script_18_49_17_031449573_recovery.log
#  ^^^^^^^^^^^                       ^^^^^^^^  ^^^^^^^^^
# $script_name"_post-power_on_script_"$(date +"%H_%M_%S_%N")

typeset tmp_dir="/var/tmp"
typeset SRM_LOG_DIR=${SGRUN}/srm
typeset _input_file=$tmp_dir/$srm_log_prefix"_input_file_$$.txt"
typeset SG_SCRIPT_LOG_FILE=$SRM_LOG_DIR/$log_prefix_for_srm"_recovery.log"
typeset -i ucount=0
typeset -i pcount=0

declare -a all_pkg_list
declare -a only_vmfs_pkgs
typeset -i R=0
typeset package_node_name=""
typeset -i local_node_not_least=0
typeset -i local_node_eligible_to_detach=0
typeset vmfs_output=""
typeset -i vcenter_found_in_cluster=0

local_host_name=$(echo $(uname -n) | sed -e 's/\..*$//')
typeset CMVIEWCL="$SGSBIN/cmviewcl"
typeset VMWUTIL="$SGSBIN/vmwutil"

date=`date`

sg_log 0 "+----------------------------------------------------------------------+"
sg_log 0 "Logs of HPE Serviceguard post power on script $script_name executed on"
sg_log 0 "Virtual Machine: $local_host_name"
sg_log 0 "Date: $date"
sg_log 0 "Recovery Plan Name: $SRM_RECOVERY_PLAN_NAME"
sg_log 0 "+----------------------------------------------------------------------+"
   
# This script must be executed in VMware SRM environment with Serviceguard 
# Enterprise license. 

# Check for Enterprise License

license=`$SGSBIN/cmgetlicense -f line  |  head -n 1 | cut -f 2 -d '='`
lic=`echo $license| sed 's/^ *//g' | sed 's/ *$//g'`
if [[ "$lic" ==  "Enterprise" ]] || [[ "$lic" ==  "Instant_ON" ]]; then
   sg_log 0 "SUCCESS: $local_host_name contains valid $lic license."
else
   $SYSLOGGER_CMD_BEGIN
   $SYSLOGGER_CMD "ERROR: Invalid license on $local_host_name."
   $SYSLOGGER_CMD "Execution of $script_name is supported on a system "
   $SYSLOGGER_CMD "which has valid Enterprise License."
   $SYSLOGGER_CMD "Contact HPE for purchase of Enterprise License."
   $SYSLOGGER_CMD_END
   exit 1
fi

# Check for type of recovery from the protected site to the recovery site.
# In Serviceguard environment only Planned Migration and Disaster recovery
# are supported. Testing the recovery is not supported because we won't be
# able to form a test cluster with applications on recovery site while the
# protected site is having up and running cluster. Through test networks
# we can't communicate to QS, VMs of other hosts and vCenter from the 
# recovery site.

if [ "$SRM_RECOVERY_MODE" == "test" ]; then
   
   $SYSLOGGER_CMD_BEGIN
   $SYSLOGGER_CMD "Unsupported:VMware SRM recovery plan is called in test mode on $local_host_name."
   $SYSLOGGER_CMD "WARNING: Running the recovery plan in test mode is not supported for HPE"
   $SYSLOGGER_CMD "Serviceguard cluster nodes. However script returns success to allow the"
   $SYSLOGGER_CMD "recovery plan to execute to its logical conclusion."
   $SYSLOGGER_CMD_END
   exit 0
fi

# Get the new vcenter username and password from user input
while getopts ":u:p:h" opt; do
    case "$opt" in
        u)
            username="$OPTARG"
            ucount=$(($ucount+1))
            ;;
        p)
            pswd="$OPTARG"
            pcount=$(($pcount+1))
            ;;
        h)
            echo "Usage : cmsrmconfig -u <username> -p <password>"
            echo "INFO: This configuration script must be used in VMware SRM"
            echo "environment. Any other usage will lead to irrelavant results "
            echo "and it is unsupported."
            exit 0
            ;;
        *)  echo "ERROR:Invalid options for cmsrmconfig script."
            echo "Usage : cmsrmconfig -u <username> -p <password>"
            $SYSLOGGER_CMD_BEGIN
            $SYSLOGGER_CMD "ERROR:Invalid options for cmsrmconfig script."
            $SYSLOGGER_CMD "Usage: cmsrmconfig -u <username> -p <password>"
            $SYSLOGGER_CMD "WARNING: This configuration script must be used in VMware SRM"
            $SYSLOGGER_CMD "environment. Contrary to use in VMware SRM environment is"
            $SYSLOGGER_CMD "unsupported and may lead to producing the undefined results."
            $SYSLOGGER_CMD_END
            exit 1
            ;;
    esac
done
shift $(( OPTIND - 1))

if [[ $pcount > 1 ]] || [[ $ucount > 1 ]]; then

    echo "ERROR:Invalid number of options for cmsrmconfig script."
    echo "Usage: cmsrmconfig -u <username> -p <password>"

    $SYSLOGGER_CMD_BEGIN
    $SYSLOGGER_CMD "ERROR:Invalid number of options for cmsrmconfig script."
    $SYSLOGGER_CMD "Usage: cmsrmconfig -u <username> -p <password>"
    $SYSLOGGER_CMD_END
    exit 1
fi

# Check if recovery vCenter is primary vCenter
# if yes, reverse the mapping file.
if [[ -z $SRM_VCENTER ]]; then
     $SYSLOGGER_CMD_BEGIN
     $SYSLOGGER_CMD "ERROR: Unable to get recovery site vCenter name from VMware SRM environment variable."
     $SYSLOGGER_CMD_END
     srm_cleanup_before_exit 1
else
    if [[ -f $mapping_file ]]; then
        primary_vcenter=`cat $mapping_file | grep -e '^VCENTER'| awk -F '|' {'print $2'}`
        vcenter_name=`cat /etc/hosts| grep -w ${primary_vcenter} | awk '{print $1" "$2" "$3}'| sort -u`
        for i in $vcenter_name
        do
            if [ "$i" = "$SRM_VCENTER" ]; then
                reverse_mapping_file
            fi
        done
    fi
fi

ip_config_file=$SGRUN/srm/srm-ip-config
if [ -f $ip_config_file -a -s $ip_config_file ]; then
   update_etc_hosts
fi

# Check for Serviceguard cluster configuration
if [ -f $SGCONF/cmclconfig ] ; then
   cache_local_variables
else
   $SYSLOGGER_CMD_BEGIN
   $SYSLOGGER_CMD "WARNING: HPE Serviceguard cluster is not configured in the node."
   $SYSLOGGER_CMD "Returning SUCCESS to VMware SRM recovery plan."
   $SYSLOGGER_CMD_END
   srm_cleanup_before_exit 0
fi

# Check for only RDM disk package configuration in the cluster.
# Incase if the script is accidently executed on non-VMFS package environment. Inform the user
# and return success. We don't want to fail the execution of the script for RDM package cluster.
# Even though we have documented about not to use this script in only RDM package cluster. It's
# better if we handle it in the script also.

num_vmdk=`cat "$cmviewcl_output" | grep -w vmdk_file_name | wc -l`
if [[ $num_vmdk -eq 0 ]]; then
   sg_log 0 "WARNING: None of the package configuration in Serviceguard cluster contains"
   sg_log 0 "the VMFS type of storage. This script is intended to be used when at least one"
   sg_log 0 "VMFS disk is used in the package configuration or network configuration between"
   sg_log 0 "protected site and recovery site are different."  
fi

#Check for Serviceguard version A.12.10.00 or later.
version=`cat "$cmviewcl_output" | grep -w $local_host_name | awk -F= '/^node:[^|]*\|sg_version=/ {print $2}'`

if [[ "$version" < "A.12.10.00" ]]; then

   $SYSLOGGER_CMD_BEGIN
   $SYSLOGGER_CMD "ERROR: Minimum required Serviceguard version not installed on all nodes."
   $SYSLOGGER_CMD "Co-existence support for VMware SRM feature requires Serviceguard version"
   $SYSLOGGER_CMD "A.12.10.00 or higher to be installed on all cluster nodes."
   $SYSLOGGER_CMD_END
   srm_cleanup_before_exit 1
fi

# Get the vcenter name from VMware SRM environment variable.

to_be_changed_vcenter=$SRM_VCENTER
if [ -z $to_be_changed_vcenter ]; then

   $SYSLOGGER_CMD_BEGIN
   $SYSLOGGER_CMD "ERROR: VMware SRM environment variable returned NULL value for "
   $SYSLOGGER_CMD "host name of the vCenter Server at the recovery site. For the "
   $SYSLOGGER_CMD "successful execution of the script it is mandatory to have "
   $SYSLOGGER_CMD "vCenter server name. Hence failing the script execution."
   $SYSLOGGER_CMD_END
   srm_cleanup_before_exit 1
fi

# Before calling the conversion check for vCenter presence in the cluster
# configuration. If it is not present fail the recovery plan and return.
vcenter_present=`cat "$cmviewcl_output" | grep vcenter_name`
ret=$?
if [[ $ret -eq 1 ]]; then

    vcenter_found_in_cluster=0
    $SYSLOGGER_CMD_BEGIN
    $SYSLOGGER_CMD "FAILURE: HPE Serviceguard configuration file does not contain the vCenter "
    $SYSLOGGER_CMD "information associated with recovery site vCenter $to_be_changed_vcenter. "
    $SYSLOGGER_CMD "vCenter is a mandatory information which needs to be present in cluster "
    $SYSLOGGER_CMD "configuration. Hence failing the recovery plan."
    $SYSLOGGER_CMD_END
    srm_cleanup_before_exit 1
fi

if [[ $ret -eq 0 ]] && [[ ! -z "$vcenter_present" ]]; then
    vcenter_found_in_cluster=1
fi

# From the output of cmviewcl get the list of packages which are running on
# the cluster. Do not construct input file for non least members.

package_node_name=`cat "$cmviewcl_output" | awk '/^node:[^|]*\|name=/' | cut -d "=" -f 2`

all_pkg_list=`cat "$cmviewcl_output" | awk '/^package:[^|]*\|name=/' | cut -d "=" -f 2`
# Trim the leading/trailing spaces and new lines from the package list.
all_pkg_list=`echo $all_pkg_list| sed 's/^ *//g' | sed 's/ *$//g'`

vmware_package_node_names=`create_vmware_only_package_node_list "$package_node_name"`

package_node_names=$vmware_package_node_names

# Before proceeding check whether the local node is least node or not?
find_if_local_node_has_least_node_id
ret=$?
if [[ $ret -ne 0 ]]; then
    local_node_not_least=1
else
 # This is the eligible node. Return TRUE
  local_node_eligible_to_detach=1
fi

if [[ $local_node_eligible_to_detach -eq 1 ]] && [[ $vcenter_found_in_cluster -eq 1 ]]; then

   sg_log 0 "Updating Serviceguard Credential Store(SCS) database"
   sg_log 0 "with recovery site vCenter user credentials "

   set_vcenter_credentials "$username" "$pswd" "$SRM_VCENTER"
   vcenter_credential_ret=$?

   if [[ $vcenter_credential_ret -eq 0 ]]; then
      sg_log 0 "SUCCESS: Updated SCS database with recovery site vCenter credentials"
   else
      $SYSLOGGER_CMD_BEGIN
      $SYSLOGGER_CMD "ERROR: SCS is not updated with recovery site vCenter credentials."
      $SYSLOGGER_CMD_END
      srm_cleanup_before_exit 1
   fi
fi

# Convert the network ips and subnets in CDB
ip_config_file=$SGRUN/srm/srm-ip-config
if [ -f $ip_config_file -a -s $ip_config_file ]; then
    do_convert_networks "$ip_config_file"
    ret_network_change=$?

    if [[ $ret_network_change -eq 0 ]]; then
        sg_log 0 "SUCCESS: Converting HPE Serviceguard CDB for the site"
        sg_log 0 "managed by vCenter Server $to_be_changed_vcenter"
    else       

       $SYSLOGGER_CMD "FAILURE: Converting HPE Serviceguard configuration file for the site "
       $SYSLOGGER_CMD "managed by vCenter Server $to_be_changed_vcenter failed. "
       $SYSLOGGER_CMD_END
       srm_cleanup_before_exit 1
    fi
else
    sg_log 0 "NOTE: SRM Mapping file for HPE Serviceguard Network configuration"
    sg_log 0 "is not present."
    sg_log 0 "This is a mandatory file only if your sites are on different networks."
fi

# Convert the existing configuration file from protected vCenter to recovery vCenter

if [[ $vcenter_found_in_cluster -eq 1 ]]; then
    do_convert_vcenter "$to_be_changed_vcenter"
    convert_vcenter_return=$?
fi

if [[ $convert_vcenter_return -eq 0 ]]; then
    sg_log 0 "SUCCESS: Converting the HPE Serviceguard configuration file with"
    sg_log 0 "recovery site vCenter $to_be_changed_vcenter is done."
else

    $SYSLOGGER_CMD_BEGIN
    $SYSLOGGER_CMD "FAILURE: Converting the HPE Serviceguard configuration file with "
    $SYSLOGGER_CMD "recovery site vCenter $to_be_changed_vcenter is failed."
    $SYSLOGGER_CMD_END
    srm_cleanup_before_exit 1
fi

# In some scenarios shutdown of the cluster nodes (from SRM recovery plan) at the
# primary site may leave LVS and VGS in activation state. It may lead to package
# start failures at the recovery site during cluster start. So we need to
# deactivate both of them.

lvs_list=`cat "$cmviewcl_output" | awk '/^package:[^|]*\|filesystem:[^|]*\|fs_name=/'| cut -d "=" -f 2`
lvs_list=`echo $lvs_list| sed 's/^ *//g' | sed 's/ *$//g'`
for lv in $lvs_list
do
    lvchange -a n $lv > /dev/null 2>&1
done

# Before sending detach request to vCenter lets deactivate all the VGs
# of the packages.

vgs_list=`cat "$cmviewcl_output" | awk '/^package:[^|]*\|vg:/' | cut -d "=" -f 2`
vgs_list=`echo $vgs_list| sed 's/^ *//g' | sed 's/ *$//g'`
for vg in $vgs_list
do
    vgchange -a n $vg > /dev/null 2>&1
done

# Lets handle the case of removing the attached disks from the node through new 
# vCenter. Some times there is a possibility of VMFS disks may remain attached to
# VMs during shutdown at the protected site. To avoid package failures during
# start we have to detach the disks at recovery site.

if [[ $local_node_eligible_to_detach -eq 1 ]] && [[ $vcenter_found_in_cluster -eq 1 ]] && [[ $num_vmdk -gt 0 ]]; then

    # Remove the earlier input files
    rm -f $_input_file

    # First create the uuids of all the nodes of the cluster.
    for node in $vmware_package_node_names
    do
       uuid=`get_uuid_from_node_name $node`
       echo "UUID:$uuid" >> $_input_file
    done

    # List only the VMFS type of packages to the input file.

    for pkg in $all_pkg_list
    do
        disk_type_of_pkg=`cat "$cmviewcl_output" | grep -w $pkg | grep disk_type | awk -F= '/^package:$pkg|vmdk_file_name:[^|]*\|disk_type=/ {print $2}'`
        if [[ $disk_type_of_pkg == "VMFS" ]] || [[ $disk_type_of_pkg == "RDM" ]]; then
            vmdk=`cat "$cmviewcl_output" | grep -w $pkg | grep vmdk_file_name | awk -F= '/^package:$pkg|vmdk_file_name:[^|]*\|vmdk_file_name=/ {print $2}'`
            ds=`cat "$cmviewcl_output" | grep -w $pkg   | grep datastore_name | awk -F= '/^package:$pkg|vmdk_file_name:[^|]*\|datastore_name=/ {print $2}'`
            slot=`cat "$cmviewcl_output" | grep -w $pkg | grep scsi_controller| awk -F= '/^package:$pkg|vmdk_file_name:[^|]*\|scsi_controller=/ {print $2}'`  
            echo "VMDK:$vmdk" >> $_input_file
            echo "DS:$ds" >> $_input_file
            echo "SLOT:$slot" >> $_input_file
        fi 
    done

    #Detach request from least member VM to new vCenter.

    cmd="/usr/bin/java -jar $VMWUTIL -o detach"
    vmfs_output=`$cmd -H $to_be_changed_vcenter -F $_input_file`
    detach_ret=$?
    if [[ $detach_ret -eq 0 ]]; then
        sg_log 0 "Request to detach the disks from node $local_host_name to vCenter $to_be_changed_vcenter is successful."
    else

        $SYSLOGGER_CMD_BEGIN
        $SYSLOGGER_CMD "FAILURE: Request to detach the disks from node $local_host_name to vCenter $to_be_changed_vcenter is unsuccessful."
        $SYSLOGGER_CMD "Returning failure to recovery plan execution."
        $SYSLOGGER_CMD_END
        if [ -f $SRM_LOG_DIR/cmclconfig.srm_save ] ; then
            mv -f $SRM_LOG_DIR/cmclconfig.srm_save $SGCONF/cmclconfig
        fi
        srm_cleanup_before_exit 1
    fi
fi # end of local_node_eligible_to_detach

# Test for OS versions for the command to restart the cmproxyd

test -f /etc/redhat-release && \
if /bin/grep "Maipo" /etc/redhat-release >/dev/null 2>&1
then
    OS_Version="RH7"
else
    OS_Version="LESS_THAN_RH7"
fi

if [ -f /etc/SuSE-release ]; then
version=`/bin/grep "VERSION" /etc/SuSE-release |  awk -F '= ' '{print $2}'`
if [[ "$version" -le  "11" ]]; then
      OS_Version="LESS_THAN_SLES12"
   else
      OS_Version="SLES12"
   fi
fi

if [[ $OS_Version == "RH7" || $OS_Version == "SLES12" ]]; then
  systemctl restart cmproxy.service >/dev/null 2>&1
  ret=$?
else
  service cmproxy restart >/dev/null 2>&1
  ret=$?
fi

if [[ $ret -eq 1 ]]; then

    $SYSLOGGER_CMD_BEGIN
    $SYSLOGGER_CMD "FAILURE: Request to restart the cmproxyd daemon is failed. It may result in failure of package start."
    $SYSLOGGER_CMD "Returning failure to recovery plan execution."
    $SYSLOGGER_CMD_END
    if [ -f $SRM_LOG_DIR/cmclconfig.srm_save ] ; then
        mv -f $SRM_LOG_DIR/cmclconfig.srm_save $SGCONF/cmclconfig
    fi
    srm_cleanup_before_exit 1
else
    sg_log 0 "SUCCESS: Request to restart the cmproxyd daemon succeeded."
fi

# Post convert of the vCenter and network configurations
# run the node to start the cluster and packages.
if [[ $convert_vcenter_return -eq 0 ]]; then
    if [[ $detach_ret -eq 0 ]] || [[ $local_node_not_least -eq 1 ]]; then
        $SGSBIN/cmrunnode -v
        run_ret=$?
        if [[ $run_ret -eq 1 ]]; then

            $SYSLOGGER_CMD_BEGIN
            $SYSLOGGER_CMD "FAILURE: cmrunnode on node $local_host_name failed."
            $SYSLOGGER_CMD "Correct the reported errors and attempting the operation again may succeed."
            $SYSLOGGER_CMD_END
            if [ -f $SRM_LOG_DIR/cmclconfig.srm_save ] ; then
                mv -f $SRM_LOG_DIR/cmclconfig.srm_save $SGCONF/cmclconfig
            fi
            srm_cleanup_before_exit 1
        fi

        # Log the cmrunnode status to log file.
        if [[ $run_ret -eq 0 ]]; then
            sg_log 0 "SUCCESS: cmrunnode on $local_host_name is succeeded."
        fi

    fi
fi

# Final status of the script
if [[ $convert_vcenter_return -eq 0 ]] && [[ $run_ret -eq 0 ]]; then

    $SYSLOGGER_CMD_BEGIN
    $SYSLOGGER_CMD "Successfully completed the execution of post power on script."
    $SYSLOGGER_CMD "Check the syslog files and package log files on all nodes in the cluster"
    $SYSLOGGER_CMD "to verify that no warnings/errors occurred during startup."
    $SYSLOGGER_CMD_END

    sg_log 0 "----Successfully completed the execution of post power on script---"
    sg_log 0 "Check the syslog files and package log files on all nodes in the cluster"
    sg_log 0 "to verify that no warnings/errors occurred during startup."
    sg_log 0 "End of script: $script_name, returning SUCCESS to VMware SRM plan."
    sg_log 0 "*********************************************************************"
    srm_cleanup_before_exit 0
fi
