#!/usr/bin/perl -w
##############################################################################
# (C) Copyright 2013-2015 Hewlett-Packard Enterprise Development Company, L.P.
# @(#) Serviceguard shared storage creation command
# @(#) Product Name                : HP Serviceguard
# @(#) Product Version             : A.12.10.00
# @(#) Patch Name                  : 
##############################################################################
use strict;

use Getopt::Long qw(GetOptions HelpMessage :config bundling no_ignore_case);
use Pod::Usage;
use Sys::Hostname;
use File::Path;
use Data::Dumper;
use File::Basename;
use lib '/opt/drenabler/utils/';
eval 'use MCEDTools';
my $sgeasylib = undef;
my @file_out = undef;

BEGIN {
    my $sgconffile = "/etc/cmcluster.conf";
    if (-f $sgconffile) {
        @file_out = qx(cat $sgconffile);
    } else {
        print STDERR "ERROR: Unable to run cmpreparestg command on ".
                     "unsupported OS\n";    
      exit 1;
    }
    
    my @sglib_out = grep(/^SGLIB/, @file_out);
    my @split_sglib_name = split(/=/,$sglib_out[0]);
    chomp($split_sglib_name[1]);
    $sgeasylib = $split_sglib_name[1]; 
}
use lib "$sgeasylib";
use EDTools;
use EDPkgTools;
=head1 NAME

cmpreparestg - create a shared volume group for specified/cluster nodes.

=cut

# agruments
# -n <node>          --node         Name of the node. If none local
#                                   node is assumed.
# -t                 --preview      Preview only
# -l vgname          --vgname       LVM volume group name
# -g dgname          --dgname       VxVM disk group name
# -p pv_path         --pv           Create vg1 on disk1
# -P filename        --pvs_file     File containing list of PVs.
# -L [lvname]        --lvname       LVM/VxVM logical volume name
# -c lv_counts       --lv_counts    Creates logical volumes equal to
#                                   counts
# -m mountpoint      --mountpoint   File system mount point
# -d locklun         --locklun      Create Locklun
# -v verbose         --verbose      Display command executed by
#                                   cmpreparestg
# -o "option string" --optionstring String containing options for
#                                   underlying command called by
#                                   cmpreparestg. Option strings
#                                   are supported for creation of
#                                   volume group, disk group,
#                                   locklun, logical volume &
#                                   file system.
#   EXAMPLES:
#            They can be specified as
#            -o vg_opts="-s 32 -p 10 -l 5" for LVM volume group,
#            -o dg_opts="cds=off" for VxVM disk group,
#            -o lv_opts="-L 100M -m 1" for LVM & VxVM logical volume,
#            -o lv_opts="10G layout=mirror nmirror=2" for VxVM lv,
#            -o fs_opts="-t ext3" for file system [On Linux in case of LVM  ]
#            -o fs_opts="-t vxfs" for file system [On Linux in case of VxVM ]
#            -o fs_opts="-F vxfs" for file system [On HP-UX in case of LVM
#                                                            as well as VxVM]
#


=head1 SYNOPSIS

cmpreparestg
[-t]
S<[-n node]...>
S<{-l vgname| -g dgname} [{-p pv_path ...| -P pvs_file}]>
S<{-L [lvname]|-c lv_counts}>
S<[-m mountpoint]>
S<[-o "option string" ...]>
S<[-v]>

cmpreparestg [-h]

=cut

# Constant
use constant EXPECT_SUCCESS => 0;
use constant EXPECT_ANY => 1;

# Globals (@ARGV is also global)
my (@nodes, @allnodes, @localnodes, @remotenodes, $preview, @block_pvs,
    @tot_pvs, @opt_pvs, $opt_pvs_file, $opt_vgname, $opt_dgname,
    $opt_lvname, $opt_lv_counts, $opt_mountpoint, $localnode,
    $opt_verbose, @opt_opstring);

# It need to be updated if number of node supported by Serviceguard
# increases.
my $MAX_NUMBER_OF_NODE_SUPPORTED_BY_SG = 32;
my $sgeasyTool   = new EDTools("cmpreparestg");
my $opt_locklun  = 0;
my $vg_activated = 0;
my $dg_imported  = 0;
my $vg_opts      = "";
my $dg_opts      = "";
my $lv_opts      = "";
my $fs_opts      = "";
my $MC_SCRIPT    = "/opt/drenabler/utils/MCEDTools.pm";
my $SGBIN        = get_sgsbin_path(); # SGSBIN value obtained from
                                      # /etc/cmcluster.conf
my $SGLIB        = $sgeasyTool->get_sglib_path();  # SGLIB value obtained from
                                                   # /etc/cmcluster.conf
my $SG_CONF_PATH = $sgeasyTool->get_sgconf_path(); # SGCONF value obtained from
                                                   # /etc/cmcluster.conf
my $SBIN         = "/sbin";
my $BIN          = "/bin";
my $MKFS         = "/sbin/mkfs";
my $SG_INQ       = "/usr/bin/sg_inq";
my $DM_DETAILS   = "$SBIN/dmsetup info";
my $MPATH_LIST   = "$SBIN/multipath -ll";
my $LVM_CONF     = "/etc/lvm/lvm.conf";
my $locklun_opt  = "-d|--locklun";
my $SGLX12       = 12;
my $MAX_LEN_OF_DEV_SUPPORTED = 39;
my ($distroName, $distroVer) = $sgeasyTool->getDistroInfo();
$preview         = 0;

my $os_name = "$distroName$distroVer";

## CMPREPARESTG
# It calls main function which configures storages
&main();

#------------------------------------------------------------------------------
#
# Subroutine    : check_for_root_user()
# Calls         : -
# Called by     : main()
# Globals       : -
# Input Params  : user of command and command name
# Return Value  : -
#
# This function checks if user of command is root or not.
# If command's user is non-root then command will be failed.
#-----------------------------------------------------------------------------
sub check_for_root_user {
    my ($user, $command_name) = @_;
    if ($user != 0) {
        print STDERR "ERROR: Must be root to use $command_name command\n";
        exit(1);
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : set_local_node()
# Calls         : -
# Called by     : main()
# Globals       : $localnode
# Input Params  : $localnode
# Return Value  : -
#
# This function takes reference of global var $localnode, it populates with
# local host name.
#-----------------------------------------------------------------------------
sub set_local_node {
   my ($localnode_ref) = @_;
   $$localnode_ref = hostname();
   chomp $$localnode_ref;
   my @temp = split(/\./, $$localnode_ref);
   $$localnode_ref = shift(@temp);
}

#------------------------------------------------------------------------------
#
# Subroutine    : fail()
# Calls         : -
# Called by     : main(), test_cmexec_on_all_nodes(), createBackup(),
#                 check_for_duplicate_entries(), check_sg_version_and_setup_node(),
#                 validate_input(), handle_pvs_file(), check_if_vg_activated(),
#                 getRemNodeDiskByRawDsk(), validateDiskIsUnused(),
#                 getRemNodeDevice(), populate_pvs().
# Globals       : -
# Input Params  : Message to be shown as error
# Return Value  : -
#
# This function logs error into log file as well as display error msg on
# stdout. It also do retore of lvm conf file and deactive/deport
# in case of vg opt and dg opt respectivly.
#-----------------------------------------------------------------------------
sub fail {
    my $msg = shift;
    my $cmd = "";
    my $exitcode = 0;
    if (!$opt_locklun) {
        $sgeasyTool->screen_and_log($msg, "cmpreparestg:");
    }
    if ($opt_locklun) {
       print STDERR "$msg\n";
    }
    doRestore($LVM_CONF);
    # Restore back to the orignal VG activation mode before
    # this command
    if ($vg_activated) {
        $cmd = "$SBIN/vgchange -a n $opt_vgname";
        ($exitcode) = captureCmd($cmd, "Deactivating LVM volume group",
                                 $localnode, EXPECT_ANY);
        if ($exitcode) {
            print STDERR "WARNING: Unable to deactivate [$opt_vgname] ".
                         "LVM volume group\n";
        }
    }
    if ($dg_imported) {
        $cmd = "$SBIN/vxdg deport $opt_dgname";
        ($exitcode) = captureCmd($cmd, "Deporting VxVM disk group",
                                 $localnode, EXPECT_ANY);
        if ($exitcode) {
            print STDERR "WARNING: Unable to deport [$opt_dgname] ".
                         "VxVM disk group\n";
        }
    }
    exit(1);
}

#------------------------------------------------------------------------------
#
# Subroutine    : test_cmexec_on_all_nodes()
# Calls         : cmexec
# Called by     : main()
# Globals       : -
# Input Params  : List of node
# Return Value  : -
#
# This function test successful execution of cmexec command on all nodes.
#-----------------------------------------------------------------------------
sub test_cmexec_on_all_nodes {
    my (@node_list) = @_;
    foreach my $node (@node_list) {
        if ($sgeasyTool->test_cmexec($node) != 0) {
            fail("ERROR: Authentication failed for node $node.\n" .
                 " Run cmpreparecl(1m) command to setup ".
                 "\nauthentication on the cluster nodes.");
        }
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : parse_args()
# Calls         : -
# Called by     : -
# Globals       : @nodes, $preview, $opt_vgname, $opt_dgname, @opt_pvs,
#                 $opt_pvs_file, $opt_lvname, $opt_lv_counts, $opt_mountpoint,
#                 @opt_opstring, $opt_locklun
# Input Params  : None
# Return Value  : -
#
# This function does parsing of command line opts and arguments.
#-----------------------------------------------------------------------------
sub parse_args {
    GetOptions(
        "node|n=s"         => \@nodes,
        "preview|t"        => \$preview,
        "vgname|l=s"       => \$opt_vgname,
        "dgname|g=s"       => \$opt_dgname,
        "pv|p=s"           => \@opt_pvs,
        "pvs_file|P=s"     => \$opt_pvs_file,
        "lvname|L:s"       => \$opt_lvname,
        "lv_counts|c=i"    => \$opt_lv_counts,
        "mountpoint|m:s"   => \$opt_mountpoint,
        "optionstring|o=s" => \@opt_opstring,
        "locklun|d"        => \$opt_locklun,
        "verbose|v"        => \$opt_verbose,
        "help|h|?"         => sub { pod2usage(-verbose => 1) },
     ) || pod2usage("Invalid argument");

     if (@ARGV){
         pod2usage("Unknown arguments on command line : ".
                   join (" ", @ARGV));
     }
}

#------------------------------------------------------------------------------
#
# Subroutine    : setup_logging()
# Calls         : -
# Called by     : -
# Globals       : $sgeasyTool, $opt_locklun, $preview, @nodes
# Input Params  : None
# Return Value  : -
#
# This function does setting up log file to enable logging event into
# that file.
#-----------------------------------------------------------------------------
sub setup_logging {
    if ($opt_locklun == 0) {
        if ($preview) {
            $sgeasyTool->screen_and_syslog("Previewing cmpreparestg on nodes ".
                                           join(" ", @nodes));
        } else {
            $sgeasyTool->screen_and_syslog("Running cmpreparestg on nodes " .
                                           join(" ", @nodes));
            $sgeasyTool->screen_and_syslog("Saving command output to ".
                                           "$sgeasyTool->{logfile}");
            $sgeasyTool->screen_and_log("Running cmpreparestg on nodes ".
                                        join(" ", @nodes), "cmpreparestg:");
        }
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : check_for_duplicate_entries()
# Calls         : -
# Called by     : -
# Globals       : None
# Input Params  : None
# Return Value  : -
#
# This function does checking for duplicate entries with same option.
#-----------------------------------------------------------------------------
sub check_for_duplicate_entries {
    my ($input_name, $input_ref) = @_;
    my @input = @$input_ref;

    if (@input) {
        my %seen = ();
        my @input_tmp = @input;
        @input = ();
        foreach my $elem (@input_tmp) {
            fail("ERROR: Duplicate $input_name $elem found in input.")
                if $seen{ $elem }++;
            push @input, $elem;
        }
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : doRestore()
# Calls         : mv command
# Called by     :
# Globals       :
# Input Params  : Name of the file to be backed up.
# Return Value  : -
#
# This function does restore the orignal file (<file_name>.original) of file
# which is passed to.
#-----------------------------------------------------------------------------
sub doRestore {
    my $fname = shift;
    my $original = $fname.".original";
    my $cmd;
    my $errorFile = undef;
    if (-e $fname) {
        if (-e $original) {
            $cmd = "mv $original $fname";
            (my $ret) = captureCmd($cmd, "Restoring orignal file of ".
                                   "$fname", $localnode, EXPECT_SUCCESS);
            if($ret != 0) {
                    print("WARNING: Could not restore orignal file of ".
                          "$fname.\n");
            }
        }
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : setup_lvm_command_path()
# Calls         : getDistroInfo() from EDPkgTools.pm
# Called by     :
# Globals       : SBIN
# Input Params  : None
# Return Value  : -
#
# This function does setting correct sbin directory location with respect to
# OS.
#-----------------------------------------------------------------------------
sub setup_lvm_command_path {
    if (($os_name eq "RHEL5")) {
        $SBIN = "/usr/sbin";
    } else {
        $SBIN = "/sbin";
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : check_sg_version_and_setup_node()
# Calls         : -
# Called by     : -
# Globals       : $sgeasyTool, @nodes
# Input Params  : None
# Return Value  : -
#
# This function does populating node list in @nodes and check for Same
# Serviceguard version installed on all nodes.
# It also check Serviceguard version 12.00 or later version must be installed
# on all nodes.
#-----------------------------------------------------------------------------
sub check_sg_version_and_setup_node {
    # If nodes are not specified add cluster nodes if cluster configured
    # otherwise localnode.
    if (!@nodes) {
        my $cmd = "$SGBIN/cmviewcl -f line -s config";
        my ($exit, @output) = captureCmd($cmd, "Checking cluster", $localnode);
        if ($exit != 0) {
            if (!$opt_locklun) {
                #no cluster. Add local node to nodes.
                $sgeasyTool->screen_and_log("No cluster and no -n option. ".
                                            "Using local node $localnode",
                                            "cmpreparestg:");
            }
            push(@nodes, $localnode);
        } else {
            #node:bermese|name=bermese
            foreach my $line (@output) {
                if ($line =~ /^node:(.+)\|name=/) {
                    my @parts = split('=',$line);
                    chomp $parts[1];
                    if (!$opt_locklun) {
                        $sgeasyTool->screen_and_log("Using node $parts[1] to ".
                                                    "configure storages",
                                                    "cmpreparestg:");
                    }
                    push(@nodes, $parts[1]);
                }
            }
        }
    }

    # All the nodes should be on the same SG version.
    my $sg_version;
    foreach my $node_name (@nodes) {
        my $cmd = "$SGBIN/cmversion 2> /dev/null";
        my ($exit, $version) = captureCmd($cmd, "Checking Serviceguard ".
                                          "version", $node_name);
        if ($exit != 0) {
            fail("ERROR: Unable to get Serviceguard version from $node_name.");
        }
        if (!$sg_version) {
            $sg_version = $version;
            my @sgversion = split /\./, $sg_version;
            if (scalar($sgversion[1]) < $SGLX12) {
                fail("ERROR: $node_name must have Serviceguard version ".
                     "A.$SGLX12.00 or higher");
            }
        }
        elsif ($version ne $sg_version) {
            fail("ERROR: All the nodes should have the same version of ".
                 "the serviceguard.");
        }
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : validate_length_of_device()
# Calls         : -
# Called by     : validate_input()
# Globals       : $localnode
# Input Params  : $device and $remNode
# Return Value  : -
#
# This function validate a given device to make sure device name must not have
# more than 39 char.
#-----------------------------------------------------------------------------
sub validate_length_of_device {
    my ($device, $remNode) = @_;
    if (!defined($remNode)) {
        $remNode = $localnode;
    }
    if (length($device) > $MAX_LEN_OF_DEV_SUPPORTED) {
        fail("ERROR: Serviceguard does not support device [$device] ".
             "as device on node [$remNode] contains more than ".
             "$MAX_LEN_OF_DEV_SUPPORTED char\n");
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : check_if_multipath()
# Calls         : - 
# Called by     : populate_pvs(), configureDeviceAsLocklun(), 
#                 validateDiskIsUnused(), is_disk_used_by_lvm(),
#                 is_disk_used_by_cluster(), validate_input(), getRawDisk()
# Globals       : -
# Input Params  : A device name and node name on which it need to be validated.
# Return Value  :  
#                 0 -> If device is not a Multipath device (May be a Raw disk)
#                 1- > If device is a Multipath device.
# This function accepts a mapper device and extracts only partition number of 
# device from device name then return the same.
#-----------------------------------------------------------------------------
sub check_if_multipath {
    my ($var, $nodeName) = @_;
    my $silent = 1;
    if ($var =~ m/^\/dev\/mapper/ || $var =~ m/^\/dev\/mpath/ ||
        $var =~ m/^\/dev\/dm-+/) {
        return 1;
    } else {
       my $cmd = "/usr/bin/readlink $var";
       if (!defined($nodeName)) {
           $nodeName = $localnode;
       }
       my ($exitcode,
       @output) = captureCmd($cmd, "Checking for multipath",
                             $nodeName, EXPECT_ANY,
                             0, $silent);
        if ($exitcode == 0 && "@output" =~ m/dm-/) {
            return 1;
        }
    }
    return 0;
}

#------------------------------------------------------------------------------
#
# Subroutine    : get_part_number_of_mapper_dev()
# Calls         : perl function [split] 
# Called by     : get_short_name_mapper_dev()
# Globals       : -
# Input Params  : A mapper device name with partition number.
# Return Value  : device partition number if device is not a 
#                 partitioned device then returns 0.
#
# This function accepts a mapper device and extracts only partition number of 
# device from device name then return the same.
#-----------------------------------------------------------------------------
sub get_part_number_of_mapper_dev {
    my ($var)        = @_;
    my @personal     = ();
    my $number       = undef;
    my @num          = ();
    my $last_element = undef;
    $last_element = $var;
    if ($os_name eq "RHEL7") {
        #On RH7
        # Device can be like /dev/mapper/mpathXY
        # X will be only alphabet and Y will be number 
        if ($last_element =~ m/[0-9]+$/) {
            @num=split(/mpath[a-zA-Z]+/, $last_element);
            return $num[$#num];
        }
    } else {
        #On SLES11
        # Device can be like /dev/mapper/mpathX_partY
        if ($last_element =~ m/_part[0-9]+$/) {
            @num=split(/_part/, $last_element);
            return $num[$#num];
        #On SLES12
        # Device can be like /dev/mapper/mpathX-partY
        } elsif ($last_element =~ m/-part[0-9]+$/) {
            @num=split(/-part/, $last_element);
            return $num[$#num];
        #On RH5 and RH6
        # Device can be like /dev/mapper/mpathXpY
        # or
        # Device can be like /dev/mpath/mpathXpY
        # X can be be alphabet or number  and 
        # Y will be only number 
        } elsif ($last_element =~ m/p[0-9]+$/) {
            @num=split(/p/, $last_element);
            return $num[$#num];
        } else {
            return 0;
        }
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : get_short_name_mapper_dev()
# Calls         : -
# Called by     : populate_pvs(), validateDiskIsUnused()
# Globals       : -
# Input Params  : device name and variable that enable/disable logging of
#                 command. If $silent is set to "1" then logging of command
#                 by calling function captureCmd() will be disabled and for
#                 "0" then logging of command will be enabled.
# Return Value  : short name of device (like "mpatha"/"mpath1"),
#                 device partition number and
#                 middle name (mpath/mapper).
#
# This function accepts full path of mapper device and returns short name
# mapper device, partition number (if device is partitioned) and middle
# name (it can be "mpath" or "mapper").
#-----------------------------------------------------------------------------
sub get_short_name_mapper_dev {
    my ($var, $silent)    = @_;
    my $temp_dev = undef;
    my $number   = undef;
    my $cmd      = undef;
    my $exitcode = 1;
    my @output   = ();
    my @dm_dev_name = ();
    my $middle_path_name = "mapper";
    my @middle_name = ();
    if (!defined($silent)) {
       $silent = 1;
    }
    if ($os_name eq "RHEL5") {
        $middle_path_name = "mpath";
    }
    if ($var =~ m/^\/dev\/mapper/ || $var =~ m/^\/dev\/mpath/ ) {
        @middle_name = split('\/', $var);
        $middle_path_name = $middle_name[2];
        $var =~ s/\/dev\/mpath\///;
        $var =~ s/\/dev\/mapper\///;
    } else {
        $cmd = "/usr/bin/readlink $var";
        ($exitcode,
         @output) = captureCmd($cmd, "Getting short name of ".
                               "mapper device", $localnode,
                               EXPECT_ANY, 0, $silent);
        if ($exitcode == 0) {
            $temp_dev = basename("@output");
            $temp_dev = trim($temp_dev);
            @output = ();
            $cmd = "$DM_DETAILS /dev/$temp_dev 2>/dev/null|".
                   "grep Name| awk '{print \$2}'";
            ($exitcode, @output) = captureCmd($cmd, "Getting short ".
                                              "name of device by ".
                                              "$DM_DETAILS", $localnode,
                                              EXPECT_ANY, 0,
                                              $silent);
            if ($exitcode == 0 && (scalar(@output) > 0)) {
                $var = "@output";
            } else {
                @output = ();
                $cmd = "$MPATH_LIST 2>/dev/null | grep $temp_dev";
                ($exitcode, @dm_dev_name) = captureCmd($cmd, "Getting short ".
                                                       "name of device by ".
                                                       "$MPATH_LIST",
                                                       $localnode,
                                                       EXPECT_ANY, 0,
                                                       $silent);
                if (($exitcode == 0) && (scalar(@dm_dev_name) > 0)) {
                    (@output) = split(' ', "@dm_dev_name");
                    $var = $output[0];
                }
            }
        }
    }
    $number = get_part_number_of_mapper_dev($var);
    if ($os_name eq "RHEL7") {
        # need to add extra logic for RH7 only
        $var =~ s/[0-9]+$//;
    } else {
        $var =~ s/-part[0-9]+$//;
        $var =~ s/_part[0-9]+$//;
        $var =~ s/p[0-9]+$//;
    }
    $var = trim($var);
    return ($var, $number, $middle_path_name);
}

#------------------------------------------------------------------------------
#
# Subroutine    : get_udev_name()
# Calls         : -
# Called by     : get_udev_info(), configureDeviceAsLocklun(), is_device_udev()
# Globals       : localnode
# Input Params  : device name and node name where we need to find udev name
#                 of device if exists.
# Return Value  : If found then returns udev name of device otherwise "NO_DEV".
#
# This function returns udev name if udev name mapping found on node passed
# to this function otherwise "NO_DEV" (means there is no udev mapping exists
# on node)
#-----------------------------------------------------------------------------
sub get_udev_name {
    my ($dev, $rem_node) = @_;
    my $silent = 1;
    my $cmd    = undef;
    my $udev   = "NO_DEV";
    my $exitcode = undef;
    my @output = ();
    my @dev_list = ();
    if (defined($preview) && ($preview == 1)) {
        return $udev;
    }
    if ($os_name eq "RHEL5") {
        $cmd = "/usr/bin/udevinfo -q 'symlink' -n $dev";
    } else {
        $cmd = "/sbin/udevadm info -q 'symlink' -n $dev";
    }
    if ($rem_node eq $localnode) {
        ($exitcode,
        @output) = captureCmd($cmd, "Finding udev symlink of device [$dev]",
                              $rem_node, EXPECT_ANY, 0, $silent);
    } else {
       ($exitcode,
        @output) = captureCmd($cmd, "Finding udev symlink of device [$dev]",
                              $rem_node, EXPECT_SUCCESS);
    }
    if (($exitcode == 0) && (scalar(@output)>0)) {
        @dev_list = split(" ", "@output");
        if ((scalar(@dev_list)>1) && (!(($dev_list[0] =~ m/^block\//)||
            ($dev_list[0] =~ m/^disk\/by-*/)))) {
            if (!((($dev_list[0] =~ m/mapper\/mpath+/))||
                (($dev_list[0] =~ m/mpath\/mpath+/)) ||
                (($dev_list[0] =~ m/mapper\/[[:alnum:]]+/)) ||
                (($dev_list[0] =~ m/mpath\/[[:alnum:]]+/)))) {
                $udev = "/dev/$dev_list[0]";
            }
        }
    }
    return $udev;
}

#------------------------------------------------------------------------------
#
# Subroutine    : is_device_udev()
# Calls         : -
# Called by     : get_udev_info()
# Globals       : localnode
# Input Params  : device name
# Return Value  : 1. If device itself is udev type then returns "1"
#                 2. If device is not udev type but there is udev device found
#                    for same device then returns "2"
#                 3. If device is not udev type as well as no udev device
#                    found for same device then returns "0"
#
#-----------------------------------------------------------------------------
sub is_device_udev {
    my ($dev) = @_;
    my $silent = 1;
    my $cmd    = "/usr/bin/readlink $dev";
    my ($exitcode,
        @output) = captureCmd($cmd, "Checking if device [$dev] ".
                              "is a udev device", $localnode,
                              EXPECT_ANY, 0, $silent);
    if ($exitcode == 0) {
        if (get_udev_name($dev, $localnode) ne "NO_DEV") {
            return 1;
        }
    } else {
        if (get_udev_name($dev, $localnode) ne "NO_DEV") {
            return 2;
        }
    }
    return 0;
}

#------------------------------------------------------------------------------
#
# Subroutine    : getRawDisk()
# Calls         : get_udev_info(), check_if_multipath(),
#                 get_short_name_mapper_dev(),
#                 get_multipath_device_name().
# Called by     : get_storage_type()
# Globals       : localnode
# Input Params  : Name of the device file.
# Return Value  : Full path of Raw disk without partition number.
#
# Logic         :
#                            --Y--> DM -> Mapper dev > Get Raw disk > Remove num
#           --Y-> Multipath-| 
#          |                 ---N-------> Raw Disk -> Remove number
# is_udev -|
#          |
#          |                  --Y-->Mapper device > Get Raw disk > Remove number 
#           --N-> Multipath -|
#                             -----N----> Raw Disk -> Remove number
#
# This function excepts all type of supported device file like Udev (it might
# be created on dm device or raw device or raw device partition)/ Mapper (it
# can be whole disk or partition)/ Raw device (it can be whole disk or
# partition) and returns full path of under lying Raw disk name without
# partition number.

#-----------------------------------------------------------------------------
sub getRawDisk {
    my ($device_name) = @_;
    my $raw_disk_name = undef;
    my @rawDiskListInMpathDev = ();
    my ($is_dev_udev, $udev_name) = get_udev_info($device_name);
    my $mapper_dev = undef;
    my $dm_dev_name = undef;
    my $cmd = undef;
    my $exitcode = 1;
    my @output = ();
    if ($is_dev_udev == 0) {
        if (check_if_multipath($device_name) == 0) {
            $raw_disk_name = $device_name;
        } else {
            ($mapper_dev) = get_short_name_mapper_dev($device_name);
            get_raw_disk_list_of_mp($mapper_dev,
                                    \@rawDiskListInMpathDev,
                                    $device_name);
            if (scalar(@rawDiskListInMpathDev) >= 1) {
                $raw_disk_name = "/dev/$rawDiskListInMpathDev[0]";
            } else {
                fail("Unable to find any under lying Raw disk for device ".
                     "[$device_name] on local node [$localnode]");
            }
        }
    } else {
        if (check_if_multipath($device_name) == 0) {
            if ($is_dev_udev == 1) {
                # is_dev_udev => 1, that means device is a udev
                # device name. If device is a udev so it will
                # have soft link to Raw disk or Raw disk partition.
                $cmd    = "/usr/bin/readlink $device_name";
                ($exitcode,
                @output) = captureCmd($cmd, "Getting short name of ".
                                      "raw device", $localnode,
                                      EXPECT_SUCCESS);
                chomp(@output);
                $raw_disk_name = "/dev/@output";
            } else {
                # In rare senario below else block will be executed,
                # if we are here that means device inself is a Raw
                # disk or Raw disk partition.
                $raw_disk_name = $device_name;
            }
        } else {
            if ($is_dev_udev == 1) {
                # is_dev_udev => 1, that means device is a udev
                # device name. If device is a udev so it will
                # have soft link to dm device. 
                $cmd    = "/usr/bin/readlink $device_name";
            } else {
                # is_dev_udev => 2, that means mapper device has
                # a udev device name. We will be in this else block
                # only when a mapper device exists but does not have
                # soft link to dm device instead of that a udev
                # device name has soft link to dm device but
                # device name which has been supplied to command is a
                # mapper device instead of udev device name.
                $cmd    = "/usr/bin/readlink /dev/$udev_name";
            }
            ($exitcode,
            @output) = captureCmd($cmd, "Getting short name of ".
                                  "dm device", $localnode,
                                  EXPECT_SUCCESS);
            chomp(@output);
            $dm_dev_name = "/dev/@output";
            $mapper_dev = get_multipath_device_name($dm_dev_name,
                                                    $device_name,
                                                    $localnode);
            get_raw_disk_list_of_mp($mapper_dev,
                                    \@rawDiskListInMpathDev,
                                    $device_name);
            if (scalar(@rawDiskListInMpathDev) >= 1) {
                $raw_disk_name = "/dev/$rawDiskListInMpathDev[0]";
            } else {
                fail("Unable to find any under lying Raw disk ".
                     "for device [$device_name] on local node ".
                     "[$localnode]");
            }
        }
    }
    $raw_disk_name =~s/\d//g;
    return $raw_disk_name;
}

#------------------------------------------------------------------------------
#
# Subroutine    : get_storage_type()
# Calls         : getRawDisk(),
# Called by     : validate_input(),
# Globals       : localnode
# Input Params  : Name of device.
# Return Value  :
#                 iscsi     -> If device is iSCSI disk.
#                 non-iscsi -> If device is FC disk or unable to find storage 
#                              type
# This function expects device name and find storage type of device.(iSCSI/FC)
#-----------------------------------------------------------------------------
sub get_storage_type {
    my ($device_name) = @_;
    my $raw_disk = getRawDisk($device_name);
    my $cmd = "$SG_INQ -d $raw_disk";
    my ($exitcode, @output) = captureCmd($cmd, "Finding storage type of ".
                                         "$device_name", $localnode,
                                         EXPECT_SUCCESS);
    if ("@output" =~ m/iSCSI /) {
       return "iscsi";
    } else {
       return "non-iscsi";
    }
}

sub handle_pvs_file {
    open (SRC, $opt_pvs_file) or fail("ERROR: Failed to read PV file $opt_pvs_file");
    my @tot_pvs_tmp = <SRC>;
    close(SRC);
    foreach my $line (@tot_pvs_tmp) {
        chomp($line);
        $line =~ s/^\s+//;
        $line =~ s/\s+$//;
        next if $line =~ s/^#//g;
        push(@tot_pvs, $line);
    }
    if (!@tot_pvs) {
        fail("ERROR: The input file $opt_pvs_file does not contain any disks or all " .
             "the disk entries are commented. Populate the file " .
             "$opt_pvs_file with correct disk entries and re-try.");
    }
}

sub check_vg_display {
    my ($vgname, $node, $my_preview) = @_;
    my $cmd = "$SBIN/vgdisplay $vgname";
    my ($exit, @output) = captureCmd($cmd, "Validating $vgname existence",
                                     $node, EXPECT_ANY, $my_preview);
    if ($my_preview){
        return 1;
    }
    if ($exit == 0) {
        return 1; # VG exist on node #
    }
    return 0; # VG does not exist on node #
}

sub find_if_vg_exist {
    my ($vgname, $node, $my_preview) = @_;
    my $found_vg = check_vg_display($vgname, $node, $my_preview);
    return $found_vg;
}

sub find_if_vg_exist_anywhere {
    my ($vgname, $nodelist_ref) = @_;
    my @nodelist = @$nodelist_ref;
    foreach my $node (@nodelist) {
       my $found_vg = find_if_vg_exist($vgname, $node);
       return ($found_vg, $node) if $found_vg;
    }
    return 0;
}

sub find_if_dg_exist {
    my ($dgname, $node, $my_preview) = @_;
    my $cmd = "$SGBIN/cmcheckdg -d $dgname -n $node";
    my ($exit, @output) = captureCmd($cmd, "Validating $dgname existence",
                                     $node, EXPECT_ANY, $my_preview);
    if ($exit == 0) {
        return 1;
    }
    return 0;
}

sub find_if_dg_exist_anywhere {
    my ($dgname, $nodelist_ref, $my_preview) = @_;
    my @nodelist = @$nodelist_ref;
    foreach my $node (@nodelist) {
       my $found_dg = find_if_dg_exist($dgname, $node, $my_preview);
       return ($found_dg, $node) if $found_dg;
    }
    return 0;
}

sub check_if_vg_activated {
    my ($vgname, $node, $allow_failure, $my_preview) = @_;
    my $cmd = "$SBIN/vgdisplay $vgname";
    my $found_vg = 0;
    my $expect_success = EXPECT_SUCCESS;
    $expect_success = undef if $allow_failure;
    my ($exit, @output) = captureCmd($cmd, "Validating $vgname activation",
                                     $node, $expect_success, $my_preview);
    if ($exit == 0) {
        return 1; # VG activated on node #
    }
    return 0;     # VG does not exist on node #
}

sub check_if_vg_activated_elsewhere {
    my ($vgname, $my_preview) = @_;
    foreach my $node (@nodes) {
       if ($node eq $localnode) {
           next;
       }
       my $activated = check_if_vg_activated($vgname, $node, 1, $my_preview);
       return ($activated, $node) if $activated;
    }
    return 0;
}

#------------------------------------------------------------------------------
#
# Subroutine    : check_naming_scheme()
# Calls         : captureCmd()
# Called by     : validate_input()
# Globals       : $opt_dgname
# Input Params  : None
# Return Value  : -
#
# This function validate which namingscheme has been set on local node
# if it has been set other than "OS Native" 
#-----------------------------------------------------------------------------
sub check_naming_scheme {
    my $cmd = "command -v vxddladm 2>/dev/null";
    my @output = ();
    my $exitcode = 0;
    ($exitcode) = captureCmd($cmd, "Check for command [vxddladm] existence",
                             $localnode, EXPECT_ANY, undef, 1);
    if ($exitcode != 0) {
        fail("ERROR: Command [vxddladm] not found, fails to validate ".
             "naming scheme on local node.");
    } else {
        $cmd="/usr/sbin/vxddladm get namingscheme 2>/dev/null";
        ($exitcode, @output) = captureCmd($cmd, "Checking if naming ".
                                          "scheme is \"OS Native\"",
                                          $localnode, EXPECT_SUCCESS);
        if ("@output" !~ m/OS Native/) {
            fail("ERROR: Naming scheme is not set as \"OS Native\" on local ".
                 "node, other naming scheme is not supported for VxVM disk ".
                 "group. Use command [/usr/sbin/vxddladm set namingscheme=osn]".
                 " to set Naming scheme as \"OS Native\"");
        }
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : validate_input()
# Calls         : handle_pvs_file(), fail(), check_for_duplicate_entries()
#                 find_if_vg_exist_anywhere(), find_if_dg_exist_anywhere(),
#                 find_if_vg_exist(), find_if_dg_exist()
# Called by     : main()
# Globals       : @nodes, $opt_dgname, $opt_locklun, $opt_vgname, @opt_pvs
#                 $opt_pvs_file, @tot_pvs, @opt_pvs, $localnode, $opt_lv_counts
#                 $opt_lvname, $opt_mountpoint.
# Input Params  : None
# Return Value  : -
#
# This function validate all command line's arguments and options.
#-----------------------------------------------------------------------------
sub validate_input {
    my $using_multipath = 0;
    # we don't support more than $MAX_NUMBER_OF_NODE_SUPPORTED_BY_SG nodes
    if (@nodes > $MAX_NUMBER_OF_NODE_SUPPORTED_BY_SG){
        fail("ERROR: More than $MAX_NUMBER_OF_NODE_SUPPORTED_BY_SG ".
             "nodes are specified.");
    }

    #Ensure local node is included in provided nodes
    if (!grep {$_ =~ /$localnode/} @nodes) {
        pod2usage("Local node should be included for $0\n");
    }

    if ($preview && $opt_locklun) {
        pod2usage("Preview option [-t] and $locklun_opt option ".
                  "cannot be specified together");
    }
    # Atleast vgname or dgname is required otherwise one pvname
    # with "-d|--locklun" option is required with command
    # But as "-d|--locklun" option is not for external use
    # It should not be appeared on ERROR message.
    if ((!$opt_vgname && !$opt_dgname && !$opt_locklun)) {
        pod2usage("Volume group or disk group name ".
                  "is required.");
    } 
    else {
        # vgname, dgname and locklun opts are mutually exclusive
        # and should not be specified together.
        if ($opt_vgname && $opt_dgname) {
            pod2usage("Volume group and disk group names cannot ".
                      "be specified together");
        }
        elsif ($opt_vgname && $opt_locklun)
        {
            pod2usage("Unsupported -d option cannot be specified with ".
                      "Volume group name");
        }
        elsif ($opt_dgname && $opt_locklun)
        {
            pod2usage("Unsupported -d option cannot be specified with ".
                      "Disk group name");
        }
        elsif ((scalar(@opt_pvs) > 1) && $opt_locklun)
        {
            pod2usage("Unsupported -d option.");
        }
    }

    if ($opt_vgname) {
        $opt_vgname =~ s:/dev::;
    }


    if ((@opt_pvs) && defined($opt_pvs_file)) {
        pod2usage("Either -p or -P can be specified but not both");
    }

    if (@opt_pvs) {
        @tot_pvs = @opt_pvs;
    }

    if (defined($opt_pvs_file)) {
        handle_pvs_file();
    }

    # Check the duplicate entries for input disks
    check_for_duplicate_entries("disk", \@tot_pvs);

    # Extract any lv and fs opstrings
    foreach my $opstring (@opt_opstring) {
        my @name_value = split('=', $opstring);
        if ($name_value[0] eq "vg_opts") {
            if (!@tot_pvs) {
                pod2usage("LVM volume group options can only be specified ".
                          "when creating a new volume group.");
            }
            if ($vg_opts ne "") {
                pod2usage("LVM volume group options -o vg_opts cannot be " .
                          "specified multiple times.");
            }
            if (!$opt_vgname || $opt_dgname) {
                pod2usage("LVM volume group options -o vg_opts can only be ".
                          "specified, when creating a new volume group.");
            }
            $vg_opts=$name_value[1];
        }
        elsif ($name_value[0] eq "dg_opts") {
            if (!@tot_pvs) {
                pod2usage("VxVM disk group options can only be specified, " .
                          "when creating a new disk group.");
            }
            if ($dg_opts ne "") {
                pod2usage("VxVM disk group options -o dg_opts cannot be " .
                          "specified multiple times.");
            }
            if ($opt_vgname || !$opt_dgname) {
                pod2usage("VxVM disk group options -o dg_opts can only be ".
                          "specified, when creating a new disk group.");
            }
            $dg_opts=$name_value[1];
            my $length_array = @name_value;
            for (my $cnt = 2; $cnt < $length_array; $cnt++) {
                 $dg_opts="$dg_opts=$name_value[$cnt]";
            }
        }
        elsif ($name_value[0] eq "lv_opts") {
            if ((!defined($opt_lvname)) and (!defined($opt_lv_counts))) {
                pod2usage("Logical volume name or count required");
            }
            if ($lv_opts ne "") {
                pod2usage("-o lv_opts specified multiple times.");
            }
            $lv_opts=$name_value[1];
            my $length_array = @name_value;
            for (my $cnt = 2; $cnt < $length_array; $cnt++) {
                 $lv_opts="$lv_opts=$name_value[$cnt]";
            }
        }
        elsif ($name_value[0] eq "fs_opts") {
            if (!defined($opt_mountpoint)) {
                pod2usage("-m option must be specified to use fs_opts.");
            }
            if ($fs_opts ne "") {
                pod2usage("-o fs_opts specified multiple times.");
            }
            $fs_opts=$name_value[1];
        }
        else {
            pod2usage("Invalid opstring type specified.");
        }
    }
    if ($opt_dgname && $opt_lvname && !$lv_opts) {
        pod2usage("Logical volume options for VxVM logical volumes are not"
                  . " specified. Specify for example -o lv_opts=10G"
                  . " layout=mirror nmirror=2");
    }

    # Check on local system to find which naming_scheme has been set.
    # Fails the command if VxVM opts has been selected and naming_scheme
    # is not set as "OS Native". 
    if ($opt_dgname) {
        check_naming_scheme();
    }
    # Validate specified vg or dg doesn't exist if pv_paths are specified
    if (@tot_pvs) {
        if ($opt_vgname) {
            my ($found_vg, $node) = find_if_vg_exist_anywhere($opt_vgname,
                                                              \@nodes);
            if ($found_vg) {
                fail("ERROR: LVM volume group \"$opt_vgname\" already exists ".
                     "on $node.Choose a different name.");
            }
        }
        if ($opt_dgname) {
            my ($found_dg, $node) = find_if_dg_exist_anywhere($opt_dgname,
                                                              \@nodes);
            if ($found_dg) {
                fail("ERROR: VxVM disk group \"$opt_dgname\" already exists ".
                     "on $node. Choose a different name.");
            }
        }
        # Validate pv to make sure for VxVM disk group, pv must be
        # OS defined raw device and if device's storage type is iSCSI then
        # pv can not be used with VxVM and Cluster Lock Lun. This validation
        # can provide better error message.
        foreach my $pv (@tot_pvs) {
            fail("ERROR: Specified $pv does not exist or is not a block device.")
                 unless (-b $pv);
            # VG may contain PVs with Name more than 39 char.
            # Only in case of locklun we need to validate to
            # To make sure more than 39 char device name must
            # not be used as Lock lun.
            # DG if Non-user friendly device will be used then
            # it will be failed in below code so we need not to 
            # check for device name with more than 40 char.
            if ($opt_locklun) {
                validate_length_of_device($pv);
            }

            my $storage_type = get_storage_type($pv);
            if (($storage_type eq "iscsi") && ($opt_locklun)) {
                fail("$pv is a iSCSI disk and can not be used as Cluster ".
                     "Lock Lun")
            } elsif (($storage_type eq "iscsi") && ($opt_dgname)) {
                fail("Use of a iSCSI disk [$pv] is not supported for ".
                     "VxVM disk group");
            }
            if (check_if_multipath($pv)) {
                $using_multipath = 1;
            }
            if (($opt_dgname) && ($using_multipath == 1)) {
                fail("Encountered multipath device file $pv ".
                     "Multipath device file is not supported for ".
                     "VxVM disk group. Use raw device with only ".
                     "OS Native naming scheme");
            }
        }
    } else {
        # No pvs_paths. Ensure that specified vg or dg exists on at least
        # local node.
        if ($opt_vgname) {
            my $found_vg = find_if_vg_exist($opt_vgname, $localnode);
            if (!$found_vg) {
                fail("ERROR: Specified LVM volume group \"$opt_vgname\" doesn't ".
                     "exist on local node");
            }
        }
        if ($opt_dgname) {
            my $found_dg = find_if_dg_exist($opt_dgname, $localnode);
            if (!$found_dg) {
                fail("ERROR: Specified VxVM disk group \"$opt_dgname\" doesn't ".
                     "exist on local node");
            }
        }
        if (!$opt_vgname && !$opt_dgname) {
            fail("ERROR: Either \"$opt_vgname\" or \"$opt_dgname\" doesn't ".
                 "exist on local node");
        }
    }
    if (defined($opt_lvname) && defined($opt_lv_counts)) {
       pod2usage("Either logical volume name or logical volume counts ".
                 "can be specified");
    }
    my $lvopt = 0;
    if (defined($opt_lvname) || defined($opt_lv_counts)) {
        $lvopt = 1;
    }
    if (defined($opt_mountpoint) && ($lvopt == 0)) {
       pod2usage("Logical volume name/counts required");
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : createBackup()
# Calls         : cp command
# Called by     :
# Globals       :
# Input Params  : Name of the file to be backed up.
# Return Value  : -
#
# This function makes a backup of the file passed to <file_name>.original.
#-----------------------------------------------------------------------------
sub createBackup {
    my $fname = shift;
    my $original = $fname.".original";
    my $cmd;
    my $errorFile = undef;
    if (-e $fname) {
        if((-e $original)) {
            $cmd = "unlink $original";
            (my $ret) = captureCmd($cmd, "Cleanup of old backup file ".
                                  "$original", $localnode,
                                  EXPECT_SUCCESS, undef, 1);
        }
        $cmd = "cp $fname $original";
        (my $ret) = captureCmd($cmd, "Creating backup of file of $fname ".
                               "as orignal file $original",
                               $localnode, EXPECT_SUCCESS);
        if ($ret != 0) {
            print STDERR ("WARNING: Could not create a backup of $fname.\n");
        } else {
            $cmd = "cat $original |".
                   "sed -e 's/hosttags = 1/hosttags = 0/' > $fname";
            (my $ret) = captureCmd($cmd, "Disabling activation filter ".
                                   "by setting hosttags value as 0 in ".
                                   "$fname", $localnode,
                                   EXPECT_SUCCESS);
            if ($ret != 0) {
                fail("ERROR: Fails to disable activation filter by setting ".
                     "hosttags value as 0 in $fname\n");
            }
        }
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : transform_mpath_to_mapper()
# Calls         : -
# Called by     : populate_pvs()
# Globals       : -
# Input Params  : device name with middle name as "mpath"
# Return Value  : Converted device name
#
# This function will be used only in RHEL5.X version, where device name can
# like "/dev/mpath/devXXX" or "/dev/mapper/devXXX". This function will replace
# device name with word "mpath" into "mapper".
# Note: In RHEL5.X, In some scenario device might not be exist on directory
# "/dev/mpath" but exist on directory "/dev/mapper".
#-----------------------------------------------------------------------------
sub transform_mpath_to_mapper {
    my ($dev_name) = @_;
    my @arr = split('\/', $$dev_name);
    $arr[2] = "mapper";
    $$dev_name = join("/", @arr);
}

#------------------------------------------------------------------------------
#
# Subroutine    : trim()
# Calls         : -
# Called by     : get_short_name_mapper_dev(), get_part_number_of_raw_dev(),
#                 get_dm_device().
# Globals       : -
# Input Params  : device name
# Return Value  : Trimmed device name
#
# This function removes all trailing and leading whilte spaces.
#-----------------------------------------------------------------------------
sub trim {
    my ($str) = @_;
    $str =~ s/^\s+//;
    $str =~ s/\s+$//;
    return $str;
}

sub get_part_number_of_raw_dev {
    my ($var) = @_;
    $var = trim($var);
    if ($var =~ m/[0-9]+/) {
        $var =~s/[^0-9]//g;
        return $var;
    } else {
        return 0;
    }
}

sub get_short_name_raw_dev {
    my ($var)  = @_;
    my $number = -1;
    my $silent = 1;
    my $temp   = undef;
    my $cmd    = "/usr/bin/readlink $var";
    my ($exitcode,
        @output) = captureCmd($cmd, "Getting short name of ".
                              "raw device", $localnode,
                              EXPECT_ANY, 0, $silent);

    if ($exitcode == 0) {
        $temp = basename("@output");
    } else {
        $temp = basename($var);
    }
    $number = get_part_number_of_raw_dev($temp);
    $temp =~ s/[0-9]+$//;
    $temp = trim($temp);
    return ($temp, $number);
}

sub get_wwn_of_device_mapper {
    my ($dm_dev) = @_;
    my $number   = undef;
    my $cmd      = undef;
    my $exitcode = 1;
    my @num      = ();
    my @values   = ();
    my @output   = ();
    my $temp     = undef;
    chomp($dm_dev);
    $cmd = "ls -l /dev/disk/by-id 2>/dev/null |grep ".
           "-e \"dm-uuid-mpath-[[:alnum:]]\\+\\ -> ..\/\\+\"|grep ".
           "$dm_dev\$";
    ($exitcode, @output) = captureCmd($cmd, "Finding SSN of [$dm_dev]",
                                      $localnode, EXPECT_ANY, $preview);
    if (($exitcode == 0) && (scalar(@output) > 0)) {
        $temp = "@output";
        @output = ();
        @output = split(' ', $temp);
        $temp = $output[$#output-2];
        @output = ();
        @output = split('-mpath-', $temp);
        if (scalar(@output) > 0) {
            chomp($output[$#output]);
            return $output[$#output];
        }
    }
    return;
}

sub get_wwn_of_device_rw {
    my ($raw_dev) = @_;
    my $number   = undef;
    my $cmd      = undef;
    my $exitcode = undef;
    my @num      = ();
    my @output   = ();
    my @values   = ();
    $cmd = "ls -l /dev/disk/by-id 2>/dev/null |grep ".
           "-e \"scsi-[[:alnum:]]\\+\\ -> ..\/\\+\" |grep ".
           "$raw_dev\$";
    ($exitcode, @output) = captureCmd($cmd, "Finding SSN of [$raw_dev]",
                                      $localnode, EXPECT_ANY, $preview);
    if (($exitcode == 0) && (scalar(@output) > 0)) {
        @values = split(' ', "@output");
        @output = ();
        @output = split('-', $values[$#values-2]);
        if (scalar(@output) > 0) {
            chomp($output[$#output]);
            return $output[$#output];
        }
    }
    return;
}

sub getRemNodeMpathByMpath {
    my ($device_name, $remote_node, $ssn,
        $middle_path_name, $silent_opt) = @_;
    my $mapper_basepath = undef;
    my $temp = "NO_DEV";
    my $cmd = "$MPATH_LIST 2>/dev/null";
    my @values = ();
    my $exitcode = 0;
    my @output   = ();
    if (!defined($middle_path_name)) {
        $mapper_basepath = "/dev/mapper/";
    } else {
        $mapper_basepath="/dev/$middle_path_name/";
    }
    ($exitcode, @output) = captureCmd($cmd, "Validating device ".
                                      "[$device_name] existence ".
                                      "of same SSN",
                                      $remote_node, EXPECT_ANY,
                                      $preview, $silent_opt);
    if (($exitcode == 0) && (scalar(@output)> 0)) {
        foreach (@output) {
            if ((/$ssn/) && (/mpath||[[:alnum:]]+/)) {
                @values = split(' ', $_);
                $temp = "$mapper_basepath"."$values[0]";
                chomp($temp);
                next;
            }
        }
    }
    return $temp;
}

sub getRemNodeDiskByRawDsk {
    my ($device_name, $remote_node,
        $ssn, $called_by_func) = @_;
    my $disk_basepath = "/dev/";
    my $temp          = "NO_DEV";
    my @values        = ();
    my $exitcode      = 0;
    my @output        = ();
    my $cmd = "ls -l /dev/disk/by-id 2>/dev/null|".
              "grep -e \"scsi-[[:alnum:]]\\+\\ -> ..\/\\+\"";
    ($exitcode,
    @output) = captureCmd($cmd, "Validating device [$device_name] ".
                          "existence of same SSN", $remote_node,
                          EXPECT_ANY, $preview);
    if (($exitcode == 0) && (scalar(@output)> 0)) {
        foreach (@output) {
            if (/$ssn/) {
                @values = split(' ', $_);
                $temp = basename($values[$#values]);
                chomp($temp);
                $temp = "$disk_basepath$temp";
                next;
            }
        }
    }
    if (!$preview && !defined($called_by_func)) {
        if ($temp eq "NO_DEV") {
            fail("ERROR: Unable to find Raw device with ".
                 "same SSN [$ssn] on node [$remote_node].");
       }
    }
    return $temp;
}

sub get_dm_device {
    my ($mapper_dev_ref, $device_name) = @_;
    my $mapper_dev = $$mapper_dev_ref;
    my $dm_device = undef;
    my $cmd = "$MPATH_LIST $mapper_dev 2>/dev/null |".
              "grep dm-";
    $mapper_dev = trim($mapper_dev);
    my ($exitcode,
        @dm_dev_name) = captureCmd($cmd,
                                   "Finding Multipath DM device ".
                                   "name of device ".
                                   "$device_name",
                                   $localnode,
                                   EXPECT_ANY, $preview);
    my $dm_element_number = 0;
    if (!$preview) {
        if (($exitcode == 0) && (scalar(@dm_dev_name) > 0)) {
            my (@output) = split(' ', "@dm_dev_name");
            if ($output[0] =~ m/mpath+/) {
                $dm_element_number = 2;
            } else {
                $dm_element_number = 1;
            }
            $dm_device = $output[$dm_element_number];
        }
    }
    return $dm_device;
}

sub get_multipath_device_name {
    my ($dm_dev, $device_name, $remNode) = @_;
    my $mapper_device = "NO_DEV";
    $dm_dev = trim($dm_dev);
    $dm_dev = basename($dm_dev);
    my $cmd = "$MPATH_LIST 2>/dev/null|grep $dm_dev";
    my ($exitcode,
        @dm_dev_name) = captureCmd($cmd,
                                   "Finding Multipath mapper device ".
                                   "name of device ".
                                   "$device_name",
                                   $remNode,
                                   EXPECT_ANY, $preview);
    if (!$preview) {
        if (($exitcode == 0) && (scalar(@dm_dev_name) > 0)) {
            my (@output) = split(' ', "@dm_dev_name");
            $mapper_device = $output[0];
        }
    }
    return $mapper_device;
}

sub dev_name_with_partition {
    my ($dev, $num) = @_;
    if (check_if_multipath($dev) == 1) {
        if (($os_name eq "RHEL7")) {
            $dev = $dev.$num;
        } elsif (($os_name eq "RHEL5") || ($os_name eq "RHEL6")) {
            $dev = $dev."p".$num;
        } elsif ($os_name eq "SLES11") {
            $dev = $dev."_part".$num;
        } elsif ($os_name eq "SLES12") {
            $dev = $dev."-part".$num;
        }
    } else {
        $dev = $dev.$num;
    }
    return $dev;
}

#------------------------------------------------------------------------------
#
# Subroutine    : is_disk_used_by_cluster
# Calls         : getRemNodeMpathByMpath(), dev_name_with_partition(),
# Called by     : validateDiskIsUnused(),
# Globals       : localnode
# Input Params  : full path of device name, short name of device name,
#                 which might be "undef" in case of raw device, partition
#                 number, middle name of multipath device if configured
#                 otherwise "undef", udev devicea if configured otherwise
#                 "undef" and SSN of device and silent_opt to execute
#                 this function silently and nothing will be logged
#                 in log file if it is "undef" then it will be logged
#                 into log file.
# Return Value  :
#                0 -> Not used by any cluster as lock lun.
#                1 -> Whole device used by any cluster as lock lun.
#                2 -> Either partion of device or whole device
#                     used by any cluster as lock lun.
#-----------------------------------------------------------------------------
sub is_disk_used_by_cluster {
    my ($device_name, $dev_name, $dev_num,
        $middle_name, $udev_name,
        $ssn_of_dev, $silent_opt) = @_;
    my $result = 0;
    my $exitcode = 0;
    my @output = ();
    my $mapper_dev_name = undef;
    my $cmd = "$SGBIN/cmviewcl -v -f line -s config 2>/dev/null|".
              "grep -i \"cluster_lock\" | grep -i \"lun=\" |".
              "grep -i $localnode";
    if (!defined($middle_name)) {
        $mapper_dev_name = getRemNodeMpathByMpath($device_name,
                                                  $localnode,
                                                  $ssn_of_dev,
                                                  $middle_name);
        if ($mapper_dev_name ne "NO_DEV") {
            if ($dev_num > 0) {
                $mapper_dev_name = dev_name_with_partition($mapper_dev_name,
                                                           $dev_num);
            }
            $mapper_dev_name = basename($mapper_dev_name);
        }
    }
    if (check_if_multipath($device_name)) {
       $dev_name = "/dev/$middle_name/$dev_name";
    } else {
       $dev_name = "/dev/$dev_name";
    }
    if ($dev_num > 0) {
        $dev_name = dev_name_with_partition($dev_name, $dev_num);
    }
    $dev_name = basename($dev_name);
    ($exitcode, @output) = captureCmd($cmd, "Checking disk, ".
                                      "if used by cluster as ".
                                      "locklun", $localnode,
                                      EXPECT_ANY, $preview,
                                      $silent_opt);
    my $cmd_op = "@output";
    $cmd_op = trim($cmd_op);
    $_ = $cmd_op;
    if ($exitcode == 0) {
        # If PV name found in cmviewcl command output as lock lun
        # then return 1 otherwise 0 that means PV does not use by
        # existing cluster as lock lun.
        # PV can be in three form:
        # 1. Raw device name
        # 2. Udev Name created on raw device
        # 3. Mapper device created on raw device (& multipthd is running)
        if ($dev_num > 0) {
            if ((/$dev_name$/) ||
                (defined($udev_name) && (/$udev_name$/)) ||
                (defined($mapper_dev_name) &&
                ($mapper_dev_name ne "NO_DEV") &&
                (/$mapper_dev_name$/))) {
                 return 1;
            }
        } else {
            if ((/$dev_name/) ||
                (defined($udev_name) && (/$udev_name/)) ||
                (defined($mapper_dev_name) &&
                ($mapper_dev_name ne "NO_DEV") &&
                (/$mapper_dev_name/))) {
                return 2;
            }
        }
    }
    return $result;
}

#------------------------------------------------------------------------------
#
# Subroutine    : is_disk_used_by_lvm
# Calls         : getRemNodeMpathByMpath(), dev_name_with_partition()
# Called by     : validateDiskIsUnused(),
# Globals       : localnode
# Input Params  : full path of device name, short name of device name,
#                 which might be "undef" in case of raw device, partition
#                 number, middle name of multipath device if configured
#                 otherwise "undef", udev devicea if configured otherwise
#                 "undef" and SSN of device and silent_opt to execute
#                 this function silently and nothing will be logged
#                 in log file if it is "undef" then it will be logged
#                 into log file.
# Return Value  :
#                0 -> Not used by LVM.
#                1 -> Whole device used by LVM.
#                2 -> Either partion of device or whole device
#                     used by LVM.
#-----------------------------------------------------------------------------
sub is_disk_used_by_lvm {
    my ($device_name, $dev_name, $dev_num,
        $middle_name, $udev_name,
        $ssn_of_dev, $silent_opt) = @_;
    my $called_by_func = 1;
    my $exitcode = 0;
    my @output = ();
    my $mapper_dev_name = undef;
    my $cmd = "$SBIN/pvs -o pv_name";
    if (!defined($middle_name)) {
        $mapper_dev_name = getRemNodeMpathByMpath($device_name,
                                                  $localnode,
                                                  $ssn_of_dev,
                                                  $middle_name);
        if ($mapper_dev_name ne "NO_DEV") {
            if ($dev_num > 0) {
                $mapper_dev_name = dev_name_with_partition($mapper_dev_name,
                                                           $dev_num);
            }
            $mapper_dev_name = basename($mapper_dev_name);
        }
    }
    if (check_if_multipath($device_name)) {
       $dev_name = "/dev/$middle_name/$dev_name";
    } else {
       $dev_name = "/dev/$dev_name";
    }
    if ($dev_num > 0) {
        $dev_name = dev_name_with_partition($dev_name, $dev_num);
    }
    $dev_name = basename($dev_name);
    ($exitcode, @output) = captureCmd($cmd, "Checking disk, ".
                                      "if used by LVM",
                                      $localnode, EXPECT_ANY,
                                      $preview, $silent_opt);
    if (($exitcode == 0)&& (scalar(@output) > 1)) {
        # If PV name found in array populated by pvs command,
        # then return 1 otherwise 0 that means PV does not use by
        # LVM.
        # PV can be in three form:
        # 1. Raw device name
        # 2. Udev Name created on raw device
        # 3. Mapper device created on raw device (& multipthd is running)
        foreach (@output) {
            $_ = trim($_);
            if ($dev_num > 0) {
                if ((/$dev_name$/) ||
                    (defined($udev_name) && (/$udev_name$/)) ||
                    (defined($mapper_dev_name) &&
                    ($mapper_dev_name ne "NO_DEV") &&
                    (/$mapper_dev_name$/))) {
                    return 1;
                }
            } else {
                if ((/$dev_name/) ||
                    (defined($udev_name) && (/$udev_name/)) ||
                    (defined($mapper_dev_name) &&
                    ($mapper_dev_name ne "NO_DEV") &&
                    (/$mapper_dev_name/))) {
                    return 2;
                }
            }
        }
    }
    return 0;
}

#------------------------------------------------------------------------------
#
# Subroutine    : is_disk_used_by_vxvm
# Calls         : -
# Called by     : validateDiskIsUnused(),
# Globals       : localnode
# Input Params  : short name of raw device name without partition number
#                 and silent_opt to execute this function silently and
#                 nothing will be logged in log file if it is "undef"
#                 then it will be logged into log file.
# Return Value  :
#                0 -> Not used by VxVM.
#                1 -> Whole device used by VxVM.
#-----------------------------------------------------------------------------
sub is_disk_used_by_vxvm {
    my ($dev, $silent_opt) = @_;
    my $result = 0;
    my $silent = 1;
    my $cmd = "/usr/sbin/vxdisk -o alldgs list 2>/dev/null";
    my ($exitcode, @output) = captureCmd($cmd, "Checking disk, ".
                                         "if used by VxVM",
                                         $localnode, EXPECT_ANY,
                                         $preview, $silent_opt);
    if ($exitcode == 0) {
        shift(@output);
        foreach my $line (@output) {
            # Capture the fields we're interested in
            if ($line =~ /^(\S+)\s+\S+\s+\S+\s+\(?([^)\s]+)\)?/) {
                # Found disk $1 on local node " .
                # for disk group $2\n");
                if (($1 eq $dev) && ($2 ne "-")) {
                    $result = 1;
                    next;
                }
            }
        }
    }
    return $result;
}

sub get_raw_disk_list_of_mp {
    my ($mapper_dev, $rawDiskListInMpathDev_ref,
        $device_name, $remNode, $silent) = @_;
    my $exitcode = 0;
    my $index = 0;
    my $pv_name = undef;
    if (!defined($remNode)) {
        $remNode = $localnode;
    }
    my $cmd = "$MPATH_LIST $mapper_dev 2>/dev/null ".
              "|grep sd |grep active |".
              "sed 's/|//g'";
    ($exitcode,
    @$rawDiskListInMpathDev_ref) = captureCmd($cmd,
                                             "Finding all active ".
                                             "raw device paths ".
                                             "of [$device_name]",
                                             $remNode, EXPECT_ANY,
                                             undef, $silent);
    if ($exitcode == 0 && scalar(@$rawDiskListInMpathDev_ref)>0) {
        foreach $pv_name (@$rawDiskListInMpathDev_ref) {
            my (@output) = split(' ', $pv_name);
            $$rawDiskListInMpathDev_ref[$index] = $output[2];
            $index++;
        }
    }
    return;
}

#------------------------------------------------------------------------------
#
# Subroutine    : validateDiskIsUnused()
# Calls         : is_disk_used_by_vxvm(), is_disk_used_by_lvm(),
#                 is_disk_used_by_cluster(), check_if_multipath(),
#                 get_short_name_mapper_dev(), get_raw_disk_list_of_mp(),
#                 get_short_name_raw_dev(),
# Called by     : validateDiskIsUnused(),
# Globals       : localnode
# Input Params  : Full path of device name (it can be mapper or raw device)
#                 message string depends on command line option as error
#                 message content and SSN of device name.
# Return Value  :
#                 NONE
# This function do all validation to make sure device is not used by LVM, VxVM
# or as lock lun device by existing cluster and fails script of device is in
# use.
#-----------------------------------------------------------------------------
sub validateDiskIsUnused {
    my ($device_name, $udev_name,
        $msgStringsDepOnOptions,
        $ssn_of_dev) = @_;
    my $mp_dev_mid_name = undef;
    my @rawDiskListInMpathDev = undef;
    my $dev_part_num = undef;
    my $mapper_or_raw_dev = undef;
    my $device_number = 0;
    if (check_if_multipath($device_name)) {
       ($mapper_or_raw_dev, $dev_part_num,
        $mp_dev_mid_name) = get_short_name_mapper_dev($device_name);
       get_raw_disk_list_of_mp($mapper_or_raw_dev, \@rawDiskListInMpathDev,
                               $device_name, undef, 1);
       if (scalar(@rawDiskListInMpathDev) > 0 ) {
            foreach my $disk_to_check (@rawDiskListInMpathDev) {
                if (is_disk_used_by_vxvm($disk_to_check)) {
                    fail("ERROR: Whole device is used by VxVM ".
                         "on [$localnode], fails to use device ".
                         "[$device_name] for creating ".
                         "$msgStringsDepOnOptions");
                }
            }
        }
    } else {
        ($mapper_or_raw_dev,
         $dev_part_num) = get_short_name_raw_dev($device_name);
        if (is_disk_used_by_vxvm($mapper_or_raw_dev)) {
                fail("ERROR: Whole device is used by VxVM on ".
                     "[$localnode], fails to use device ".
                     "[$device_name] for creating ".
                     "$msgStringsDepOnOptions");
        }
    }
    if (!(defined($opt_dgname) && $opt_dgname)) {
        $device_number = $dev_part_num;
    }
    my $isUsedByLvm = is_disk_used_by_lvm($device_name, $mapper_or_raw_dev,
                                          $device_number, $mp_dev_mid_name,
                                          $udev_name, $ssn_of_dev);
    if ($isUsedByLvm == 1) {
        fail("ERROR: Whole device is used by LVM on [$localnode], ".
             "fails to use device [$device_name] for creating ".
             "$msgStringsDepOnOptions");
    } elsif ($isUsedByLvm == 2) {
        fail("ERROR: Either partition of device or whole device ".
             "is used by LVM on [$localnode], fails to use device ".
             "[$device_name] for creating $msgStringsDepOnOptions");
    }
    my $isUsedByCluster = is_disk_used_by_cluster($device_name,
                                                  $mapper_or_raw_dev,
                                                  $device_number,
                                                  $mp_dev_mid_name,
                                                  $udev_name,
                                                  $ssn_of_dev);
    if ($isUsedByCluster == 1) {
        fail("ERROR: Whole device is used by existing cluster as locklun ".
             "on [$localnode], fails to use device [$device_name] ".
             "for creating $msgStringsDepOnOptions");
    } elsif ($isUsedByCluster == 2) {
        fail("ERROR: Either partition of device or whole device ".
             "is used by existing cluster as locklun on [$localnode], ".
             "fails to use device [$device_name] for creating ".
             "$msgStringsDepOnOptions");
    }
    return;
}

sub get_wwn_of_device {
    my ($device_name, $dm_device, $mapper_dev, $rawDiskListInMpathDev_ref) = @_;
    my $ssn_of_dev = undef;
    my $pv_name = undef;
    if (defined($mapper_dev)) {
        $ssn_of_dev = get_wwn_of_device_mapper($dm_device);
    }
    if (!defined($ssn_of_dev)) {
        if (!$opt_locklun) {
            print("INFO: Using all possible active paths of device [$device_name] ".
                  "to find SSN on $localnode\n");
        }
        get_raw_disk_list_of_mp($mapper_dev, $rawDiskListInMpathDev_ref,
                                $device_name);
        if ((scalar(@$rawDiskListInMpathDev_ref)> 0)) {
            foreach $pv_name (@$rawDiskListInMpathDev_ref) {
                $ssn_of_dev = get_wwn_of_device_rw($pv_name);
                if (defined($ssn_of_dev)) {
                    last;
                }
            }
        }
    }
    return $ssn_of_dev;
}

sub getRemNodeDevice {
    my ($device_name, $nodename,
        $ssn_of_dev, $mp_dev_mid_name) = @_;
    my $called_by_func = 1;
    my $temp = getRemNodeMpathByMpath($device_name,
                                      $nodename,
                                      $ssn_of_dev,
                                      $mp_dev_mid_name);
    if (!$preview) {
        if ($temp eq "NO_DEV") {
            if (!$opt_locklun) {
                  print "WARNING: Mapper device does not exist on ".
                        "node [$nodename], checking existence of ".
                        "Raw device with same SSN [$ssn_of_dev].\n";
            }
            $temp = getRemNodeDiskByRawDsk($device_name,
                                           $nodename,
                                           $ssn_of_dev,
                                           $called_by_func);
            if ($temp eq "NO_DEV") {
                fail("ERROR: Unable to find either Mapper or ".
                     "Raw device with same SSN [$ssn_of_dev] on ".
                     "node [$nodename].");
            }

        }
    }
    return $temp;
}

sub get_udev_info {
    my ($device_name) = @_;
    my $udev_name = undef;
    my $is_dev_udev = is_device_udev($device_name);
    # If is_dev_udev -> 1 means device name passed to
    # command itself a udev device.
    # If is_dev_udev -> 2 means device name passed to
    # command is not udev device but it also has udev
    # device mapping.
    if ($is_dev_udev  == 1) {
        $udev_name = basename($device_name);
    }
    if ($is_dev_udev == 2) {
        $udev_name = get_udev_name($device_name, $localnode);
        $udev_name = basename($udev_name);
    }
    return ($is_dev_udev, $udev_name);
}

sub check_device_existence_by_ls {
    my ($device_name, $nodename) = @_;
    my ($exitcode) = captureCmd("ls -l $device_name",
                                "Validating device [$device_name] ".
                                "existence", $nodename,
                                EXPECT_ANY);
    return $exitcode;
}

sub check_device_existence {
    my ($dev_part_num, $device_name, $nodename) = @_;
    my $dev_with_partition = undef;
    my $exitcode = 1;
    if ($dev_part_num > 0) {
        $dev_with_partition =
        dev_name_with_partition($device_name,
                                $dev_part_num);
       ($exitcode) = check_device_existence_by_ls($dev_with_partition,
                                                  $nodename);
    } else {
       ($exitcode) = check_device_existence_by_ls($device_name,
                                                  $nodename);
    }
    return $exitcode;
}

sub display_warning_msg_in_RHEL5 {
    my ($dev_part_num, $device_name,
        $ssn_of_dev, $nodename) = @_;
    my $dev_with_partition = undef;
    if ($dev_part_num > 0) {
        $dev_with_partition =
        dev_name_with_partition($device_name,
                                $dev_part_num);
        print STDERR "WARNING: mpath device ".
                     "[$dev_with_partition] does not exist ".
                     "on node [$nodename], ".
                     "using mapper device with same SSN ".
                     "[$ssn_of_dev].\n";
    } else {
        print STDERR "WARNING: mpath device [$device_name] ".
                     "does not exist on node [$nodename], ".
                     "using mapper device with same SSN ".
                     "[$ssn_of_dev].\n";
    }
}

#------------------------------------------------------------------------------
#
# Subroutine    : configureDeviceAsLocklun()
# Calls         : get_udev_info(), check_if_multipath(),
#                 validate_length_of_device(),
# Called by     : populate_pvs(),
# Globals       : localnode
# Input Params  : device_name, using_multipath(variable which contains 0/1),
#                 nodeDevList_ref(reference of device list on all node),
#                 nodeListWithOutLocNode_ref(reference of node list except
#                 local node name)
# Return Value  : NONE
# This function updates list of device on all nodes as per Cluster lock
# lun creation rules and prints on console output as string
# "device:<device name>|node:<node name>" for each nodes.
# Note: This fucntions output will be used by cmdeploycl command.
#-----------------------------------------------------------------------------
sub configureDeviceAsLocklun {
    my ($device_name, $using_multipath,
        $nodeListWithOutLocNode_ref, $nodeDevList_ref) = @_;
    my @output_msg  = ();
    my $exitcode    = 1;
    my %nodeDevList = %$nodeDevList_ref;
    my $nodeDevice  = undef;
    my @nodeListWithOutLocNode = @$nodeListWithOutLocNode_ref;
    my @nodeListWithLocNode = ();
    my ($is_dev_udev, $udev_name) = get_udev_info($device_name);
    foreach my $nodename (@nodeListWithOutLocNode) {
        my $using_multipath_remNode = check_if_multipath($nodeDevList{$nodename},
                                                         $nodename);
        if ($is_dev_udev == 1) {
            $nodeDevList{$nodename} = get_udev_name($nodeDevList{$nodename},
                                                    $nodename);
            if ($nodeDevList{$nodename} eq "NO_DEV") {
                fail("ERROR: No equivalent UDEV device found on ".
                     "node [$nodename], fails to use device ".
                     "as Locklun");
            }
        } else {
            if (($using_multipath_remNode && !$using_multipath) ||
                (!$using_multipath_remNode && $using_multipath)) {
                fail("ERROR: No equivalent Mapper device found on ".
                     "node [$nodename], fails to use device ".
                     "as Locklun");
           }
        }
        validate_length_of_device($nodeDevList{$nodename}, $nodename);
    }
    push(@nodeListWithLocNode, $localnode);
    foreach my $nodename (@nodeListWithLocNode) {
        if ($nodename eq $localnode) {
            $nodeDevice = $device_name;
        } else {
            $nodeDevice = $nodeDevList{$nodename};
        }
        ($exitcode,
        @output_msg) = captureCmd("$SGBIN/cmdisklock check $nodeDevice",
                                  "Verifying device [$nodeDevice], ".
                                  "if it can be used as locklun", $nodename,
                                  EXPECT_ANY);
        if ($exitcode) {
            if (("@output_msg" =~ m/is inaccessible/) ||
                ("@output_msg" =~ m/used by LVM/)) {
                # Do nothing as device is accessiable as already validated
                # and also not used by LVM/VxVM or existing cluster.
            } elsif ("@output_msg" =~ m/Could not open cluster lock lun/) {
                fail("ERROR: Command [cmdisklock check $nodeDevice] fails ".
                     "on node [$nodename] as device might be inaccessiable.".
                     "\nUnable to use device [$device_name] for creating Locklun");
            } else {
                fail("ERROR: Command [cmdisklock check $nodeDevice] fails ".
                     "on node [$nodename] with unknown reason.\nUnable to use".
                     " device [$device_name] for creating Locklun");
            }
        }
    }
    captureCmd("$SGBIN/cmdisklock reset $device_name",
               "Reseting device [$device_name] to use ".
               "for creating Locklun", $localnode, EXPECT_SUCCESS);
    print("device:$device_name|node:$localnode\n");
    foreach my $nodename (@nodeListWithOutLocNode) {
        print("device:$nodeDevList{$nodename}|node:$nodename\n");
    }
}

sub getMsgStringsDepOnOptions {
    my $msgStringsDepOnOptions = undef;
    if ($opt_locklun) {
        $msgStringsDepOnOptions = "Locklun";
    } else {
        $msgStringsDepOnOptions = "any volume group or disk group";
    }
    return $msgStringsDepOnOptions;
}

sub get_list_of_all_possible_path {
    my ($phyPathListOfDisk_ref, $device_name) = @_;
    my $exitcode = 0;
    my $index = 0;
    my $pv_name = undef;
    my $cmd = "/usr/sbin/vxdmpadm getsubpaths dmpnodename=".
              "$device_name 2>/dev/null |grep -v NAME|".
              "grep -v \"====\"";
    ($exitcode,
    @$phyPathListOfDisk_ref) = captureCmd($cmd,
                                          "Finding all active ".
                                          "raw device path ".
                                          "of [$device_name]",
                                          $localnode, EXPECT_ANY,
                                          undef);
    if ($exitcode == 0 && scalar(@$phyPathListOfDisk_ref)>0) {
        foreach $pv_name (@$phyPathListOfDisk_ref) {
            my (@output) = split(' ', $pv_name);
            $$phyPathListOfDisk_ref[$index] = $output[0];
            $index++;
        }
    }
    return;
}

#------------------------------------------------------------------------------
#
# Subroutine    : populate_pvs().
# Calls         : getMsgStringsDepOnOptions(), get_udev_info(),
#                 is_disk_used_by_cluster(), check_if_multipath(),
#                 check_if_multipath, get_short_name_mapper_dev(),
#                 get_wwn_of_device(), get_raw_disk_list_of_mp(),
#                 get_short_name_raw_dev(), validateDiskIsUnused(),
#                 get_wwn_of_device_rw(), get_udev_name(),
#                 getRemNodeDiskByRawDsk(), getRemNodeDevice(),
#                 display_warning_msg_in_RHEL5(), transform_mpath_to_mapper(),
#                 dev_name_with_partition(), configureDeviceAsLocklun()
# Called by     : validateDiskIsUnused(),
# Globals       : localnode
# Input Params  : list of device name (which has been supplied with command)
# Return Value  : NONE
# This function will traverse a list of device and ensures that device or device
# partition must exist on all nodes. It will also do validation to ensure a
# device must not be in use by LVM/VxVM/Cluster lock lun.
# With lock lun option it will also configure device as a Cluster lock lun.
#-----------------------------------------------------------------------------
sub populate_pvs {
    my ($pvlist_ref) = @_;
    my @pvlist = @$pvlist_ref;
    my $msgStringsDepOnOptions = getMsgStringsDepOnOptions();
    foreach my $device_name (@pvlist) {
        my $exitcode = 1;
        my $using_multipath = 0;
        my $nodename      = undef;
        my $dev_file      = undef;
        my $raw_dev       = undef;
        my $pv_name       = undef;
        my $dev_part_num  = undef;
        my $dm_dev        = undef;
        my $udev_name     = undef;
        my $ssn_of_dev    = undef;
        my $mapper_dev    = undef;
        my $dm_device     = undef;
        my $cmd           = undef;
        my $shortMapperDevNm = undef;
        my %nodeDevList   = ();
        my @rawDiskListInMpathDev = ();
        my @phyPathListOfDisk = ();
        my @nodeListWithOutLocNode = ();
        my $rawDiskInMpathDev = undef;
        my $mp_dev_mid_name = undef;
        my $dev_with_partition = undef;
        my $is_dev_udev = 0;
        @nodeListWithOutLocNode = grep {$_ ne $localnode} @nodes;
        ($is_dev_udev, $udev_name) = get_udev_info($device_name);
        if (check_if_multipath($device_name)) {
            $using_multipath = 1;
            ($mapper_dev, $dev_part_num,
            $mp_dev_mid_name) = get_short_name_mapper_dev($device_name);
            $dm_device = get_dm_device(\$mapper_dev, $device_name);
            if (!$preview) {
                $ssn_of_dev = get_wwn_of_device($device_name, $dm_device,
                                                $mapper_dev,
                                                \@rawDiskListInMpathDev);
                if (!defined($ssn_of_dev)) {
                    fail("ERROR: Unable to find SSN of device [$device_name] ".
                         "on local node [$localnode]");
                } else {
                    if (!$opt_locklun) {
                        print("INFO: Found SSN [$ssn_of_dev] of device ".
                              "[$device_name] on local node [$localnode]\n");
                    }
                    if (scalar(@rawDiskListInMpathDev) == 0) {
                        get_raw_disk_list_of_mp($mapper_dev,
                                                \@rawDiskListInMpathDev,
                                                $device_name);
                    }
                    validateDiskIsUnused($device_name, $udev_name,
                                         $msgStringsDepOnOptions, $ssn_of_dev);
                }
            }
            foreach $nodename (@nodeListWithOutLocNode) {
                $nodeDevList{$nodename} = getRemNodeDevice($device_name,
                                                           $nodename,
                                                           $ssn_of_dev,
                                                           $mp_dev_mid_name);
                my $tempDev = "NO_DEV";
                if ($is_dev_udev == 1) {
                    $tempDev = get_udev_name($nodeDevList{$nodename},
                                             $nodename);
                    if ($tempDev ne "NO_DEV") {
                        $nodeDevList{$nodename} = $tempDev;
                    }
                }
                if (($is_dev_udev == 0) or ($tempDev eq "NO_DEV")) {
                    if (check_if_multipath($nodeDevList{$nodename}, $nodename)) {
                        $shortMapperDevNm = get_multipath_device_name($nodeDevList{$nodename},
                                                                      $device_name,
                                                                      $nodename);
                        get_raw_disk_list_of_mp($shortMapperDevNm,
                                                \@rawDiskListInMpathDev,
                                                $nodeDevList{$nodename},
                                                $nodename, 1);
                        if (scalar(@rawDiskListInMpathDev) >= 1) {
                            foreach my $rawDevName (@rawDiskListInMpathDev) {
                                $rawDiskInMpathDev = "/dev/".$rawDevName;
                                ($exitcode) = captureCmd("ls -l $rawDiskInMpathDev",
                                                         "Validating device ".
                                                         "[$rawDiskInMpathDev] ".
                                                         "existence", $nodename,
                                                         EXPECT_ANY, 0, 1);
                                if ($exitcode == 0) {
                                    next;
                                }
                            }
                        } else {
                            fail("Unable to find any Raw device on remote node [$nodename]".
                                 "with same SSN of device [$device_name] on local node ".
                                 "[$localnode]");
                        }
                    }
                    my $tempDev = "NO_DEV";
                    if ($is_dev_udev == 1) {
                        $tempDev = get_udev_name($rawDiskInMpathDev,
                                                 $nodename);
                        if ($tempDev ne "NO_DEV") {
                            $nodeDevList{$nodename} = $tempDev;
                        }
                    }
                }
            }
        } else {
            ($raw_dev, $dev_part_num) = get_short_name_raw_dev($device_name);
            if (!$preview) {
                $ssn_of_dev = get_wwn_of_device_rw($raw_dev);
                if (!defined($ssn_of_dev) && $opt_dgname) {
                    if (!$opt_locklun) {
                        print("INFO: Using all possible active paths of ".
                              "device [$raw_dev] to find SSN on $localnode\n");
                    }
                    get_list_of_all_possible_path(\@phyPathListOfDisk,
                                                  $raw_dev);
                    if (scalar(@phyPathListOfDisk) >= 1) {
                        foreach my $rawDevName (@phyPathListOfDisk) {
                            if ($rawDevName !~ m/$raw_dev/) {
                                $ssn_of_dev = get_wwn_of_device_rw($rawDevName);
                                if (defined($ssn_of_dev)) {
                                    next;
                                }
                            }
                        }
                    }
                }
                if (!defined($ssn_of_dev)) {
                    if (!$opt_dgname) {
                        fail("ERROR: Unable to find SSN of device [$device_name] ".
                             "on local node [$localnode],\n device might have multiple ".
                             "paths and it is recommended to use only Mapper device");
                    } else {
                        fail("ERROR: Unable to find SSN of device [$device_name] ".
                             "on local node [$localnode]");
                    }
                } else {
                    if (!$opt_locklun) {
                        print("INFO: Found SSN [$ssn_of_dev] of device ".
                              "[$device_name] on local node [$localnode]\n");
                    }
                    validateDiskIsUnused($device_name, $udev_name,
                                         $msgStringsDepOnOptions,
                                         $ssn_of_dev);
                }
            }
            foreach $nodename (@nodeListWithOutLocNode) {
                $nodeDevList{$nodename} = getRemNodeDiskByRawDsk($device_name,
                                                                 $nodename,
                                                                 $ssn_of_dev);
                my $tempDev = "NO_DEV";
                if ($is_dev_udev == 1) {
                    $tempDev = get_udev_name($nodeDevList{$nodename},
                                             $nodename);
                    if ($tempDev ne "NO_DEV") {
                        $nodeDevList{$nodename} = $tempDev;
                    }
                }
                if (($is_dev_udev == 0) or ($tempDev eq "NO_DEV")) {
                    if (check_if_multipath($nodeDevList{$nodename}, $nodename)) {
                        $shortMapperDevNm = get_multipath_device_name($nodeDevList{$nodename},
                                                                      $device_name,
                                                                      $nodename);
                        if ($os_name eq "RHEL5") {
                            $nodeDevList{$nodename} = "/dev/mpath/$shortMapperDevNm";
                        } else {
                            $nodeDevList{$nodename} = "/dev/mapper/$shortMapperDevNm";
                        }
                        get_raw_disk_list_of_mp($shortMapperDevNm,
                                                \@rawDiskListInMpathDev,
                                                $nodeDevList{$nodename},
                                                $nodename, 1);
                        if (scalar(@rawDiskListInMpathDev) >= 1) {
                            foreach my $rawDevName (@rawDiskListInMpathDev) {
                                $nodeDevList{$nodename} = "/dev/".$rawDevName;
                                ($exitcode) = captureCmd("ls -l $nodeDevList{$nodename}",
                                                         "Validating device ".
                                                         "[$nodeDevList{$nodename}] ".
                                                         "existence", $nodename,
                                                         EXPECT_ANY, 0, 1);
                                if ($exitcode == 0) {
                                    next;
                                }
                            }
                        } else {
                            fail("Unable to find any Raw device on remote node [$nodename]".
                                 "with same SSN of device [$device_name] on local node ".
                                 "[$localnode]");
                        }
                    }
                    my $tempDev = "NO_DEV";
                    if ($is_dev_udev == 1) {
                        $tempDev = get_udev_name($nodeDevList{$nodename},
                                                 $nodename);
                        if ($tempDev ne "NO_DEV") {
                            $nodeDevList{$nodename} = $tempDev;
                        }
                    }
                }
            }
        }
        if (!$preview) {
            foreach $nodename (@nodeListWithOutLocNode) {
                $exitcode = 0;
                $dev_with_partition = undef;
                if ($dev_part_num > 0) {
                    $dev_with_partition =
                    dev_name_with_partition($nodeDevList{$nodename},
                                            $dev_part_num);
                }
                ($exitcode) = check_device_existence($dev_part_num,
                                                     $nodeDevList{$nodename},
                                                     $nodename);
                if ($exitcode != 0) {
                    my $using_multipath_remNode  = check_if_multipath($nodeDevList{$nodename},
                                                                      $nodename);
                    if (($os_name eq "RHEL5") and
                        ($using_multipath_remNode)) {
                            if (!$opt_locklun) {
                                display_warning_msg_in_RHEL5($dev_part_num,
                                                             $nodeDevList{$nodename},
                                                             $ssn_of_dev, $nodename);
                            }
                            transform_mpath_to_mapper(\$nodeDevList{$nodename});
                    }
                    captureCmd("partprobe ".
                               "$nodeDevList{$nodename}",
                               "Running partprobe on ".
                               "[$nodeDevList{$nodename}]",
                               $nodename, EXPECT_SUCCESS);
                    if ($dev_part_num > 0) {
                        $nodeDevList{$nodename} =
                        dev_name_with_partition($nodeDevList{$nodename},
                                                $dev_part_num);
                        captureCmd("ls -l $nodeDevList{$nodename}",
                                   "Validating device [$nodeDevList{$nodename}] ".
                                   "existence", $nodename, EXPECT_SUCCESS);
                    }
                } else {
                    if ($dev_part_num > 0) {
                        $nodeDevList{$nodename} =
                        dev_name_with_partition($nodeDevList{$nodename},
                                                $dev_part_num);
                    }
                }
            }
        }
        if ($opt_locklun) {
            configureDeviceAsLocklun($device_name, $using_multipath,
                                     \@nodeListWithOutLocNode,
                                     \%nodeDevList);
        }
    }
}

sub find_dev_file_on_node {
    my ($pv_ref, $pv, $nodename) = @_;

    my @pvs = @$pv_ref;
    my %pv_hash;

    foreach $pv_ref (@pvs) {
        %pv_hash = %$pv_ref;
        if ($pv_hash{$localnode} eq $pv) {
            return $pv_hash{$nodename};
        }
    }
    return;
}

sub captureCmd {
    my ($cmd, $logstr, $node, $expect_success, $preview_opt, $silent_opt) = @_;

    my @output = ();

    if ($preview_opt) {
        if (!defined($silent_opt)) {
            if ($opt_locklun == 0) {
                if ($opt_verbose) {
                    $sgeasyTool->screen_and_log("== $logstr on $node:\n\t[$cmd]", undef, $preview);
                } else {
                    $sgeasyTool->screen_and_log("== $logstr on $node", undef, $preview);
                }
            }
        }
        return (0, ());
    } else {
        if (!defined($silent_opt)) {
            if ($opt_locklun == 0) {
                if ($opt_verbose) {
                    $sgeasyTool->screen_and_log("$logstr on $node:\n\t[$cmd]", "cmpreparestg:");
                } else {
                    $sgeasyTool->screen_and_log("$logstr on $node", "cmpreparestg:");
                }
            }
        }
        my $my_cmd;
        if ( $localnode eq $node ){
            $my_cmd = $cmd . "  2>&1 |";
        } else {
            $my_cmd = "$SGBIN/cmexec " . $node . " " . $cmd. "  2>&1 |";
        }

        open (CMD, $my_cmd) or fail("Cannot run command $my_cmd : $!");
        @output = <CMD>;
        close(CMD);
        my $exitCode = ($? >> 8);

        if ($exitCode !=0 && defined($expect_success) &&
            $expect_success eq EXPECT_SUCCESS) {
            fail("ERROR: ". $cmd. " failed with exit status of ". $exitCode.
                 " with errors: \n". join(" ", @output));
        }
        return ($exitCode, @output);
    }
}


sub pvcreate {
    foreach my $pv (@tot_pvs) {
        my $cmd = "$SBIN/pvcreate " . $pv;
        captureCmd($cmd, "Preparing disk for LVM volume group $opt_vgname",
                   $localnode, EXPECT_SUCCESS, $preview);
    }
}


sub vgcreate {

    #Create complete vgname
    my $vg = "/dev/$opt_vgname";
    my $pvList = "";

    foreach my $pv (@tot_pvs) {
        $pvList = $pvList . "$pv ";
    }

    my $cmd = "$SBIN/vgcreate $vg_opts $vg $pvList";
    captureCmd($cmd, "Creating LVM volume group", $localnode, EXPECT_SUCCESS,
               $preview);

    $cmd = "$SBIN/vgchange -a n $vg" ;
    captureCmd($cmd, "Deactivating LVM volume group", $localnode, EXPECT_SUCCESS,
               $preview);
}

sub get_lvname_list {
    my ($vgname, $node) = @_;
    my $cmd = "$SBIN/vgs -o lv_all --row $vgname";
    my @lvs = ();
    my $temp = undef;
    my ($exit, @output) = captureCmd($cmd, "vgs", $node, EXPECT_SUCCESS,
                                     $preview, 1);
    foreach (@output) {
        if (/Path/) {
            @lvs = split(" ", $_);
            if ("$lvs[0]" eq "Path") {
                shift(@lvs);
            }
        }
    }
    return @lvs;
}


sub find_new_in_list2 {
    my ($list1, $list2) = @_;
    my @new = ();
    foreach my $e2 (@$list2) {
        if (!(grep /$e2/, @$list1)) {
            push(@new, $e2);
        }
    }
    return @new;
}

sub make_check_filesystem_dir {
    my ($mountpoint, $nodelist_ref) = @_;
    my @nodelist = @$nodelist_ref;
    my $cmd = "/bin/mkdir -p $mountpoint";
    my $exit;
    my @output;
    foreach my $node (@nodelist) {
        ($exit, @output) = captureCmd($cmd,
                                      "Making file system directory",
                                      $node, EXPECT_SUCCESS, $preview);
        fail("ERROR: Fail to make file system directory ".
             "$mountpoint on $node") if ($exit != 0);
    }
    $cmd = "/bin/df $mountpoint";
    ($exit, @output) = captureCmd($cmd,
                                  "Checking parent file system directory",
                                  $localnode, EXPECT_SUCCESS, $preview);
    fail("ERROR: Failed to retrieve parent file system ".
         "directory $mountpoint") if ($exit != 0);
    my @dfoutput;
    my $localFS;
    if (!$preview) {
        chomp $output[0];
        @dfoutput = split(' ', $output[0]);
        $localFS = $dfoutput[0];
    }
    foreach my $node (@nodes) {
        if ($node eq $localnode) {
            next;
        }
        ($exit, @output) = captureCmd($cmd, "Checking parent file ".
                                      "system directory", $node,
                                      EXPECT_SUCCESS, $preview);
        fail("ERROR: Failed to make file system directory".
             "$mountpoint") if ($exit != 0);
        if (!$preview) {
            chomp $output[0];
            @dfoutput = split(' ', $output[0]);

            if ($localFS ne $dfoutput[0]) {
                print STDERR "WARNING: Parent filesystem for $mountpoint ".
                             "is not same on all nodes. $localnode has ".
                             "$localFS and $node has $dfoutput[0] \n "
            }
        }
    }
}

sub lvcreate_for_vg {
    my $exit;
    my @output;
    my $cmd;
    my $counts = 1;
    my @before;
    my @after;
    my @new;
    my $lvs;
    my $lvname;
    my $lvname_b;
    my $mountpoint;
    my $found_vg = find_if_vg_exist($opt_vgname, $localnode, $preview);
    fail("ERROR: LVM volume group \"$opt_vgname\" does not exist ".
         "on local node") if (!$found_vg);
    my $vg_activated_initial = check_if_vg_activated($opt_vgname, $localnode,
                                                     $preview);
    if (!$vg_activated_initial) {
        my ($activated2, $node) = check_if_vg_activated_elsewhere($opt_vgname);
        fail("ERROR: LVM volume group \"$opt_vgname\" is activated ".
             "on $node") if ($activated2);
        $cmd = "/$SBIN/vgchange -a y $opt_vgname";
        if (@tot_pvs) {
            ($exit, @output) = captureCmd($cmd, "Activating LVM volume group",
                                          $localnode, EXPECT_SUCCESS, $preview);
            if (!$preview) {
                $vg_activated = 1;
            }
        }
        else {
            ($exit, @output) = captureCmd($cmd, "Activating LVM volume group",
                                          $localnode, EXPECT_SUCCESS);
            $vg_activated = 1;
        }
        fail("ERROR: Failed to activate LVM volume ".
             "group \"$opt_vgname\"") if ($exit != 0);
    }
    if (defined($opt_lv_counts)) {
        $counts = $opt_lv_counts;
    }
    if (!defined($opt_lvname)) {
        $opt_lvname = "";
    }
    for (my $cnt = $counts; $cnt >= 1; $cnt--) {
        @before = get_lvname_list($opt_vgname, $localnode);
        my $found_lv = find_if_lvol_exist($opt_lvname, \@before);
        fail("ERROR: Logical volume \"$opt_lvname\" already exists. ".
             "Choose a different name.") if ($found_lv);
        $cmd = "$SBIN/lvcreate $lv_opts";
        if (length $opt_lvname > 0) {
            $cmd = $cmd . " -n $opt_lvname";
        }
        $cmd = $cmd . " $opt_vgname";
        ($exit, @output) = captureCmd($cmd, "Creating logical volume",
                                      $localnode, EXPECT_SUCCESS, $preview);
        fail("ERROR: Failed to create logical volume $opt_lvname") if ($exit != 0);
        @after = get_lvname_list($opt_vgname, $localnode);
        @new = find_new_in_list2(\@before, \@after);
        my $tmp_no = $counts - $cnt + 1;
        if ($preview) {
            if (!$opt_lvname) {
                $lvs = "lvol" . $tmp_no;
            }
            else {
                $lvs = $opt_lvname;
            }
        }
        else {
            $lvs = join(" ", @new);
        }
        $sgeasyTool->screen_and_log("Created logical volume(s): $lvs",
                                    "cmpreparestg:");
        if ($preview) {
            if (!$opt_lvname) {
                $lvname = "lvol" . $tmp_no;
            }
            else {
                $lvname = $opt_lvname;
            }
        }
        else {
            $lvname = $new[0];
        }
        if (defined($opt_mountpoint)) {
            $cmd = "$MKFS $fs_opts $lvname";
            ($exit, @output) = captureCmd($cmd, "Creating file system",
                                          $localnode, EXPECT_SUCCESS,
                                          $preview);
            fail("ERROR: Failed to create file system") if ($exit != 0);
            next unless $opt_mountpoint;
            if (defined($opt_lv_counts)) {
                $lvname_b = basename($lvname);
                $lvname_b  =~ m/^lvol(\S+)/ ;
                $mountpoint = $opt_mountpoint . $1;
            } else {
                $mountpoint = $opt_mountpoint;
            }
            make_check_filesystem_dir($mountpoint, \@allnodes);
        }
    }
    # Restore back to the orignal VG activation mode before this command
    if ($vg_activated) {
        $cmd = "/$SBIN/vgchange -a n $opt_vgname";
        captureCmd($cmd, "Deactivating LVM volume group", $localnode,
                   EXPECT_SUCCESS, $preview);
        $vg_activated = 0;
    }
}

sub update_vxvm_disk_from_tot_pvs {
    my ($block_pv_ref) = @_;
    my $pv;

    if ($$block_pv_ref =~ /\/dev/) {
        $pv = basename($$block_pv_ref);
        $$block_pv_ref = $pv;
    }
}

sub dgcreate {
    #Create VxVM disk group
    my $dg = $opt_dgname;
    my @pvlist;
    my $vxvm_disk = "";
    my $i = 0;
    my $cmd = "";

    foreach my $j (0..$#tot_pvs) {
        update_vxvm_disk_from_tot_pvs(\$tot_pvs[$j]);
        $cmd = "/usr/lib/vxvm/bin/vxdisksetup -i $tot_pvs[$j]";
        captureCmd($cmd, "Preparing the disk for VxVM disk group $dg",
                   $localnode, EXPECT_SUCCESS, $preview);
    }

    $cmd = "vxdg init $dg $tot_pvs[0] $dg_opts";
    captureCmd($cmd, "Creating VxVM disk group", $localnode,
               EXPECT_SUCCESS, $preview);

    foreach my $j (1..$#tot_pvs) {
        $cmd = "vxdg -g " . $dg . " adddisk " . $tot_pvs[$j];
        captureCmd($cmd, "Adding disk $tot_pvs[$j] to VxVM disk group $dg",
                   $localnode, EXPECT_SUCCESS, $preview);
    }

    $cmd = "vxdg deport $dg" ;
    captureCmd($cmd, "Deporting VxVM disk group $dg", $localnode,
               EXPECT_SUCCESS, $preview);
}

sub lvcreate_for_dg {
    my $exit;
    my @output;
    my $cmd;
    my $counts = 1;
    my @before;
    my @after;
    my @new;
    my $lvs;
    my $lvname;
    my $mountpoint;
    my $found_dg = find_if_dg_exist($opt_dgname, $localnode, $preview);
    fail("ERROR: VxVM disk group \"$opt_dgname\" does not exist ".
         "on local node") if (!$found_dg);
    my $dg_imported_local = check_if_dg_imported($opt_dgname, $localnode);
    if (!$dg_imported_local) {
        my ($imported_any_node,
            $node) = check_if_dg_imported_elsewhere($opt_dgname);
        fail("ERROR: VxVM disk group \"$opt_dgname\" is imported ".
             "on $node") if ($imported_any_node);
        $cmd = "/sbin/vxdg import $opt_dgname";
        if (@tot_pvs) {
            ($exit, @output) = captureCmd($cmd, "Importing VxVM disk group",
                                          $localnode, EXPECT_SUCCESS, $preview);
            if (!$preview) {
                $dg_imported = 1;
            }
        }
        else {
            ($exit, @output) = captureCmd($cmd, "Importing VxVM disk group",
                                          $localnode, EXPECT_SUCCESS);
            $dg_imported = 1;
        }
        fail("ERROR: Failed to import VxVM disk ".
             "group \"$opt_dgname\"") if ($exit != 0);
    }
    if (defined($opt_lv_counts)) {
        $counts = $opt_lv_counts;
    }
    if (!defined($opt_lvname)) {
        $opt_lvname = "lvol";
    }
    for (my $cnt = 1; $cnt <= $counts; $cnt++) {
        @before = get_lvname_list_from_dg($opt_dgname, $localnode);

        if (!defined($opt_lv_counts)){
            my $found_lv = find_if_lvol_exist($opt_lvname, \@before);
            fail("ERROR: Logical volume \"$opt_lvname\" already exists.".
                 " Choose a different name.") if ($found_lv);
        }

        $cmd = "/usr/sbin/vxassist -g $opt_dgname make";
        $cmd = $cmd . " $opt_lvname";
        if (defined($opt_lv_counts) && length $opt_lvname > 0) {
            $cmd = $cmd . $cnt;
        }
        $cmd = $cmd . " $lv_opts";
        ($exit, @output) = captureCmd($cmd, "Creating logical volume",
                                      $localnode, EXPECT_SUCCESS, $preview);
        fail("ERROR: Failed to create logical volume $opt_lvname") if ($exit != 0);
        @after = get_lvname_list_from_dg($opt_dgname, $localnode);
        @new = find_new_in_list2(\@before, \@after);
        if ($preview) {
            if ($opt_lvname eq "lvol") {
                $lvs = $opt_lvname . $cnt;
            }
            else {
                $lvs = $opt_lvname;
            }
        }
        else {
            $lvs = join(" ", @new);
        }
        $sgeasyTool->screen_and_log("Created logical volume(s): $lvs",
                                    "cmpreparestg:");
        if ($preview) {
            if ($opt_lvname eq "lvol") {
                $lvname = $opt_lvname . $cnt;
            }
            else {
                $lvname = $opt_lvname;
            }
        }
        else {
            $lvname = $new[0];
        }
        if (defined($opt_mountpoint)) {
            $cmd = "$MKFS $fs_opts /dev/vx/rdsk/$opt_dgname/$lvname";
            ($exit, @output) = captureCmd($cmd, "Creating file system",
                                          $localnode, EXPECT_SUCCESS,
                                          $preview);
            fail("ERROR: Failed to create file system") if ($exit != 0);
            next unless $opt_mountpoint;
            if (defined($opt_lv_counts)) {
                $lvname  =~ m/^lvol(\S+)/ ;
                $mountpoint = $opt_mountpoint . $1;
            } else {
                $mountpoint = $opt_mountpoint;
            }
            make_check_filesystem_dir($mountpoint, \@allnodes);
        }
    }
    # Restore back to the orignal state of DG by deporting disk group
    if ($dg_imported) {
        $cmd = "/sbin/vxdg deport $opt_dgname";
        captureCmd($cmd, "Deporting VxVM disk group", $localnode,
                   EXPECT_SUCCESS, $preview);
        $dg_imported = 0;
    }
}

sub check_if_dg_imported_elsewhere {
    my ($dgname) = @_;
    foreach my $node (@nodes) {
       if ($node eq $localnode) {
           next;
       }
       my $imported = check_if_dg_imported($dgname, $node, 1);
       return ($imported, $node) if $imported;
    }
    return 0;
}

sub check_if_dg_imported {
    my ($dgname, $node, $allow_failure) = @_;
    my $cmd = "/usr/sbin/vxdg list";
    my $found_imported_dg = 0;
    my $expect_success = EXPECT_SUCCESS;
    $expect_success = undef if $allow_failure;
    my ($exit, @output) = captureCmd($cmd, "Validating $dgname existence",
                                     $node, $expect_success, $preview);
    return 0 if $exit;
    foreach my $line (@output) {
        if ($line =~ /$dgname/) {
            return 1;
        }
    }
    return 0;

}

sub get_lvname_list_from_dg {
    my ($dgname, $node) = @_;
    my $line = "";
    my $cmd = "/sbin/vxprint -q -v -F 'vol:%name' -g $dgname";
    my @lvs = ();
    my ($exit, @output) = captureCmd($cmd, "vxprint", $node, EXPECT_ANY);
    # if the command returns 11 means, no volumes in disk group
    if ($exit == 11) {
        return @lvs;
    }
    foreach $line (@output) {
        chomp($line);
        push(@lvs, $line);
    }
    return @lvs;
}

sub find_if_lvol_exist {
    my ($lvolname, $lvols_ref) = @_;
    my @lvols = @$lvols_ref;

    foreach my $lvol (@lvols) {
        if ($lvol eq $lvolname) {
            return 1;
        }
    }

}

#------------------------------------------------------------------------------
#
# Subroutine    : main()
# Calls         : check_for_root_user(), set_local_node(), parse_args(),
#                 setup_logging(), check_for_duplicate_entries(),
#                 test_cmexec_on_all_nodes(),
#                 check_sg_version_and_setup_node(), setup_lvm_command_path(),
#                 validate_input(),
#
# Called by     : -
# Globals       : @nodes, $LVM_CONF, @tot_pvs, $opt_dgname,
#                 $opt_locklun, $opt_vgname, @opt_pvs
#                 $opt_pvs_file, @tot_pvs, @opt_pvs, $localnode,
# Input Params  : -
# Return Value  : -
#
# This Main function which performs all checks to commandline arguments/opts
# as well as Serviceguard version on all nodes, tests cmexec command on
# all nodes then finally create vg/dg/lv/locklun/mount points appropriatly.
#-----------------------------------------------------------------------------
sub main {
    # only root can run this
    check_for_root_user($<, $0);

    # Find local hostname and set in global variable "$localnode"
    set_local_node(\$localnode);

    parse_args();

    # Setting up log file to enable logging event into that file.
    setup_logging();

    # Check for the duplicate node names entered on the command line
    check_for_duplicate_entries("node name", \@nodes);

    # Test whether cmexec to remote node is successful. If not ask the
    # user to run cmpreparecl, which does all the required things for
    # the cmexec.
    test_cmexec_on_all_nodes(@nodes);

    # Populate @nodes array with existing cluster node otherwise only
    # local node, if they is no node name passed with command. (size of
    # @nodes is zero) and also check same version of Serviceguard is
    # installed on all nodes.
    check_sg_version_and_setup_node();

    # Set base path of all lvm related commands like (pvcreate,vgs etc..)
    # in global variables $SBIN also correct distro name as well as
    # distro version are also set in global variables $distroName,
    # $distroVer.
    setup_lvm_command_path();

    # Copy the node list to @allnodes. At any time if we want full
    # node list, we can get it from @allnodes.
    @allnodes = @nodes;

    # Verify all input parameters with command line are valid.
    validate_input();

    if (-e $MC_SCRIPT) {
        # Obtain the list of local and remote nodes as discovered by
        # Metro Cluster script.
        my $cmcheck_feature_cmd = "$SG_CONF_PATH/scripts/cmcheck_features mcrac";
        my ($localnodes_ref,
            $remotenodes_ref) = get_local_remote_nodes(\@tot_pvs,
                                                       \@allnodes);
        @localnodes = @$localnodes_ref;
        @remotenodes = @$remotenodes_ref;
        @nodes = @localnodes;
        if (scalar(@remotenodes)) {
            for my $node (@nodes) {
                my ($exitcode,
                    @output) = captureCmd($cmcheck_feature_cmd,
                                          "Checking if Metrocluster ".
                                          "is installed.", $node,
                                          EXPECT_SUCCESS);
                }
        }
    }

    if (!($preview || $opt_locklun || $opt_dgname)) {
        createBackup($LVM_CONF);
    }

    if (@tot_pvs) {
        populate_pvs(\@tot_pvs); # Find shared pvs
        if ($opt_vgname) {
           pvcreate();
           vgcreate();
        }
        if ($opt_dgname) {
           dgcreate();
        }
    }
    if (($opt_vgname) && (defined($opt_lvname) ||
                          defined($opt_lv_counts))) {
        lvcreate_for_vg();
    }

    if (($opt_dgname) && (defined($opt_lvname) ||
                          defined($opt_lv_counts))) {
        lvcreate_for_dg();
    }

    if (-e $MC_SCRIPT && $opt_vgname && scalar(@remotenodes)) {
        # Metro Cluster checking for LVM vgs here
        mc_verify_storage(\@allnodes, \@localnodes, \@remotenodes,
                          $opt_vgname, "LVM", \@tot_pvs);
    }

    if (-e $MC_SCRIPT && $opt_dgname && scalar(@remotenodes)) {
        # Metro Cluster checking for VxVM can be do here..
        mc_verify_storage(\@allnodes, \@localnodes, \@remotenodes,
                          $opt_dgname, "VxVM", \@tot_pvs);
    }

    if ($preview) {
        print STDOUT "Preview complete. To apply changes, run ".
                     "cmpreparestg without -t option.\n";
    }

    if (!($preview || $opt_locklun || $opt_dgname)) {
        doRestore($LVM_CONF);
    }

    exit(0);
}
