#!/usr/bin/perl -w
###########################################################################
# (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
# @(#) utilities for Serviceguard Cluster Analytics Daemon Admin Operations
# @(#) Product Name     : HP Serviceguard Cluster Analytics 
# @(#) Product Version  : A.12.10.00
# @(#) Patch Name       : 
#
#
# *** Note: This file MUST NOT be edited. *****
#
# Any changes made to it will be overwritten when you upgrade to the
# next release of HP Serviceguard Cluster Analytics.
#
# Changing this file may lead to unrecoverable
# Cluster Analytics daemon failure or may lead to issues related
# to administration of Cluster Analytics daemon and displaying 
# KPIs 
#
###########################################################################

use strict;
use Pod::Usage;
use File::Path;
use Errno qw(EAGAIN);
use Sys::Hostname;
use constant ERROR_WITHOUT_SYSLOG_ENTRY => -2;
use constant CMCAADMIN_CMD => "cmcaadmin";

=head1 NAME

cmcaadmin - command to perform admin operations for Cluster Analytics daemon. 

=cut

# agruments
# {start|stop|status|cleanup} 
#
# example:-    
# cmcaadmin start 
# cmcaadmin stop
# cmcaadmin status
# cmcaadmin cleanup


=head1 SYNOPSIS

cmcaadmin {start|stop|status|cleanup} 

=cut

my $oper = undef;
my $nodeName = undef;
my @node_unreachable = ();
my @file_out = undef;
my $errors = 0;
my $SGSBIN = undef;
my $SGCONF = undef;
my $SGRUN = undef;
my $FIND_CMD="/usr/bin/find";
my $cmca_start_file;
my $cmca_conf_file;
my $hostname = hostname();
$hostname =~ s/\..*//; 
my $cmd = "";
my @nodelist = ();
my @existing_cls_nodelist = ();
my @node_accessible = ();
my %cmexecOutput;
my %cmexecFiles;
my $daemon_retval=0;
my $distro = undef;
my $cl_exists_on_local_node;
my @cmviewcl_lineoutput = ();
my $cmca_start_file_name = ".cmanalyticsd_start";
my $cmca_conf_file_name = "cmanalytics.conf";

# This can be used to debug command and get more command line output.
# To enable debug msg set $debug to "1".
my $debug = 0;

#########################################

exit (1) if  (parse_args());
# check that root privileges are required to start or stop 
# Cluster Analytics daemon.
if ($oper eq "start" || $oper eq "stop"|| $oper eq "cleanup") {
   unless ($< == 0) {
       print STDERR "ERROR: Root priveleges are required to start ";
       print STDERR "or stop Cluster Analytics daemon";
       exit(1);
   }
}

my $sgconffile = "/etc/cmcluster.conf";
if (-f $sgconffile) {
    @file_out = qx(cat $sgconffile);
} else {
    print STDERR "ERROR: Unable to run cmcaadmin ".
                 "command on unsupported OS\n";
    exit(1);	                 
}

my @sgsbin_out = grep(/^SGSBIN/, @file_out);
if (@sgsbin_out) {
    my @split_sgsbin_name = split(/=/,$sgsbin_out[0]);
    chomp($split_sgsbin_name[1]);
    $SGSBIN = $split_sgsbin_name[1];
} else {
    print STDERR "ERROR: Unable to run cmcaadmin command. ".
                 "Failed to source SGSBIN\n";
    exit(1);	                 
}    

my @sgconf_out = grep(/^SGCONF/, @file_out);
if (@sgconf_out) {
    my @split_sgconf_name = split(/=/,$sgconf_out[0]);
    chomp($split_sgconf_name[1]);
    $SGCONF = $split_sgconf_name[1];
} else {
    print STDERR "ERROR: Unable to run cmcaadmin command. ".
                 "Failed to source SGCONF\n";
    exit(1);	                 
}    

my @sgrun_out = grep(/^SGRUN/, @file_out);
if (@sgrun_out) {
    my @split_sgrun_name = split(/=/,$sgrun_out[0]);
    chomp($split_sgrun_name[1]);
    $SGRUN = $split_sgrun_name[1];
} else {
    print STDERR "ERROR: Unable to run cmcaadmin command. ".
                 "Failed to source SGRUN\n";
    exit(1);	                 
}    

if ($oper eq "cleanup") {
    print STDOUT "@{[CMCAADMIN_CMD]}: Cleaning ".
                 "Cluster Analytics state configuration ".
                 "on node $hostname\n";
    my $cmd = "$SGSBIN/cmcaadmin status";
    my ($ret, @output)=runandCaptureCommand($cmd);
    if (!("@output" =~ m/not running/)) {
        my @out = split(/ /, "$output[$#output-2]");
        shift(@out);
        chomp($out[$#out]);
        if ("$hostname." eq "$out[$#out]") {
            print STDERR "ERROR: Fail to clean Cluster Analytics ".
                         "state configuration, @out\n";        
            exit(1);
        }
    } 
    $cmd = "$SGSBIN/cmviewcl" . " -v -f line -s config 2>&1|grep $hostname";
    my ($retval) = runandCaptureCommand($cmd);
    if ($retval == 0) {
        print STDERR "ERROR: Fail to clean Cluster ".
                     "Analytics state configuration ".
                     "as node is part of cluster.\n";        
        exit(1);
    }
    $cmd = "$FIND_CMD $SGRUN -name $cmca_start_file_name -delete";
    ($ret) = runandCaptureCommand($cmd);
    if ($ret) {
        print STDERR "ERROR: ERROR: Fail to clean Cluster Analytics ".
                     "state configuration on node $hostname.\n";
        exit(1);
    }	 	
    print STDOUT "@{[CMCAADMIN_CMD]}: Cleaned Cluster Analytics state ".
                 "configuration successfully on node $hostname\n";
    exit(0);
}

$cmca_start_file = "$SGRUN/$cmca_start_file_name";
$cmca_conf_file = "$SGCONF/$cmca_conf_file_name";
$cl_exists_on_local_node = check_if_cluster_exists();
if ($cl_exists_on_local_node == 0) {
    #we do not have cluster on local node, if the user
    #has provided -n option that is different than local node 
    # - if cluster exists on the remote node
    #   - if yes then error out saying local node needs to be part of the cluster
    #     else start Cluster Analytics daemon on remote node.
    if(defined($nodeName)) {
        if($nodeName ne $hostname) {
            @nodelist = ();
            push(@nodelist,$nodeName);
            my $retval = cmexecAndCapture("$SGSBIN/cmviewcl");
            if ($retval != 0) {
                if(grep(/Cannot view the cluster configuration/,
                        @{$cmexecOutput{$nodeName}}) ne 0) {
                    #Cluster does not exist.
                    print STDOUT "Cluster is not configured on $nodeName, ";
                    print STDOUT "Cluster Analytics daemon will be ";
                    if ($oper eq "start") {
                        print STDOUT "started";
                    }  
                    else {
                        print STDOUT "stopped";
                    }
                    print STDOUT " on the node $nodeName\n"; 
                    chk_and_start_analytics_daemon($nodeName);
                }
                elsif(grep(/Permission denied/,@{$cmexecOutput{$nodeName}}) ne 0) {
                    print STDERR "ERROR: Cluster Analytics daemon cannot be started on the ";
                    print STDERR "node $nodeName\n";
                    print STDERR "Either cluster is configured on the node or ";
                    print STDERR "sufficient\n";
                    print STDERR "persmissions are not given to $hostname or\n";
                    print STDERR "$SGCONF/cmclnodelist does not have $hostname ";
                    print STDERR "entries.\n";   
                    exit(1);
                }
                else {
                    print STDERR "ERROR: Unknown error from command\n";
                    exit(1);
                }
            }
            else {
                # Cluster exists on the node and local node is not part of the cluster.
                # fail the command.
                print STDERR "ERROR: Node $hostname is not part of the ";
                print STDERR "cluster on node $nodeName\n";
                print STDERR "cannot start Cluster Analytics daemon when local node ";
                print STDERR "$hostname is not part\n";
                print STDERR "of the cluster.\n"; 
                exit(1);
            }
        }    
        else {
            # User wants to start the Cluster Analytics daemon on local node.
            $daemon_retval = chk_and_start_analytics_daemon($hostname); 
        }
    }
    else {
        # Cluster does not exist on the local node and also node was not 
        # specified using -n option, so start the daemon based 
        # on $SGCONF/cmclnodelist.
        $daemon_retval = chk_and_start_analytics_daemon();
    }
} 
else {
    $cmd = "$SGSBIN/cmviewcl" . " -v -f line";
    my $retval = 0 ;
    ($retval,@cmviewcl_lineoutput) = runandCaptureCommand($cmd);
    if ($retval) {
        print STDERR "ERROR: Unable to retrieve cluster configuration ";
        print STDERR "information\n";
        exit(1);
    }
    # To populate the a nodes in existing cluster so that 
    # for start operation .cmanalyticsd_start file can be 
    # distributed to all nodes in cluster and for stop
    # operation .cmanalyticsd_start file can be removed 
    # from all nodes in cluster.
    populate_node_list_from_existing_cls(\@existing_cls_nodelist);

    # check if the coordinator is available. if yes then the start the 
    # Cluster Analytics daemon on that node.
    my @coordinator_lines = grep {/coordinator=/} @cmviewcl_lineoutput; 
    if (scalar(@coordinator_lines) ne 0) {
        chomp($coordinator_lines[0]);
        my @nodevals = split('=',$coordinator_lines[0]);
        if (defined($nodeName) && $nodeName ne $nodevals[1]) {
            print STDERR "ERROR: Cluster is UP on node $nodeName and it is not ";
            print STDERR "cluster coordinator,\n";
            print STDERR "Cluster Analytics daemon can be started/stopped on ";
            print STDERR "cluster coordinator only.\n";
            exit(1);
        }
        if($oper eq "start") {
            print STDOUT "@{[CMCAADMIN_CMD]}: Starting ";
        }
        elsif($oper eq "status") {
            print STDOUT "@{[CMCAADMIN_CMD]}: Checking ";
        }
        else {
            print STDOUT "@{[CMCAADMIN_CMD]}: Stopping ";
        }
        print STDOUT "Cluster Analytics daemon on cluster coordinator $nodevals[1]\n";
        $daemon_retval = chk_and_start_analytics_daemon($nodevals[1]); 
    }
    else {
        if(!defined($nodeName)) {
            my @node_id_lines = grep {/node:[a-zA-Z0-9_\\-]+\|id=1$/} @cmviewcl_lineoutput;
            if (scalar(@node_id_lines)>1) {
                print STDERR "ERROR: Cluster configuration has inconsistency\n";
                exit(1);    
            }
            chomp($node_id_lines[0]);
            my @nodenames = split('\|',$node_id_lines[0]);
            chomp($nodenames[0]);
            my @nodevals = split(':',$nodenames[0]);
            if($oper eq "start") {
                print STDOUT "@{[CMCAADMIN_CMD]}: Starting ";
            }
            elsif($oper eq "status") {
                print STDOUT "@{[CMCAADMIN_CMD]}: Checking ";
            }
            else {
                print STDOUT "@{[CMCAADMIN_CMD]}: Stopping ";
            }
            print STDOUT "Cluster Analytics daemon on node $nodevals[1]\n";
            $daemon_retval = chk_and_start_analytics_daemon($nodevals[1]);
        }
        else {
            if($oper eq "start") {
                print STDOUT "@{[CMCAADMIN_CMD]}: Starting ";
            }
            elsif($oper eq "status") {
                print STDOUT "@{[CMCAADMIN_CMD]}: Checking ";
            }
            else {
                print STDOUT "@{[CMCAADMIN_CMD]}: Stopping ";
            }
            print STDOUT "Cluster Analytics daemon on node $nodeName\n";
            $daemon_retval = chk_and_start_analytics_daemon($nodeName);
        }
    }
}
if($daemon_retval) {
    print STDERR "ERROR: Unable to perform $oper for Cluster Analytics daemon\n";
    exit(1); 
}

exit(0);

#########################################

sub populate_node_list_from_existing_cls {
    my ($populate_node_list_from_existing_cls_ref) = @_;
    # If nodes are not specified add cluster nodes if 
    # cluster configured otherwise localnode.
    if (!@$populate_node_list_from_existing_cls_ref) {
        my $cmd = "$SGSBIN/cmviewcl -f line -s config";
        my ($exit, @output) = runandCaptureCommand($cmd);
        if ($exit == 0) {
            foreach my $line (@output) {
                if ($line =~ /^node:(.+)\|name=/) {
                    my @parts = split('=',$line);
                    chomp $parts[1];
                    push(@$populate_node_list_from_existing_cls_ref,
                         $parts[1]);
                }
            }
        }
    }
    return;
}

sub process_command {
    my ($oper, $analytics_cmd, $local_host) = @_;
    my $ret = 0;
    my $retval = 0;
    my @node_on_cleanup_failed = ();
    my $suboper = "Creating";
    my $suboper1 = "Created";
    my $do_rollback_process = 1; 
    if ($debug) {
        if ($oper eq "stop") {
            $suboper = "Cleaning";
            $suboper1 = "Cleaned";
        }
        print STDOUT "@{[CMCAADMIN_CMD]}: $suboper file [$cmca_start_file]".
                     " on nodes [".join(", ", @existing_cls_nodelist)."]\n";
    }
    if ($oper eq "start") {
        #Distribute .cmanalyticsd_start file to all nodes
        $ret = run_cmsync($cmca_conf_file);
        if ($ret == 0) {
            $ret = run_cmsync($cmca_start_file);
            if ($ret == 0) {
                if (defined($local_host)) {
                    $retval = runandCaptureCommand($analytics_cmd);
                } else {
                    $retval = cmexecAndCapture($analytics_cmd);
                }
            }
        } else {
            $do_rollback_process = 0;
        }
    } elsif ($oper eq "stop") {
        #Remove .cmanalyticsd_start file to all nodes
        $ret = run_cmexec("$FIND_CMD $SGRUN -name ".
                          "$cmca_start_file_name -delete");
        if ($ret==0) {
            if (defined($local_host)) {
                 $retval = runandCaptureCommand($analytics_cmd);
            } else {
                 $retval = cmexecAndCapture($analytics_cmd);
            }
        }
    }
    if (($ret != 0) || ($retval != 0)) {
        if ($do_rollback_process) {
            exec_post_failure_rollback_process(\@node_accessible);
        }
        if ($ret != 0) { return ($ret) };
        if ($retval != 0) { return ($retval) };
    }
    if ($debug) {
        print STDOUT "@{[CMCAADMIN_CMD]}: $suboper1 file [$cmca_start_file] ".
                     "successfully on ".
                     "nodes [".join(", ", @existing_cls_nodelist)."]\n";
    }
    return (0);
}

sub exec_post_failure_rollback_process {
    my ($node_accessible_list_ref) = @_;
    my $node = undef;
    my $cmd = undef;
    my @node_on_cleanup_failed = ();
    my $overall_ret_val = 0;
    my $sub_op_name = "Cleaning";
    if ($debug) {
       if ($oper eq "stop") {
           $sub_op_name = "Restoring";
       }
       print STDOUT "@{[CMCAADMIN_CMD]}: $sub_op_name file [$cmca_start_file] on nodes";
       print STDOUT " [".join(", ", @$node_accessible_list_ref)."]\n";
    }

    if ($oper eq "start") {
        $cmd = "$FIND_CMD $SGRUN -name $cmca_start_file_name -delete";
    } elsif($oper eq "stop") {
        $cmd = "\">$cmca_start_file\"";
    } else {
        return;
    }
    if (@$node_accessible_list_ref) {
        foreach $node (@$node_accessible_list_ref) {
            my $final_cmd = "$SGSBIN/cmexec ". $node." ".$cmd;
            my ($ret, @out) = runandCaptureCommand($final_cmd);
            if ($ret != 0) {
                push(@node_on_cleanup_failed, $node);
            }
        }
        if (@node_on_cleanup_failed) {
            if ($oper eq "start") {
                if ($debug) {
                    print STDERR "WARNING: Fail to clean file ".
                                 "[$cmca_start_file] on nodes ".
                                 "[".join(", ", @node_on_cleanup_failed)."]\n";
                } else {
                    print STDERR "WARNING: Fail to clean Cluster Analytics ".
                                 "state configuration on nodes ".
                                 "[".join(", ", @node_on_cleanup_failed)."]\n";

                }
            } else {
                if ($debug) {
                    print STDERR "WARNING: Fail to restore file ".
                                 "[$cmca_start_file] on nodes ".
                                 "[".join(", ", @node_on_cleanup_failed)."]\n";
                } else {
                    print STDERR "WARNING: Fail to restore Cluster Analytics ".
                                 "state configuration on nodes ".
                                 "[".join(", ", @node_on_cleanup_failed)."]\n";
                }
            }
        } else {
            if ($debug) {
                if ($oper eq "start") {
                    print STDOUT "@{[CMCAADMIN_CMD]}: Cleaned file [$cmca_start_file] ".
                                 "successfully on ".
                                 "nodes [".join(", ", @$node_accessible_list_ref)."]\n";
                } else {
                    print STDOUT "@{[CMCAADMIN_CMD]}: Restored file [$cmca_start_file] ".
                                 "successfully on ".
                                 "nodes [".join(", ", @$node_accessible_list_ref)."]\n";
                }	    
            }
        }
    } else {
        if ($debug) {
            print STDOUT "Nothing to rollback\n";
        }
    }
    return;
}

sub chk_and_start_analytics_daemon {
    my $opernodename = shift;
    my $operhostname = undef;
    my $analytics_cmd = "$SGSBIN/cmanalyticsd"; 
    my $ps_analytics_cmd = "/usr/bin/pgrep cmanalyticsd";
    my @cmd_output=();
    my $retval = 0;
    if ($oper eq "stop") {
        $analytics_cmd = "/bin/kill -SIGTERM ";
    } 
    if(!$opernodename) {
        # start the Cluster Analytics daemon on by looking at the node names in 
        # $SGCONF/cmclnodelist. 
        if ($oper eq "start") {
            print STDOUT "@{[CMCAADMIN_CMD]}: Starting Cluster Analytics daemon ".
                         "based on entries in ";
            print STDOUT "$SGCONF/cmclnodelist\n";
        } 
        elsif ($oper eq "status") {
            print STDOUT "@{[CMCAADMIN_CMD]}: Checking Cluster Analytics daemon ".
                         "based on entries in ";
            print STDOUT "$SGCONF/cmclnodelist\n";
        }
        else {
            print STDOUT "@{[CMCAADMIN_CMD]}: Stopping Cluster Analytics daemon ".
                         "based on entries in ";
            print STDOUT "$SGCONF/cmclnodelist\n";
        }
        if(open (NODELISTFILE, "$SGCONF/"."cmclnodelist")) {
            my @nodelist_entries = <NODELISTFILE>;    
            @nodelist = ();
            foreach my $entry (@nodelist_entries) {
                chomp($entry);
                if($entry && !($entry =~ m/^#/)) {
                    $entry =~ s/^\s+//;
                    my @nodevals = split(' ', $entry);
                    if (!($nodevals[0] =~ m/\+/)) {
                        push(@nodelist, $nodevals[0]);
                    }
                }
            }
            if(scalar(@nodelist) == 0) {
                print STDERR "ERROR: Unable to process entries in ";
                print STDERR "$SGCONF/cmclnodelist\n";
                return(1);
            }
            my %seen = ();
            my @uniq_node_name_list = grep { !$seen{$_}++ } @nodelist;    
            my @sorted_node_list = sort(@uniq_node_name_list);
            $opernodename = $sorted_node_list[0];
            close(NODELISTFILE);
        }
        else {
            # cmclnodelist file does not exist, so start/stop Cluster
            # Analytics daemon on local node   
            $opernodename = $hostname;
        }
    } 

    if ($cl_exists_on_local_node == 0) {
        # There is no cluster then add only that node into existing_cls_nodelist
        # on which operation need to be performed.
        push(@existing_cls_nodelist, $opernodename);
    }
    
    if ($opernodename eq $hostname) {
        ($retval, @cmd_output) = runandCaptureCommand($ps_analytics_cmd);
    }
    else {
        @nodelist = ();
        push(@nodelist, $opernodename);
        if ($oper eq "status") {
           $retval = cmexecAndCapture("-k CMCASTAT");
        } else {
           $retval = cmexecAndCapture($ps_analytics_cmd);
        }
    }
    if ($oper eq "start") {
        if ($retval == 0) {
            print STDOUT "@{[CMCAADMIN_CMD]}: Cluster Analytics daemon is ".
                         "already running on node ";
            print STDOUT "$opernodename\n";
            return(0);
        }
    } elsif ($oper eq "status") { 
        if ($retval == 0) {
            print STDOUT "@{[CMCAADMIN_CMD]}: Cluster Analytics daemon is ".
                         "running on node ";
            print STDOUT "$opernodename.\n";
            return(0);
        } else {
            print STDOUT "@{[CMCAADMIN_CMD]}: Cluster Analytics daemon is ".
                         "not running.\n";
            return(0);
        }
    } else {
        if ($retval != 0) {
            print STDOUT "@{[CMCAADMIN_CMD]}: Cluster Analytics daemon is ".
                         "already stopped on node ";
            print STDOUT "$opernodename.\n";
            my $fileRemovalstatus = 0;
            #Remove .cmanalyticsd_start file to all nodes
            if ($debug) {
                print STDOUT "@{[CMCAADMIN_CMD]}: Cleaning file [$cmca_start_file]".
                             " on nodes [".join(", ", @existing_cls_nodelist)."]\n";
            }
            ($fileRemovalstatus) = run_cmexec_ignore_error("$FIND_CMD $SGRUN".
                                         " -name $cmca_start_file_name -delete");
            if ($fileRemovalstatus != 0) {
                print STDERR "\tNodes might be unreachable. ".
                             "Once unreachable nodes will be up,\n\t".
                             "run \"cmcaadmin cleanup\" command on each node.\n";
            } else {
                if ($debug) {
                    print STDOUT "@{[CMCAADMIN_CMD]}: Cleaned file [$cmca_start_file] ".
                                 "successfully on nodes ".
                                 "[".join(", ", @existing_cls_nodelist)."]\n";
                }
            }
            return(0);
        }
        if($opernodename eq $hostname) {
            chomp($cmd_output[0]);
            $analytics_cmd = $analytics_cmd . $cmd_output[0];
        }
        else {
            my @cmexecnodeout = @{$cmexecOutput{$opernodename}};
            if(scalar(@cmexecnodeout) > 1) {
                print STDERR "ERROR: Multiple Cluster Analytics daemon are running on ";
                print STDERR "node $opernodename\n";
                print STDERR "Check syslog on node $opernodename for more ";
                print STDERR "details\n";
                return(1);
            }
            chomp($cmexecnodeout[0]);
            $analytics_cmd = $analytics_cmd . $cmexecnodeout[0];
        }
    }
    #start/stop the daemon on the node
    if ($opernodename eq $hostname) {
        $operhostname = $opernodename;
    } else {
        @nodelist = ();
        push(@nodelist, $opernodename);
    }
    $retval = process_command($oper, $analytics_cmd, $operhostname);
    if ($oper eq "start") {
        if ($opernodename eq $hostname) {
            ($retval) = runandCaptureCommand($ps_analytics_cmd);
        } else {
            $retval = cmexecAndCapture($ps_analytics_cmd);
        }
    }
    if ($retval != 0) {
        print STDERR "ERROR: Cannot $oper Cluster Analytics daemon on node ".
                     "$opernodename.\n";
        if ($retval != ERROR_WITHOUT_SYSLOG_ENTRY) {
            print STDERR "Check syslog for more details\n";
        }
        return(1);
    }
    if ($oper eq "start") { 
        print STDOUT "@{[CMCAADMIN_CMD]}: Started Cluster Analytics daemon ".
                     "successfully on node ";
        print STDOUT "$opernodename.\n";
    }
    else {
        print STDOUT "@{[CMCAADMIN_CMD]}: Stopped Cluster Analytics daemon ".
                     "successfully on node ";
        print STDOUT "$opernodename.\n";
    }
    return(0);
}

sub check_if_cluster_exists {
    my $file_val = "$SGCONF/cmclconfig";
    if (-f $file_val && -s $file_val) {
        return(1);
    }
    return (0);
}
sub parse_args {
    my $node = 0;
    foreach my $option (@ARGV) {
        if ($option eq "cmcaadmin") {
            # do nothing
        }
        else {
            if ($node) {
                $nodeName = $option;
                $node = 0;
            }    
            else {
                if(defined($oper)) {
                    pod2usage("Multiple operation cannot be specified\n");
                    return(1);
                }
                if ($option eq "start") {
                    $oper = "start";
                }
                elsif ($option eq "stop") {
                    $oper = "stop";
                }
                elsif ($option eq "status") {
                    $oper = "status";
                }
                elsif ($option eq "cleanup") {
                    $oper = "cleanup";
                }
                else {
                    pod2usage("Invalid option specified\n");
                    return(1);
                }
            }
        }
    } 
    if (!defined($oper)) {
        pod2usage("Opertation(start|status|stop|cleanup) must be specified\n");
        return(1);
    }
    return(0);
}

sub runandCaptureCommand {
    my ($myCmd) = @_;
    my @lines = ();
    my @output = ();
    open(CMD, "$myCmd |") or return ($?>>8 , @output);
    while (<CMD>) {
        push(@output, $_);
        last if eof(CMD);
    }
    close(CMD);
    my $exitcode = ($? >> 8);
    return($exitcode, @output);
}

sub cmexecAndCapture {
    my ($cmd) = @_;
    my $ret = 0;
    my $pid;
    my %pidNodes;
    my $node;
    foreach $node (@nodelist) {
        $cmexecFiles{$node} = "/var/tmp/cmexec.$node.$$";
        $pid = undef;
        while (!defined($pid = fork)) {
            die "ERROR: fork failed: $!" if ($! != EAGAIN);
            sleep 1;
        }
        if ($pid == 0) {
           my $final_cmd = "$SGSBIN/cmexec ". $node." ". $cmd . " > ";
           $final_cmd = $final_cmd . $cmexecFiles{$node}. " 2>&1";
           exec($final_cmd) or 
                die "exec \"$cmd\" on node $node failed";
        }
        #parent
        $pidNodes{$pid} = $node;
    }

    my $runningPids = keys %pidNodes;
    while (-1 != ($pid = wait())) {
        my $stat = $?;
        if ($stat != 0) {
            $ret = 1;
            $node = $pidNodes{$pid};
        }
        $runningPids--;
    }
    die "ERROR: Failed to wait for all commands" if ($runningPids > 0);

    foreach my $node (values %pidNodes) {
        my $file = $cmexecFiles{$node};
        open(OUTFILE, "< $file") or print "ERROR: No output for node $node" and next;
        my @output = ();
        while (my $line = <OUTFILE>) {
            chomp $line;
            push(@output, $line);
        }
        $cmexecOutput{$node} = [ @output ];
        close(OUTFILE) or die "ERROR: close $cmd failed: %!";
        unlink $file;
    }
    return $ret;
}

sub run_cmsync {
    my ($fileName) = @_;
    my $ret = 0;
    my $pid;
    my %pidNodes;
    my $state = "";
    if ($fileName ne $cmca_conf_file) {
        if (!(-e $fileName)) {
                unless(open INFILE, '>' .$fileName) {
                # Die with error message
                # if we can't open it.
                die "Unable to create $fileName on $hostname\n";
            }
            close(INFILE);
        }
        $state = "state ";
    }
    foreach my $node (@existing_cls_nodelist) {
        my $cmd = "$SGSBIN/cmsync -n ". $node." ". $fileName . " &>/dev/null";
        ($ret)=runandCaptureCommand($cmd);
        if ($ret != 0) {
            if ($debug) {
                print STDERR "ERROR: Fail to create file [$cmca_start_file] ".
                             "on node $node, nodes might be unreachable.\n";
            } else {
                print STDERR "ERROR: Fail to distribute Cluster Analytics".
                             " $state"."configuration on node $node, ".
                             "nodes might be unreachable.\n";
            }
            $ret = ERROR_WITHOUT_SYSLOG_ENTRY;
            last;
        } else {
            push(@node_accessible, $node);
        }
    }
    # Below code block will be exercised only when command 
    # issued from node which does not have lowest member id
    # and no cluster exist on local node, to ensure that 
    # .cmanalyticsd_start file to be present only on node
    # where daemon is running as .cmanalyticsd_start file 
    # has been created locally then sync to remote node so
    # it need to be removed on local node.
    if ((scalar(@existing_cls_nodelist) == 1) && 
        ($existing_cls_nodelist[0] ne $hostname)) {
        runandCaptureCommand("$FIND_CMD $SGRUN -name ".
                             "$cmca_start_file_name -delete");
    }
    return $ret;
}

sub run_cmexec_ignore_error {
    my ($cmd) = @_;
    my $ret = 0;
    my $overall_ret = 0;
    my $suboper = "create";
    my $suboper1 = "distribute";
    if ($cmd =~ m/-delete/) {
        $suboper = "clean";
        $suboper1 = "clean";
    }
    foreach my $node (@existing_cls_nodelist) {
        my $cmd = "$SGSBIN/cmexec ". $node." ". $cmd . " &>/dev/null";
        ($ret)=runandCaptureCommand($cmd);
        if ($ret != 0) {
            $overall_ret=1;
            $ret = ERROR_WITHOUT_SYSLOG_ENTRY;
            push(@node_unreachable, $node);
        } else {
            push(@node_accessible, $node);
        }
    }
    if ($overall_ret != 0) {
        if ($debug) {
            print STDERR "ERROR: Fail to $suboper file [$cmca_start_file] ";
        } else {
            print STDERR "ERROR: Fail to $suboper1 Cluster Analytics".
                         " state configuration ";
        }
        print STDERR "on nodes [".join(", ", @node_unreachable)."]\n";
    }
    return $overall_ret;
}

sub run_cmexec {
    my ($cmd) = @_;
    my $ret = 0;
    my $suboper = "create";
    if ($cmd =~ m/-delete/) {
        $suboper = "clean";    
    }
    foreach my $node (@existing_cls_nodelist) {
        my $cmd = "$SGSBIN/cmexec ". $node." ". $cmd . " &>/dev/null";
        ($ret)=runandCaptureCommand($cmd);
        if ($ret != 0) {
            if ($debug) {
                print STDERR "ERROR: Fail to $suboper file [$cmca_start_file] ";
            } else {
                if ($suboper eq "clean") {
                    print STDERR "ERROR: Fail to clean Cluster ".
                                 "Analytics state configuration ";
                } else {
                    print STDERR "ERROR: Fail to distribute Cluster ".
                                 "Analytics state configuration ";
                }
            }
            print STDERR "on node $node, nodes might be unreachable.\n";
            $ret = ERROR_WITHOUT_SYSLOG_ENTRY;
            last;
        } else {
            push(@node_accessible, $node);
        }
    }
    return $ret;
}
