#!/usr/bin/perl -w
####################################################################
# (C) Copyright 2008-2015 Hewlett-Packard Development Company, L.P.
# @(#) Product Name                :  HP Serviceguard
# @(#) Product Version             :  A.12.10.00
# @(#) Patch Name                  :  
####################################################################
# Perl script for the cmscancl command.
#
# Description:  cmscancl is a configuration report and diagnostic tool 
#               which gathers system software and hardware configuration 
#               information from a list of nodes, or from all the nodes
#               in a Serviceguard cluster.  By default, the output will 
#               go to the file /tmp/scancl.out.
#
#               For remote nodes, this same script will be executed using
#               the cmexec command.  In order to handle the remote node
#               case, the LOG_ONLY flag will be set and a temporary file
#               will be created on the remote node to save the output.  
#               The remote output file will then be copied back to the 
#               local node.  This temporary file will have a filename 
#               consisting of the remote node name and the string "scnlg".
#
# Usage: cmscancl  [-n node_name]... [-s| -o output_file]
#
#                  -n node_name       Specify the node(s) to be scanned.
#
#                  -s                 Display the configuration information
#                                     on the screen.
#
#                  -o output_file     Write the configuration information
#                                     to a specified output file.  If this
#                                     option is not specified, output will
#                                     go to the file /tmp/scancl.out.
#
#                  --help             Displays this message.
#
# Returns:         0       if successful
#                  1       if failed
#
#
###########################################################################

use strict;

#Make script immune to localization
#Set the LANG env variable to C
$ENV{LANG}='C';

chomp(my $host = `hostname`);
$host =~ s/\..*//;
my $cmd_name = $0;
$cmd_name =~ s/.*\///;
my $config_file = "/etc/cmcluster.conf";
my $outfile = "";
my $default_outfile = "/tmp/scancl.out";
my $binary_file = "";
my @nodes = ();
my %SGVAR = ();
my $display_config = 0; # boolean: display config to screen, not to file.
my $log_only = 0;
my $first_node = "";
my $check_remote = 0;
my $cmexec = "";
my $cmcp = "";

check_user();
parse_args();
load_config();
if ($display_config) {
    open(OUTFILE, ">&STDOUT");
} else {
    open(OUTFILE, ">$outfile");
}

unless ($log_only) {
    unless (@nodes) {
        cluster_check();
    }

    for my $node (@nodes) {
        if ($node eq $host) {
            $check_remote = 1;
        }
    }

    unless ($display_config) {
        print "\nThe nodes to be scanned are: @nodes";
        print "\nThe output file is: $outfile";
    }

    remote_shell_access();

    get_sg_version();

    $SIG{TERM} = $SIG{INT} = "sig_handler";

    for my $node (@nodes) {
        # determine the first node in the node list
        unless ($first_node) {
            $first_node = $node;
        }

        unless ($display_config) {
            print "Getting information from $node...\n";
        }

        print OUTFILE "\n\n################# INFORMATION FOR THE NODE $node",
                      " ############### \n";
        if ($node eq $host) {
            log_output();
        } else {
            remote_log($node);
        }

        compare_files($node);
    }

    # only check remote connections when there are multiple nodes
    if (@nodes > 1) {
        test_ipv4_network_connections_remote();
        test_ipv6_network_connections_remote();
    }

    clean_up();

    if ($outfile) {
        print "\nDone.  Check log file $outfile for all information.\n";
    }

} else {
    log_output();
}
close(OUTFILE);

# End of function body
#############################################################

#############################################################
#  Subroutine Definitions

sub load_config {
    unless (-f $config_file) {
        print STDERR "$cmd_name: $config_file is missing.\n";
        exit 1;
    }

    open (CONFIG, $config_file);
    while (<CONFIG>) {
        s/#.*//; #remove comments
        if (/\s*(\w+)=(.+)\s/) {
            $SGVAR{$1} = $2;
        }
    }
    close (CONFIG);

    # Test for required variables
    my @required_vars = ("SGSBIN", "SGCONF",);
    for my $var (@required_vars) {
        unless ($SGVAR{$var}) {
            print STDERR "$cmd_name: $var not defined in $config_file.\n";
        }
    }

    $binary_file = $SGVAR{SGCONF} . "/cmclconfig";
    $cmexec = "$SGVAR{SGSBIN}/cmexec";
    $cmcp = "$SGVAR{SGSBIN}/cmcp";
}

sub check_user {
    # Checks the EUID
    unless ($> == 0) {
        print STDERR "$cmd_name: Must be super-user to run this command.\n";
        exit 1;
    }
}

sub parse_args {
    use Getopt::Long;
    Getopt::Long::Configure ('bundling');
    GetOptions ("n=s"  => \@nodes,
                "s"    => \$display_config,
                "o=s"  => \$outfile,
                "help" => sub { usage() }
               ) or usage();

    if ($display_config and $outfile) {
        print STDERR "$cmd_name: -s option cannot be specified in conjunction",
                     " with the -o option.\n";
        exit 1;
    }

    # if output file ends with scnlg, it is called from the     
    # cmscancl command from a remote node, set the LOG_ONLY
    # flag
    if ($outfile eq "/tmp/$host.scnlg") {
        $log_only = 1;
    }

    if (not $display_config and not $log_only) {
        if ($outfile and -e $outfile) {
            printf STDERR "\n$cmd_name: The output file '$outfile' already",
			  " exists.\n\tPlease remove the file and then issue ",
			  "the command again, or\n\tuse the -o option to ",
			  "specify another output file.\n";
        } elsif (not $outfile and -e $default_outfile) {
            rename $default_outfile, ($default_outfile . ".old");
            print "\nWARNING: The default output file $default_outfile ",
                    "already exists.\n\t The old file has been saved ",
                    " $default_outfile.old.\n";
        }

    }

    unless ($outfile or $display_config) {
        $outfile = $default_outfile;
    }
}


# Check to see if there is a binary config file.
# If file does not exist, only local node information will be returned.
# If file exists, get the node names from the binary config file.
#
sub cluster_check {
    # Test if file is not empty and that it exists
    if (!(-z $binary_file) and (-f $binary_file)) {
        my @lines = `$SGVAR{SGSBIN}/cmviewcl -l node -f line 2> /dev/null`;
        if ($?) {
            print STDERR "\n$cmd_name:  Unable to obtain node names from the ",
                         "cmviewcl command.\n";
            exit 1;
        }

        foreach (@lines) {
            if (/\|name=(.+)/) {
                push(@nodes, $1);
            }
        }
    } else {
        @nodes = ($host,);
        print "\n$cmd_name:\n",
              "\tThere is no binary configuration file on the system.  Only",
              " the local\n\tnode will be scanned.  If you want to scan other",
              " nodes, please\n\tuse the -n option to specify the names of the",
              " nodes that you want\n\tto gather information from.\n";
    }
}

# Check for cmexec access to all nodes in the cluster
sub remote_shell_access {
    unless ($display_config) {
        print "\nChecking cmexec access to all the nodes...\n";
    }

    # Get the os version of each of the nodes
    for my $node (@nodes) {
        my $cmd_out = `$cmexec $node uname -a 2>&1`;
        if ($? == 127) {
            print STDERR "\n$cmd_name: Can not find $cmexec\n",
            exit 1;
        } elsif ($? != 0) {
            print STDERR "\n$cmd_name: Can not cmexec to the system $node.",
                            "\n";
            exit 1;
        }
        print OUTFILE $cmd_out;
    }

    my $date = `date`;
    chomp($date);
    print OUTFILE "\n($date)\n";
}

sub get_sg_version {
    print OUTFILE " ";
    for my $node (@nodes) {
        my $sgver = `$cmexec $node rpm -q serviceguard 2>&1`;
        chomp($sgver);
        print OUTFILE "$sgver ($node)";
    }
}


sub log_output {
    my @cmds = ("ip addr show", "netstat -in", "netstat -i", "netstat -r", 
                "mount", "strings /etc/lvmtab", "cmgetlicense", 
                "cmgetlicense -f line", "cmversion -v -b");

    for my $cmd (@cmds) {
        my $cmd_out = `$cmd 2>&1`;
        print OUTFILE "\n------ Output of $cmd ($host) ------ \n";
        print OUTFILE $cmd_out;
    }
    
    open(CFGOUT, ">/tmp/$host.cfg");
    # Test if file is not empty and that it exists
    if (!(-z $binary_file) and (-f $binary_file)) {
        print CFGOUT cmviewcl_parser();
    } else {
        print CFGOUT "\n$cmd_name:  There is no binary configuration file on",
                      " the system.";
    }
    close(CFGOUT);
}

# Expects the name of a node as the only parameter
sub remote_log {
    my $node = shift;

    `$cmexec $node cmscancl -o /tmp/$node.scnlg`;

    `$cmcp $node:/tmp/$node.scnlg $host:/tmp/ 2> /dev/null`;
    if ($?) {
        print OUTFILE "\n$cmd_name:  Failed to get config info from the remote",
                      " node $host. (/tmp/$node.scnlg)";
        exit 1;
    }

    `$cmcp $node:/tmp/$node.cfg $host:/tmp/ 2> /dev/null`;
    if ($?) {
        print OUTFILE "\n$cmd_name:  Failed to get config info from the remote",
                      " node $host. (/tmp/$node.cfg)";
        exit 1;
    }

    if (-f "/tmp/$node.scnlg") {
        my $cmd_out = `cat /tmp/$node.scnlg 2>&1`;
        print OUTFILE $cmd_out;
    } else {
        print OUTFILE "\n$cmd_name: cmscancl not found on",
                      " node.\n\tInformation cannot be obtained.\n";
    }
}

# Expects the name of a node as the only parameter
sub compare_files {
    my $node = shift;
    my $cmd_out = "";

    # check the existence of the cfg file on the first node
    if ($node eq $first_node) {
        print OUTFILE "\n\n------ Contents of the Binary Configuration File",
                      " ($node) ------\n";

        if (-f "/tmp/$node.cfg") {
            $cmd_out = `cat /tmp/$node.cfg 2>&1`;
            print OUTFILE $cmd_out;
        } else {
            print OUTFILE "\n$cmd_name:  There is no binary configuration file",
                          " on the system.";
        }

    } else {
        print OUTFILE "\n------ Comparing $node binary configuration with",
                      " $first_node ------ \n";

        # check the existence of the cfg file 
        # if the file does not exists, no comparison will be done
        if (-f "/tmp/$node.cfg") {

            if (-f "/tmp/$first_node.cfg") {
                $cmd_out =  `diff /tmp/$node.cfg /tmp/$first_node.cfg`;
                if ($cmd_out) {
                    print OUTFILE "\n$cmd_out";
                } else {
                    print OUTFILE "\n(The cluster configuration files matched.",
                                  ")\n";
                }
            } else {
                print OUTFILE "\n$cmd_name:  There is no binary configuration ",
                              "file on $first_node.\n";
            }

        } else {
            print OUTFILE "\n$cmd_name:  There is no binary configuration file",
                          " on $node.\n"
        }
    }
}

# This function tests layer 3 connectivity by pinging from interface to every
# other interface in the cluster.  The first IP address gathered from each
# remote interface is used as the target for the ping.
#
sub test_ipv4_network_connections_remote {

    for my $src_node (@nodes) {
        my @lines = `$cmexec $src_node netstat -rn`;
        my %myints = ();
        ## Collect my interface names ##
        foreach (@lines) {
            if (/(eth\S+|bond\S+|ens\S+|eno\S+|enp\S+)/) {
                $myints{$1} = 1;
            }
        }
        DST_NODE: for my $dst_node (@nodes) {
            next DST_NODE if ($dst_node eq $src_node);

            print "Checking remote IPv4 network connections ($src_node to",
                  " $dst_node)...\n";
            print OUTFILE "\n###### Checking REMOTE layer 3 (IPv4) network",
                          " connections ($src_node to $dst_node) ######\n";
            my %rmints = ();
            @lines = `$cmexec $dst_node netstat -rn`;
            ## Collect remote interface names ##
            foreach (@lines) {
                if (/(eth\S+|bond\S+|ens\S+|eno\S+|enp\S+)/) {
                    $rmints{$1} = 1;
                }
            }
            for my $src_int (sort keys %myints) {
                for my $int (sort keys %rmints) {
                    #To handle localization, set LC_ALL before executing ip
                    @lines = `$cmexec $dst_node LC_ALL=C ip -o -4 addr show dev $int`;
                    my $address = "";
                    foreach (@lines) {
                        $_ =~ /(\d+\.\d+\.\d+\.\d+)/;
                        $address = $1;
                        if ($address) {
                            last;
                        }
                    }
                    print OUTFILE " ------ IPv4: node $src_node ",
                                  "($src_int) to $address ($int) on node ",
                                  "$dst_node ------\n";
                    `$cmexec $src_node "ping -I $src_int -c 5 -W 1 $address 2>&1 > /dev/null"`;
                    if ($?) {
                        print OUTFILE "(NO CONNECTION)\n";
                    } else {
                        print OUTFILE "OK\n";
                    }
                }
            }
        }
    }
}

# This function expands the "::" that can be used in IPv6 addresses.
# Examples:
#   ff::01 => ff:0:0:0:0:0:0:01
#   :: => 0:0:0:0:0:0:0:0
#
sub expand_ipv6_address {
    my $addr = shift;
    $addr =~ s/^::/0::/; # Adds a leading 0 if necessary.
    $addr =~ s/::$/::0/; # Adds a trailing 0 if necessary.
    # Continue adding ":0" until there are 7 colons (one will be a double ::)
    until ($addr =~ /^(\w+::?){7}\w+$/) {
        $addr =~ s/::/:0::/;
    }
    $addr =~ s/::/:/;

    return $addr;
}

# This function tests IPv6 layer 3 connectivity by going through every IPv6
# interface on each node, and having it ping an address on every IPv6 subnet on
# every other node.
#
sub test_ipv6_network_connections_remote {


    for my $src_node (@nodes) {
        my @netstat = `$cmexec $src_node netstat -rn -A inet6`;
        my %src_ints = ();
        ## Get source interface names ##
        foreach (@netstat) {
            if (/(eth\S+|bond\S+|ens\S+|eno\S+|enp\S+)/) {
                $src_ints{$1} = 1;
            }
        }
        RM_NODE: for my $dst_node (@nodes) {
            next RM_NODE if ($src_node eq $dst_node);
            print "Checking remote IPv6 network connections ($src_node to",
                    " $dst_node)...\n";
            print OUTFILE "\n####### Checking REMOTE layer 3 (IPv6) ",
                          "network connections ($src_node to $dst_node)",
                          " ######\n";
            @netstat = `$cmexec $dst_node netstat -rn -A inet6`;
            my (%rmints, %subnets);
            ## Get destination interface names ##
            foreach (@netstat) {
                if (/(eth\S+|bond\S+|ens\S+|eno\S+|enp\S+)/) {
                    $rmints{$1} = 1;
                }
                if (/(.+?):\/.*(eth\S*|bond\S*|ens\S*|eno\S*|enp\S*)\s*$/) {
                    $subnets{$1} = 1 unless /\/128/;
                }
            }
            for my $src_int (sort keys %src_ints) {
                for my $rmint (sort keys %rmints) {
                    my %addrs = ();
                    #To handle localization, set LC_ALL before executing ip
                    my @ifconfig = `$cmexec $dst_node LC_ALL=C ip -o -6 addr show dev $rmint`;
                    ## Get IPv6 addresses ##
                    foreach (@ifconfig) {
                        if (/inet6\s+(\S+)\s+scope (global|site)/) {
                            my $ipv6_addr = $1;
                            $ipv6_addr =~ s/\/[0-9]+//;
                            $addrs{ expand_ipv6_address($ipv6_addr) } = 1;
                        }
                    }
                    for my $subnet (sort keys %subnets) {
                        INNER: for my $address (sort keys %addrs) {
                            next INNER unless ($address =~ /$subnet/);

                            print OUTFILE " ------ IPv6: node $src_node ",
                                          "($src_int) to $address ($rmint) ",
                                          "on node $dst_node ------\n";
                            `$cmexec $src_node "ping6 -I $src_int -c 5 -W 1 $address 2>&1 > /dev/null"`;
                            if ($?) {
                                print OUTFILE "(NO CONNECTION)\n";
                            } else {
                                print OUTFILE "OK\n";
                            }
                            last INNER;
                        }
                    }
                }
            }
        }
    }
}

sub sig_handler {
    clean_up();
    close(OUTFILE);
    exit;
}

sub clean_up {
    for my $node (@nodes) {
        unlink "/tmp/$node.cfg";

        unless ($node eq $host) {
            unlink "/tmp/$node.scnlg";
        }

        `$cmexec $node rm /tmp/$node.scnlg 2> /dev/null`;
        `$cmexec $node rm /tmp/$node.cfg 2> /dev/null`;
    }
}

sub usage {
    print STDERR << "EOF";
Usage $cmd_name [-n node ...] [-s| -o output_file ] 

 -n node_name       Specify the node(s) to be scanned.

 -s                 Display the configuration information on the screen

 -o output_file     Write the configuration information to a specified output
                    file.  If this option is not specified, output will go to
                    the file /tmp/scancl.out.

 --help             Displays this message.
EOF
    exit 1;
}

##########################################################################
# cmviewcl parser
##########################################################################

sub cmviewcl_parser {

    unless (open(INFILE, "$SGVAR{SGSBIN}/cmviewcl -f line -s config -v|")) {
        return "Could not execute cmviewcl.\n";
    }

    # Fields that need conversion to seconds
    my %convert_me = ("heartbeat_interval" => 1,
                      "node_timeout" => 1,
                      "io_timeout_extension" => 1,
                      "configured_io_timeout_extension" => 1,
                      "auto_start_timeout" => 1,
                      "network_polling_interval" => 1,
                      "polling_interval" => 1,
                      "timeout_extension" => 1,
                     );

    my ($middle, $bottom) = ("", "");
    my $top = "\nCluster information:\n";

    my $line = <INFILE>;
    until (eof(INFILE)) {
        # Have to catch "(field):name|" and "(field)|".
        if ($line =~ /(^.+?)(|(:.+?))\|/) {
            if ($1 eq "weight_default") {
                $middle .= "\nPackage Weight Defaults\n";
                while ($line && $line =~ /^weight_default:(.+?)\|(.+?)=(.+)/) {
                    if ($2 eq "value") {
                        $middle .= sprintf("  %-36s %s\n", "$1:", $3);
                    }
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "site") {
                $middle .= "\nSite Information:\n";
                while ($line && $line =~ /^site:(.+?)\|(.+?)=(.+)/) {
                    if ($2 eq "id") {
                        $middle .= sprintf("  %-36s %s\n", "Site ID $3:", $1);
                    }
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "node") {
                (my $res, $line) = process_nodes(*INFILE, $line);
                $bottom .= $res;
                next;
            } elsif ($1 eq "package") {
                (my $res, $line) = process_packages(*INFILE, $line);
                $bottom .= $res;
                next;
            } elsif ($1 eq "quorum_server") {
                if ($line =~ /ip_address:.+?\|(.+?)=(.+)/) {
                    $top .= sprintf("  %-36s %s\n", "qs ip address:", $2);
                } else {
                    $line =~ /\|(.+?)=(.+)/;
                    my $val = $2;
                    if ($convert_me{$1}) {
                        $val = sprintf("%-8.2f (seconds)", $2/1000000.0);
                    }
                    $top .= sprintf("  %-36s %s\n", "qs $1:", $2);
                }
            } elsif ($1 eq "first_cluster_lock") {
                $middle .= "\nFirst Cluster Lock:\n";
                while ($line && $line =~ /^first_cluster_lock/) {
                    if ($line =~ /\|node:(.+?)\|(.+?)=(.+)/) {
                        $middle .= sprintf("    %-34s %s\n",
                                           "physical volume on $1:", $3);
                    } else {
                        $line =~ /.+\|(.+?)=(.+)/;
                        $middle .= sprintf("  %-36s %s\n", "$1:", $2);
                    }
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "cluster_lock") {
                $middle .= "\nCluster Lock information:\n";
                while ($line && $line =~ /^cluster_lock/) {
                    if( $line =~ /^cluster_lock\|node:(.+?)\|lun=(.+)/) {
                        $middle .= sprintf("  %-36s %s\n", "node $1 lun", $2);
                    }elsif ($line =~ /^cluster_lock\|node:(.+?)\|interface_type=(.+)/) {
                        $middle .= sprintf("  %-36s %s\n", "node $1 interface_type", $2);
                    }
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "authorized_host") {
                $middle .= "\nCluster Access Policies:";
                while ($line && $line =~ /^authorized_host/) {
                    if ($line =~ /user:.+?\|(.+?)=(.+)/) {
                        $middle .= sprintf("  %-36s %s\n", "user $1:", $2);
                    } else {
                        $line =~ /(.+?)=(.+)/;
                        $middle .= sprintf("\n  %-36s %s\n", "user host:", $2);
                    }
                    $line = <INFILE>;
                }
                next;
            }
            # Eats anything that's not handled.
            $line = <INFILE>;
        } else {
            # Cluster key/val pairs
            $line =~ /(.+?)=(.+)/;
            my $val = $2;
            if ($convert_me{$1}) {
                $val = sprintf("%-8.2f (seconds)", $2/1000000.0);
            }
            if ($1 ne "status" && $1 ne "state") {
                $top .= sprintf("  %-36s %s\n", "$1:", $val);
            }
            $line = <INFILE>;
        }
    }
    return $top . $middle . $bottom;
}

sub process_packages {
    local *INFILE = shift;
    my $line = shift;
    my ($retstr, $title, $keyvals, $body) = ("", "", "", "");

    $line =~ /^package:(.+?)\|/;
    my $cur_pkg = $1;

    while ($line && $line =~ /^package:(.+?)\|/) {
        if ($cur_pkg ne $1) {
            $cur_pkg = $1;
            $retstr .= $title . $keyvals . $body;
            $title = $keyvals = $body = "";
        }
        if ($line =~ /^package:.+?\|(.+?):.+\|/) {
            if ($1 eq "environment") {
                $body .= "\n    Package Environments:";
                while ($line && $line =~ /environment:.+?\|(.+?)=(.+)/) {
                    $body .= "\n" if ($1 eq "name");
                    $body .= sprintf("      %-32s %s\n", "$1:", $2);
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "weight") {
                $body .= "\n    Package Weights:\n";
                while ($line && $line =~ /weight:(.+?)\|(.+?)=(.+)/) {
                    if ($2 eq "value") {
                        $body .= sprintf("      %-32s %s\n", "$1:", $3);
                    }
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "node") {
                $body .= "\n    Package Nodes:\n";
                while ($line && $line =~ /node:.+?\|(.+?)=(.+)/) {
                    $body .= sprintf("      %-32s %s\n", "$1:", $2);
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "dependency") {
                $body .= "\n    Package Dependencies:";
                while ($line && $line =~ /dependency:.+?\|(.+?)=(.+)/) {
                    $body .= "\n" if ($1 eq "name");
                    $body .= sprintf("      %-32s %s\n", "dependency $1:", $2);
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "subnet") {
                $body .= "\n    Package Subnets:\n";
                while ($line && $line =~ /subnet:.+?\|(.+?)=(.+)/) {
                    $body .= sprintf("      %-32s %s\n", "$1:", $2);
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "resource") {
                $body .= "\n    Package Resources:";
                while ($line && $line =~ /resource:.+?\|(.+?)=(.+)/) {
                    $body .= "\n" if ($1 eq "name");
                    $body .= sprintf("      %-32s %s\n", "$1:", $2);
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "service") {
                (my $res, $line) = process_package_services(*INFILE, $line);
                $body .= $res;
                next;
            } elsif ($1 eq "authorized_host") {
                $body .= "\n    Package Access Policies:";
                while ($line && $line =~ /authorized_host:/) {
                    if ($line =~ /\|user:.+?\|(.+?)=(.+)/) {
                        $body .= sprintf("      %-32s %s\n", "user $1:", $2);
                    } else {
                        $line =~ /(.+?)=(.+)/;
                        $body .= sprintf("\n      %-32s %s\n", "user host:",
                                         $2);
                    }
                    $line = <INFILE>;
                }
                next;
            }
            # Eats anything that's not handled.
            $line = <INFILE>;
        } else {
            $line =~ /\|(.+?)=(.+)/;
            if ($1 eq "id") {
                $title = "\n  Package ID $2:\n";
            } else {
                $keyvals .= sprintf("    %-34s %s\n", "$1:", $2);
            }
            $line = <INFILE>;
        }
    }
    return ($retstr . $title . $keyvals . $body,  $line);
}

sub process_package_services {
    local *INFILE = shift;
    my $line = shift;
    my ($retstr, $keyvals) = ("", "");
    my $title = "\n    Package Services:";

    $line =~ /^package:.+?\|service:(.+?)\|/;
    my $cur_service = $1;

    while ($line && $line =~ /^package:.+?\|service:(.+?)\|/) {
        if ($cur_service ne $1) {
            $cur_service = $1;
            $retstr .= $title . $keyvals;
            $title = $keyvals = "";
        }
        $line =~ /package:.+?\|service:.+?\|(.+?)=(.+)/;
        next if ($1 eq "command"); #Don't print the service command
        if ($1 eq "id") {
            $title .= sprintf("\n      %-32s %s\n",  "service ID:", $2);
        } else {
            $keyvals .= sprintf("        %-30s %s\n", "$1:", $2);
        }
    } continue {
        $line = <INFILE>;
    }

    return ($retstr . $title . $keyvals,  $line);
}

sub update_remote_node_info {
    my $node = shift;
    my $sginfo = "";
    my @lines = `$SGVAR{SGSBIN}/cmviewcl -v -f line -l node 2> /dev/null`;

    for my $line (@lines) {
        if ( $line =~/^node:(.+?)\|(.+?)=(.+)/) {
              if ($1 eq $node ) {
                  if ($line =~/^node:(.+?)\|(.+?)=(.+)/) {
                     if ( $2 eq "sg_version" ) {
                          $sginfo .= sprintf("    %-34s %s\n", "$2:", $3);
                     }
                     if ( $2 eq "os_release" ) {
                          $sginfo .= sprintf("    %-34s %s\n", "$2:", $3);
                     }
                     if ( $2 eq "cpu_architecture" ) {
                          $sginfo .= sprintf("    %-34s %s\n", "$2:", $3);
                     }
                     if ( $2 eq "boot_timestamp" ) {
                          $sginfo .= sprintf("    %-34s %s\n", "$2:", $3);
                     }
                   }

              }
          }
     }
     return ($sginfo);
}

sub process_nodes {
    local *INFILE = shift;
    my $line = shift;
    my ($title, $keyvals, $body) = ("", "", "");
    my $retstr = "\nCluster Node Information:\n";

    $line =~ /^node:(.+?)\|/;
    my $cur_node = $1;

    while ($line && $line =~ /^node:(.+?)\|/) {
        if ($cur_node ne $1) {
            $cur_node = $1;
            $retstr .= $title . $keyvals . $body;
            $title = $keyvals = $body = "";
        }
        if ($line =~ /^node:.+?\|(.+?):.+\|/) {
            if ($1 eq "interface") {
                (my $res, $line) = process_node_interfaces(*INFILE, $line);
                $body .= $res;
                next;
            } elsif ($1 eq "subnet") {
                $body .= "\n    Node Subnets:";
                while ($line && $line=~ /subnet:.+\|(.+?)=(.+)/) {
                    if ($1 eq "name") {
                        $body .= sprintf("\n      %-32s %s\n", "Subnet Name:",
                                         $2);
                    } else {
                        $body .= sprintf("        %-30s %s\n", "$1:", $2);
                    }
                    $line = <INFILE>;
                }
                next;
            } elsif ($1 eq "capacity") {
                $body .= "\n    Node Capacity:";
                while ($line && $line =~ /\|capacity:.+?\|(.+?)=(.+)/) {
                    if ($1 eq "name") {
                        $body .= "\n";
                    }
                    $body .= sprintf("      %-32s %s\n", "$1:", $2);
                    $line = <INFILE>;
                }
                next;
            }
        } else {
            # The key value pairs for node:
            $line =~ /\|(.+?)=(.+)/;
            if ($1 eq "id") {
                $title = "\n  Node ID $2:\n";
            } else {
              if ($cur_node ne $host) {
                    if (($1 eq "sg_version")  and  ($2 eq "unknown") ) {
                        $keyvals .= update_remote_node_info($cur_node);
                    } else {
                        $keyvals .= sprintf("    %-34s %s\n", "$1:", $2);
                    }
                } else {
                $keyvals .= sprintf("    %-34s %s\n", "$1:", $2);
                }
            }
        }
        $line = <INFILE>;
    }
    return ($retstr . $title . $keyvals . $body,  $line);
}



sub process_node_interfaces {
    local *INFILE = shift;
    my $line = shift;
    my ($retstr, $title, $keyvals, $body) = ("", "", "", "");

    $line =~ /^node:.+?\|interface:(.+?)\|/;
    my $cur_int = $1;

    while ($line && $line =~ /^node:.+?\|interface:(.+?)\|/) {
        if ($cur_int ne $1) {
            $cur_int = $1;
            $retstr .= $title . $keyvals . $body;
            $title = $keyvals = $body = "";
        }
        if ($line =~ /^node:.+?\|interface:.+?\|(.+?):.+\|/) {
            if ($1 eq "ip_address") {
                my $cur_ip = "";
                while ($line && $line =~ /\|ip_address:(.+?)\|/) {
                    if ($cur_ip ne $1) {
                        $cur_ip = $1;
                        if ($1 =~ /:/) {
                            $body .= "\n      IPv6 Information:\n";
                        } else {
                            $body .= "\n      IPv4 Information:\n";
                        }
                    }
                    if ($line =~ /ip_address:.+?\|(.+?)=(.+)/) {
                        $body .= sprintf("        %-30s %s\n", "$1:", $2);
                    }
                    $line = <INFILE>;
                }
                next;
            }
        } else {
            $line =~ /interface.+\|(.+?)=(.+)/;
            if ($1 eq "id") {
                $title = "\n    Network ID $2:\n";
            } else {
                $keyvals .= sprintf("      %-32s %s\n", "$1:", $2);
            }
        }
        $line = <INFILE>;
    }

    return ($retstr . $title . $keyvals . $body,  $line);
}

