use strict;

package Prod::VCS51::Common;
@Prod::VCS51::Common::ISA = qw(Prod);

use Obj;
use Net::IP qw(:PROC);
use Time::Local;

sub init_common {
    my ($prod,$padv);
    $prod = shift;
    $prod->{prod}="VCS";
    $prod->{abbr}="VCS";
    $prod->{vers}="5.1.100.000";
    $prod->{proddir}="cluster_server";
    $prod->{eula}="EULA_VCS_Ux_5.1SP1.pdf";
    $prod->{name}=Msg::new("Veritas Cluster Server", 40, 2219)->{msg};
    $prod->{mainpkg}="VRTSvcs51";
    $prod->{subprods}=[qw(AT50)];

    $prod->{lic_names}=["Veritas Cluster Server"];

    $prod->{menu_options}=["Global Cluster Option"];

    $prod->{minimal_memory_requirment} = "256 MB";

    $prod->{llthosts}="/etc/llthosts";
    $prod->{llttab}="/etc/llttab";
    $prod->{gabtab}="/etc/gabtab";
    $prod->{maincf}="/etc/VRTSvcs/conf/config/main.cf";
    $prod->{typescf}="/etc/VRTSvcs/conf/config/types.cf";
    $prod->{extra_types_cf}="/etc/VRTSvcs/conf/extra_types_cf";
    $prod->{vxfenmode}="/etc/vxfenmode";
    $prod->{vxfendg}="/etc/vxfendg";
    $prod->{nofinish_encap}=1;

    $prod->{configfiles}=[ $prod->{maincf}, $prod->{typescf} ];
    $prod->{secure}="/etc/VRTSvcs/conf/config/.secure";

    # define frequently referenced paths
    $prod->{confdir}="/etc/VRTSvcs/conf";
    $prod->{configdir}="/etc/VRTSvcs/conf/config";
    $prod->{bindir}="/opt/VRTSvcs/bin";
    $prod->{gcmbindir}="/opt/VRTSgcm/bin";

    $prod->{uuidconfig}="$prod->{bindir}/uuidconfig.pl";
    $prod->{uuidfile}="/etc/vx/.uuids/clusuuid";
    $prod->{responsefileupgradeok}=1;
    $prod->{cmd_hacf}="$prod->{bindir}/hacf";

    # For UXRT 5.1SP1, need define obsoleted pkgs from GA release
    $prod->{obsoleted_pkgs_of_ga_release} = [ qw(VRTScutil) ];

    # define commands consistent across platforms
    $padv = Obj::padv($prod->{padv});
    $padv->{cmd}{lltstat} = "/sbin/lltstat";
    $padv->{cmd}{lltconfig} = "/sbin/lltconfig";
    $padv->{cmd}{gabconfig} = "/sbin/gabconfig";
    $padv->{cmd}{amfconfig} = "/opt/VRTSamf/bin/amfconfig";
    $padv->{cmd}{vxfenconfig} = "/sbin/vxfenconfig";
    $padv->{cmd}{vxfenadm} = "/sbin/vxfenadm";
    $padv->{cmd}{hastop} = "/opt/VRTSvcs/bin/hastop";
    $padv->{cmd}{hasys} = "/opt/VRTSvcs/bin/hasys";
    $padv->{cmd}{hagrp} = "/opt/VRTSvcs/bin/hagrp";
    $padv->{cmd}{hares} = "/opt/VRTSvcs/bin/hares";
    $padv->{cmd}{hastatus} = "/opt/VRTSvcs/bin/hastatus";


    # define role grouping attributes
    $prod->{rolegroupingattrs} = [ qw(AdministratorGroups OperatorGroups Guests) ] ;
    
    # todo: used for dynupgrade and need to be platform specific
    $prod->{deleted_agents} = [ qw() ];

    $prod->{max_number_in_cluster} = 64;

    # limits of heartbeat links
    $prod->{max_lltlinks} = 8;
    $prod->{max_hipri_links} = 4;
    #security default port number
    $prod->{vssdefport}=2821;

    $prod->{has_config} = 1;
}

sub adjust_ru_procs {
    my $prod=shift;
    if(Cfg::opt("upgrade_nonkernelpkgs")){
        my $procs=[qw(had51 CmdServer51 hashadow51 )];
        return $procs;
    }
}

sub ask_fencing_enabled {
    my ($ayn,$help,$msg,$description);
    my $prod = shift;
    my $cfg = Obj::cfg();
    my $vxfen=Obj::proc("vxfen51",$prod->{padv});
    return "" if (Cfg::opt("responsefile"));
    $msg = Msg::new("I/O Fencing\n", 40, 1262);
    $msg->bold;
    $msg = Msg::new("It needs to be determined at this time if you plan to configure I/O Fencing in enabled or disabled mode, as well as help in determining the number of network interconnects (NICS) required on your systems. If you configure I/O Fencing in enabled mode, only a single NIC is required, though at least two are recommended.\n\nA split brain can occur if servers within the cluster become unable to communicate for any number of reasons. If I/O Fencing is not enabled, you run the risk of data corruption should a split brain occur. Therefore, to avoid data corruption due to split brain in CFS environments, I/O Fencing has to be enabled.\n", 40, 3372);
    $msg->print;
    $description = $msg->{msg};
    $msg = Msg::new("If you do not enable I/O Fencing, you do so at your own risk\n", 40, 1264);
    $msg->bold;
    $description .= $msg->{msg};
    $msg = Msg::new("See the Administrator's Guide for more information on I/O Fencing", 40, 3373);
    $msg->print;
    $description .= $msg->{msg};
    $help = $msg;
    $msg = Msg::new("\nDo you want to configure I/O Fencing in enabled mode?", 40, 1266);
    $ayn = $msg->ayny_web($help,0,$description);
    Msg::n();
    $cfg->{fencingenabled} = ($ayn eq "Y") ? 1 : 0;
}

sub set_options {
    my ($prod,$options,$id,$option);
    ($prod,$options)=@_;
    $id=0;
    foreach $option (@{$options}) {
        if($option) {
           if ($id == 0) {
               Cfg::set_opt("gco", 1);
           }
        }
        $id++;
    }
}

sub pkgs {
    my(@pkgs,@minpkgs,@recpkgs,$pkg,$prod,$rel);
    $prod=shift;
    $rel=Obj::rel($prod->{release}, $prod->{padv});
    for $pkg(@{$rel->{pkgs}}) {
        next if ($pkg=~/VRTS(aa|ccg|mh|ob)/);
        push(@pkgs,$pkg);
    }
    push(@pkgs, @{$prod->{pkgs}});
    $prod->{pkgs}=EDR::arruniq(@pkgs);
    push(@minpkgs, @{$rel->{minpkgs}}, @{$prod->{minpkgs}});
    $prod->{minpkgs}=EDR::arruniq(@minpkgs);
    push(@recpkgs, @{$rel->{recpkgs}}, @{$prod->{recpkgs}});
    $prod->{recpkgs}=EDR::arruniq(@recpkgs);

    return EDR::arruniq(@pkgs);
}

sub description {
    my ($msg);

    $msg=Msg::new("Veritas Cluster Server, the industry's leading open systems clustering solution, is ideal for eliminating both planned and unplanned downtime, facilitating server consolidation, and effectively managing a wide range of applications in heterogeneous environments.", 40, 2220);
    $msg->print;
}

sub verify_procs_list {
    my ($prod,$procs,$state)=@_;
    my $cfg = Obj::cfg();
    if ((defined $state && $state eq "stop") && (Cfg::opt("configure")) &&
        ($cfg->{donotreconfigurevcs})) {
        $procs = $prod->remove_procs_for_prod($procs);
    }
    return $procs;
}

sub stopprocs {
    my $prod=shift;
    my $ref_procs = Prod::stopprocs($prod);
    return adjust_ru_procs if(Cfg::opt("upgrade_nonkernelpkgs"));
    $ref_procs = $prod->verify_procs_list($ref_procs,"stop");
    return $ref_procs;
}

sub verify_procs_list_sys {
    my ($prod,$sys,$procs,$state)=@_;
    my $cfg = Obj::cfg();
    if ((defined $state && $state eq "stop") && (Cfg::opt("configure")) &&
        ($cfg->{donotreconfigurevcs})) {
        $procs = $prod->remove_procs_for_prod($procs);
    }
    return $procs;
}

sub stopprocs_sys {
    my ($prod,$sys)=@_;
    my $ref_procs = Prod::stopprocs_sys($prod, $sys);
    return adjust_ru_procs if(Cfg::opt("upgrade_nonkernelpkgs"));
    $ref_procs = $prod->verify_procs_list_sys($sys,$ref_procs,"stop");
    return $ref_procs;
}

sub vcs_enable_sys {
    my ($prod,$sys) = @_;
    my ($proci,$proc,@list);
    my $cfg = Obj::cfg();
    if ($cfg->{vcs_allowcomms}) {
        @list = qw(llt51 gab51 had51);
        $prod->vxfen_enable_sys($sys);
    } else {
        @list = qw(had51);
    }
    for $proci (@list) {
        $proc = $sys->proc($proci);
        $proc->enable_sys($sys);
    }
}

sub vxfen_enable_sys {
    my ($prod,$sys) = @_;
    my ($proc,$rootpath);
    $rootpath = Cfg::opt("rootpath");
    $proc = $sys->proc("vxfen51");
    if (EDR::file_sys($sys,"$rootpath$prod->{vxfenmode}") || EDR::file_sys($sys,"$rootpath$prod->{vxfendg}")) {
        $proc->enable_sys($sys);
    }
}

sub vcs_disable_sys {
    my ($prod,$sys) = @_;
    my ($proci,$proc);
    
    for $proci qw(llt51 gab51 had51 vxfen51) {
        $proc = $sys->proc($proci);
        $proc->disable_sys($sys);
    }
}

sub vcs_reservedwords {
    my (@vcs_res_words,$prod,$name,$word);
    ($prod,$name) = @_;
    @vcs_res_words = (
        "ArgListValues",
        "Cluster",
        "ConfidenceLevel",
        "ConfidenceMsg",
        "Group",
        "IState",
        "MonitorOnly",
        "Name",
        "Path",
        "Probed",
        "Signaled",
        "Start",
        "State",
        "System",
        "Type",
        "MonitorMethod",
        "NameRule",
        "action",
        "after",
        "before",
        "boolean",
        "cluster",
        "condition",
        "event",
        "false",
        "firm",
        "global",
        "group",
        "hard",
        "heartbeat",
        "i18nstr",
        "int",
        "keylist",
        "local",
        "localclus",
        "offline",
        "online",
        "remote",
        "remotecluster",
        "requires",
        "resource",
        "set",
        "soft",
        "start",
        "state",
        "static",
        "stop",
        "str",
        "system",
        "temp",
        "true",
        "type",
    );

    return 1 if (EDR::inarr($name, @vcs_res_words));
    return 0;
}

sub set_vcs_allowcomms_sys {
    my ($prod,$sys) = @_;
    my $cfg = Obj::cfg();
    my $rootpath = Cfg::opt("rootpath") || "";
    return 0 if (!$sys->system1); 
    if (EDR::file_sys($sys,"$rootpath$prod->{llthosts}")
            && EDR::file_sys($sys,"$rootpath$prod->{llttab}")
            && EDR::file_sys($sys,"$rootpath$prod->{gabtab}")) {
        $cfg->{vcs_allowcomms} = 1;
        Thr::setq($cfg->{pool}, "vcs_allowcomms", 1);
    } else {
        $cfg->{vcs_allowcomms} = 0;
        Thr::setq($cfg->{pool}, "vcs_allowcomms", 0);
    }
    return 1;
}

sub upgrade_ru_check_sys {
     my ($prod,$llt,$gab,$had,$sys,$msg);
    ($prod,$sys)=@_;
    return 1 unless (Cfg::opt("upgrade_nonkernelpkgs") || Cfg::opt("upgrade_kernelpkgs"));
    $had = $sys->proc("had51");
    $gab = $sys->proc("gab51");
    $llt = $sys->proc("llt51");
    unless ($had->check_sys($sys) && $gab->check_sys($sys) &&  $llt->check_sys($sys)) {
        $msg = Msg::new("To do rolling upgrade, VCS must be running on $sys->{sys}.", 40, 3374, "$sys->{sys}");
        EDR::push_error_sys($sys,$msg);
        return 0;
    }
    return 1;

}


sub verify_responsefile {
    my $prod=shift;
    my(@sevs,$cfg,$csg,$msg,$n,$nics,$j,$pkg,$sys,@systems,$at);
    return if (Cfg::opt("uninstall"));
    $cfg=Obj::cfg();
    @sevs=("Information","Warning","Error","SevereError");

    # verify cluster config: CLUSTERNAME, CLUSTERID, LLTLINK
    if (($cfg->{vcs_clustername}) || ($cfg->{vcs_clusterid}) || ($cfg->{"vcs_lltlink1"})) {
        unless ($prod->tsub("verify_clustername",$cfg->{vcs_clustername})) {
            $msg=Msg::new("vcs_clustername has an incorrect value", 40, 2221);
            $msg->die;
        }
        if ($cfg->{vcs_allowcomms} && ($cfg->{opt}{configure} || $cfg->{opt}{installconfig})) {
            #don't check VCS_LLTLINK when using -fencing or -cps
            if ( !Cfg::opt("fencing") ) {
                if ( !EDR::isint($cfg->{vcs_clusterid}) || ($cfg->{vcs_clusterid}<0) || ($cfg->{vcs_clusterid}>65535) ) {
                    $msg=Msg::new("vcs_clusterid must be an integer value between 0 and 65535", 40, 2222);
                    $msg->die;
                }
                if (Cfg::opt("addnode")) {
                    @systems = @{$cfg->{newnodes}};
                } else {
                    @systems = @{$cfg->{systems}};
                }
                foreach $sys (@systems) {
                    if (!$cfg->{"vcs_lltlink1"}{$sys}) {
                        $msg=Msg::new("vcs_lltlink1 is not configured for system $sys", 40, 2223, "$sys");
                        $msg->die;
                    }
                }
            }
        }
    }
    
    # verify password for secure cluster
    $at=Obj::prod("AT50",$prod->{padv});
    $at->verify_responsefile();
    
    # verify csg config: CSGVIP, CSGNETMASK
    if ((defined($cfg->{vcs_csgvip})) || $cfg->{vcs_csgnetmask} || (defined($cfg->{vcs_csgnic}))) {
        $csg=1;
        if ( $cfg->{vcs_csgvip} && !EDR::isip($cfg->{vcs_csgvip})) {
            $msg=Msg::new("vcs_csgvip is not a valid IP address", 40, 2224);
            $msg->die;
        }
        if ($cfg->{vcs_csgnetmask} && !EDR::isip($cfg->{vcs_csgnetmask})) {
            $msg=Msg::new("vcs_csgnetmask is not a valid netmask", 40, 2225);
            $msg->die;
        }
        foreach $sys (@{$cfg->{systems}}) {
            $nics++ if (defined($cfg->{vcs_csgnic}{$sys}));
        }
    }
        
    # verify smtp config: SMTPSERVER, SMTPRECP, SMTPRSEV
    if (($cfg->{vcs_smtpserver}) || (defined($cfg->{vcs_smtprecp})) || (defined($cfg->{vcs_smtprsev}))) {
        $csg=1;
        unless ($prod->tsub("verify_smtpserver",$cfg->{vcs_smtpserver})) {
            $msg=Msg::new("vcs_smtpserver has an incorrect value", 40, 2226);
            $msg->die;
        }
        if ($#{$cfg->{vcs_smtprecp}} != $#{$cfg->{vcs_smtprsev}}) {
            $msg=Msg::new("There are an unequal number of vcs_smtprecp and vcs_smtprsev array entries", 40, 2227);
            $msg->die;
        }
        foreach $n (0..$#{$cfg->{vcs_smtprecp}}) {
            unless ($prod->tsub("verify_emailadd",${$cfg->{vcs_smtprecp}}[$n])) {
                $msg=Msg::new("vcs_smtprecp has an incorrect value", 40, 2228);
                $msg->die;
            }
            unless (EDR::inarr(${$cfg->{vcs_smtprsev}}[$n],@sevs)) {
                $msg=Msg::new("${$cfg->{vcs_smtprsev}}[$n] is an incorrect vcs_smtprsev severity value.\nMust be Information, Warning, Error, or SevereError.", 40, 2229, "${$cfg->{vcs_smtprsev}}[$n]");
                $msg->die;
            }
        }
    }
    
    # verify snmp config: SNMPPORT, SNMPCONS, SNMPCSEV
    if (($cfg->{vcs_snmpport}) || (defined($cfg->{vcs_snmpcons})) || (defined($cfg->{vcs_snmpcsev}))) {
        $csg=1;
        if (!EDR::isint($cfg->{vcs_snmpport}) || (($cfg->{vcs_snmpport} > 65535) || ($cfg->{vcs_snmpport} < 0))) {
            $msg=Msg::new("vcs_snmpport has an incorrect value", 40, 2230);
            $msg->die;
        }
        if ($#{$cfg->{vcs_snmpcons}} != $#{$cfg->{vcs_snmpcsev}}) {
            $msg=Msg::new("There are an unequal number of vcs_snmpcons and vcs_snmpcsev array entries", 40, 2231);
            $msg->die;
        }
        foreach $n(0..$#{$cfg->{vcs_snmpcons}}) {
            unless (EDR::inarr(${$cfg->{vcs_snmpcsev}}[$n],@sevs)) {
                $msg=Msg::new("${$cfg->{vcs_snmpcsev}}[$n] is an incorrect vcs_snmpcsev severity value.\nMust be Information, Warning, Error, or SevereError.", 40, 2232, "${$cfg->{vcs_snmpcsev}}[$n]");
                $msg->die;
            }
        }
    }

    # validating definition, but could validate value too
    if ($csg) {
        unless (($cfg->{vcs_csgnic}{all}) || ($nics>$#{$cfg->{systems}})) {
            $msg=Msg::new("vcs_csgnic is not correctly set for all systems", 40, 2233);
            $msg->die;
        }
    }

    # check if it is single node configure/upgrade/start. Verify the value of VCS_ALLOWCOMMS in that case.
    if (Cfg::opt("configure")||Cfg::opt("upgrade")||Cfg::opt("start")) {
        if (!defined($cfg->{vcs_allowcomms})) {
            if (scalar(@{$cfg->{systems}}) == 1) {
                $msg=Msg::new("vcs_allowcomms must be set to 1 or 0 for a single node cluster configuration", 40, 3375);
                $msg->die;
            } elsif (scalar(@{$cfg->{systems}}) > 1) {
                $cfg->{vcs_allowcomms} = 1;
            }
        }
    }
    if (defined ($cfg->{fencingenabled}) &&
        ($cfg->{fencingenabled} != 0) &&
        ($cfg->{fencingenabled} != 1)) {
        $msg=Msg::new("fencingenabled has invalid value", 40, 3376);
        $msg->die;
    }
    $pkg = Obj::pkg("VRTSvxfen51", $prod->{padv});
    $pkg->verify_responsefile_for_fencing() if ($cfg->{fencingenabled});
}

sub uninstall_precheck_sys {
   my ($prod,$sys,$cpic);
   ($prod,$sys) = @_; 
    $cpic = Obj::cpic();
    return 1;
}

sub upgrade_precheck_sys {
    my ($cpic,$csystesm,$nsystems,$pids,$had,$msg,$prod,$sys);
    ($prod,$sys) = @_;
    $cpic=Obj::cpic();
    
    $prod->tsub("check_timesync_sys", $sys, $cpic->{systems}, 5) if($sys->system1 && (EDR::num_sys_keyset("stop_checks")==0));
    $prod->tsub("upgrade_ru_check_sys",$sys);
    $prod->tsub("check_cluster_members_sys",$sys);
    # check if vcs has vaild configurations and could be upgrade
    $prod->tsub("check_upgradeable_sys", $sys) unless ($sys->{vcs_upgradeable});
    # set $cfg->{vcs_allowcomms}
    $prod->tsub("set_vcs_allowcomms_sys",$sys);
    # check whether or not vxfen config files are valid or need to be updated.
    $prod->vxfen_upgrade_precheck_sys($sys);

    # check for vcs status
    unless (Cfg::opt("rootpath")) {
        $had = $sys->proc("had51");
        if($had->check_sys($sys, "start")) {
            EDR::set_value_sys($sys,'prod_running',1);
        } else {
            if ($prod->{upgrade_ignore_conf}) {
                if ($sys->system1) {
                    EDR::create_flag("maincf_upgrade_precheck_done");
                } 
                
                Msg::log("Had is not running, but upgrade_ignore_conf is set. Skipping config updates.");
                return 1;
            } else {
                if ($sys->system1) {
                    EDR::create_flag("maincf_upgrade_precheck_done");
                }  
                
                $msg = Msg::new("$prod->{abbr} is not running before upgrade. Please make sure all the configurations are valid before upgrade.", 40, 3377, "$prod->{abbr}");
                EDR::push_warning_sys($sys,$msg);
            }
        }
    }

    # check whether any upgrade issues for main.cf
    return 0 unless ($prod->maincf_upgrade_precheck_sys($sys));

    return 1;
}

sub mount_res_sys {
    my ($prod, $sys) = (@_);
    my ($mounts,$mount,$res_mp,$res_sys,$block_devices,@block_devices,$bd,@bds);
    # check Mount resource for FS.pm
    my $vcsmainpkg=Obj::pkg($prod->{mainpkg},$prod->{padv});
    my $iv=$vcsmainpkg->version_sys($sys);
    return \@block_devices unless ($iv);
    my $had = $sys->proc("had51");
    if($had->check_sys($sys, "start")) {
        $mounts = EDR::cmd_sys($sys, "_cmd_hares -display -type Mount -attribute FSType");
        for $mount (split(/\n+/,$mounts)) {
            next if ($mount =~ /^#/);
            if ($mount =~ /\s*(\S+)\s+FSType\s+(\S+)\s+vxfs/) {
                $res_mp = $1;
                $res_sys = $2;
                if (($res_sys eq $sys->{hostname}) || ($res_sys eq $sys->{sys})
                    || ($res_sys eq "global")) {
                    $block_devices = EDR::cmd_sys($sys, "_cmd_hares -display $res_mp -attribute BlockDevice");
                    for $bd (split(/\n+/,$block_devices)) {
                        if ($bd =~ /\s*$res_mp\s+BlockDevice\s+\S+\s+(\S+)/) {
                            push (@block_devices,$1);
                        }
                    }
                }
            }
        }
    }
    return \@block_devices;
}

sub patchupgrade_precheck_sys {
    my ($had,$prod,$sys,$cpic);
    ($prod,$sys) = @_;
    $cpic=Obj::cpic();
    $prod->tsub("check_timesync_sys", $sys, $cpic->{systems}, 5) if($sys->system1 && (EDR::num_sys_keyset("stop_checks")==0));
    $prod->tsub("upgrade_ru_check_sys",$sys);
    # set $cfg->{vcs_allowcomms}
    $prod->tsub("set_vcs_allowcomms_sys",$sys);    
    # check whether or not vxfen config files need to be updated.
    $prod->vxfen_upgrade_precheck_sys($sys);
    # check VCS running status
    $had = $sys->proc("had51");
    if ($had->check_sys($sys)) {
        EDR::set_value_sys($sys,'prod_running',1);
    }
    return if (Cfg::opt("upgrade_kernelpkgs"));
    # check if vcs has vaild configurations and could be upgrade
    $prod->tsub("check_upgradeable_sys", $sys) unless ($sys->{vcs_upgradeable});
    if ($sys->{vcs_upgradeable}) {
        $prod->maincf_upgrade_precheck_sys($sys);
    } elsif ($sys->system1) {
        EDR::create_flag("maincf_upgrade_precheck_done");
    }

}

sub precheck_task {
    my $prod = shift;
    if (Cfg::opt(qw(patchupgrade upgrade))) {
        $prod->tsub("phased_upgrade_precheck") unless (Cfg::opt("upgrade_kernelpkgs"));
    }
    return;
}

sub phased_upgrade_precheck {
    my($cpic,$csystems,$edr,$msg,$nsystems,$prod,$str,$sys1,$sysi,$sysj,$obj_sysi,@syslist,$vcs,$ver1,$veri);
    $vcs = shift;

    $cpic = Obj::cpic();
    $edr = Obj::edr();
    $prod = $cpic->prod;

    $sys1 = $cpic->{systems}->[0];
    # Get number of systems from the main.cf
    $nsystems = EDR::cmd_sys($sys1, "_cmd_grep '^system' $vcs->{maincf} 2> /dev/null | _cmd_wc -l");
    $csystems = $#{$cpic->{systems}} + 1;
    Msg::log("Number of systems in main.cf = $nsystems");
    Msg::log("Number of systems provided = $csystems");

    if ($csystems < $nsystems) {
        $ver1 = $prod->version_sys($sys1);
        for $sysi (@{$cpic->{systems}}) {
            push (@syslist, $sysi->{hostname});
        }
        $str = EDR::cmd_sys($sys1, "_cmd_grep '^system' $vcs->{maincf} 2> /dev/null");
        for $sysi(split(/\n/,$str)) {
            next unless ($sysi =~ /^system\s+(\S+)\s*/);
            $sysi = $1;
            $sysi =~ s/"(.*)"/$1/; # remove double quote
            unless (EDR::inarr($sysi, @syslist)) {
                $obj_sysi = EDR::tsub("Sys", "new", $sysi);
                if (!$edr->tsub("transport_sys",$obj_sysi)) {
                    Msg::log("Cannot communicate with system $obj_sysi->{sys}\n");
                    next;
                }
                if (!EDR::inarr($obj_sysi->{padv}, @{$edr->{padvs}})) {
                    Msg::log("The OS kernel release on system $obj_sysi->{sys} is not supported\n");
                    next;
                }

                $edr->tsub("packages_sys", $obj_sysi);
                $edr->tsub("patches_sys", $obj_sysi);
                $veri = $prod->version_sys($obj_sysi);
                if (EDR::compvers($ver1,$veri,4) == 2) {
                    $msg = Msg::new("You are performing $prod->{prod} phased upgrade (Phased 2) on the systems. The second subcluster will be upgraded.", 40, 3378, "$prod->{prod}");
                    EDR::push_note_sys($sys1, $msg);
                    for $sysj (@{$cpic->{systems}}) {
                        $sysj->{proc_reboot}=1;
                    }
                    $vcs->{upgrade_ignore_conf}=1;
                    return "";
                }
            }
        }
        $msg = Msg::new("You are performing $prod->{prod} phased upgrade (Phased 1) on the systems. The first subcluster will be upgraded.", 40, 3379, "$prod->{prod}");
        EDR::push_note_sys($sys1, $msg);
        for $sysj (@{$cpic->{systems}}) {
            $sysj->{proc_reboot}=1;
        }
        $vcs->{phased_upgrade_1}=1;
    } else {
        Msg::log("$cpic->{prod} Full Upgrade");
    }
    return "";
}

sub completion_messages {
    my $prod=shift;
    $prod->phased_upgrade_completion_messages($prod);
}

sub phased_upgrade_completion_messages {
    my ($prod,$msg,$padv);
    $prod=shift;

    if ($prod->{phased_upgrade_1}) {
        $msg = Msg::new("You are performing phased upgrade (Phased 1) on the systems. Follow the steps in install guide to upgrade the remaining systems.", 40, 3710);
        $msg->print;
    }
    return "";

}

sub check_timesync_sys {
    my $prod = shift;
    my $sys_t = shift; # target host of the checking
    my $syslistref = shift;
    my $time_sync  = shift;
    my $errstr = "";

    my ($item, $desc, $status, $summary, $hashref, $ret);
    my ($syncro_spec, %nodes_timer, $timer_sin, %timer_sec, @timer_coun);
    my ($sec,$min, $hour, $mday, $mon, $year, $timer_min, $timer_max);
    my ($msg, $sys, $sys1, $sys2, $ii, $flag, @diff, $timer_diff);
    my ($local_time, $time1, $time2);
     
    Msg::log("Time synchronization check for $prod->{abbr}");
    foreach $sys (@{$syslistref}) {
        $sys = Obj::sys($sys) if(ref($sys) ne "Sys");
        eval {
            $timer_sin = EDR::cmd_sys($sys, "_cmd_date +%S:%M:%H:%d:%m:%Y");
            $local_time = EDR::cmd_local("_cmd_date +%S:%M:%H:%d:%m:%Y");
        };
        $errstr = $@;
        if ($errstr) {
            Msg::log("Can't get date and error info: $errstr");
            $msg = Msg::new("Couldn't complete the Time synchronization check. Failed to get system 'date' information.", 40, 3059);
            EDR::push_warning_sys($sys, $msg);
            return 1;
        }
        
        eval {
            ($sec,$min,$hour,$mday,$mon,$year) = split(/:/,$timer_sin);
            $time1 = timegm ($sec, $min, $hour, $mday, $mon - 1, $year);
            
            ($sec,$min,$hour,$mday,$mon,$year) = split(/:/,$local_time);
            $time2 = timegm ($sec, $min, $hour, $mday, $mon - 1, $year);

            $timer_diff = $time1-$time2;
            unless(defined $timer_max){
                $timer_max=$timer_min=$timer_diff;
            }
            $timer_max = ($timer_diff>$timer_max)?$timer_diff:$timer_max;
            $timer_min = ($timer_diff<$timer_min)?$timer_diff:$timer_min;
        };
        $errstr = $@;
        if ($errstr) {
            Msg::log("Can't change \$timer_ref:$timer_sin and error info: $errstr");
            $msg = Msg::new("Couldn't complete the Time synchronization check. Failed to perform range checking on the system 'date'.", 40, 3060);
            EDR::push_warning_sys($sys, $msg);
            return 1;
        }
    }
    if ($timer_max-$timer_min>$time_sync) {
        $msg = Msg::new("Nodes have difference in clock by more than 5 sec", 40, 3047);
        EDR::push_warning_sys($sys_t, $msg);
        return 1;
    } else {
        $summary = Msg::new("Synchronized time setting on cluster nodes checking of $prod->{abbr} is OK.", 40, 3380, "$prod->{abbr}");
        $summary->log;
    }
    return 0;


}

sub check_cluster_members_sys {
    my ($conf,$msg,$m,$n,$sys0);
    my ($prod,$sys) = @_;
    my $cpic = Obj::cpic();
    $conf = $prod->get_config_sys($sys);
    if ($conf) {
        if ($sys->system1) {
            $m = scalar(@{$conf->{systems}});
            $n = scalar(@{$cpic->{systems}});
            return 1 if($m == $n);
            if ( $m < $n) {
                $msg = Msg::new("The entered systems belong to different clusters. Only the members of the same cluster can be upgraded as a whole.", 40, 2236);
                EDR::push_error_sys($sys,$msg);
            } elsif ($m > $n) {
                return 1 unless ($cpic->{prod} eq "VCS51" || $cpic->{prod} eq "SFHA51");
                $msg = Msg::new("Not all systems of the cluster, $conf->{clustername}, have been entered to be upgraded. It is recommended that all cluster systems are upgraded together, except when you plan to perform phased upgrade on the set of systems.", 40, 3381, "$conf->{clustername}");
                EDR::push_warning_sys($sys,$msg);
            }
        } else {
            $sys0 = ${$cpic->{systems}}[0];
            $conf = $prod->get_config_sys($sys0);
            return 0 unless ($conf);
            return 1 if(EDR::inarr($sys->{sys},@{$conf->{systems}}));
            return 1 if(EDR::inarr($sys->{hostname}, @{$conf->{systems}}));
            $msg = Msg::new("$sys->{sys} does not belong to the cluster, $conf->{clustername}. Only the members of the same cluster can be upgraded as a whole.", 40, 2238, "$sys->{sys}", "$conf->{clustername}");
            EDR::push_error_sys($sys,$msg);
        }
    } else {
        $msg = Msg::new("Cannot find valid $prod->{abbr} configuration information on $sys->{sys}", 40, 2239, "$prod->{abbr}", "$sys->{sys}");
        EDR::push_error_sys($sys,$msg);
    }
    return 0;    
}

sub vxfen_upgrade_precheck_sys {
    my ($prod,$sys)=@_;
    my ($conf,$msg);
    my $rootpath = Cfg::opt("rootpath") || "";
    return 1 if ((Cfg::opt("patchupgrade")) && (!$prod->check_config())); 
    if (EDR::file_sys($sys,"$rootpath$prod->{vxfenmode}")) {
        $conf=$prod->get_vxfen_config_sys($sys);
        if (($conf->{vxfen_mode} !~ /^(disabled|customized)$/) && 
            (!EDR::file_sys($sys,"$rootpath$prod->{vxfendg}"))) {
                $msg=Msg::new("Vxfen may fail to start on $sys->{sys} after upgrade due to missing $rootpath$prod->{vxfendg}", 40, 3382, "$sys->{sys}", "$rootpath$prod->{vxfendg}");
                EDR::push_warning_sys($sys,$msg);
        }
    }
    return 1;
}

sub get_vxfen_config_sys {
    my ($prod,$sys)=@_;
    my ($line,$str,$sysname);
    my ($cpserver, $port);
    my $conf={};
    my $rootpath = Cfg::opt("rootpath") || "";
    $conf->{vxfen_mode}=$conf->{vxfen_diskpolicy}=$conf->{vxfen_mechanism}="";
    $str=EDR::cmd_sys($sys,"_cmd_cat $rootpath$prod->{vxfenmode} 2> /dev/null"); 
    for $line(split(/^/,$str)) {
        $line=EDR::despace($line);
        next if ((!$line) || ($line=~/^#/));
        if ($line=~/vxfen_mode\s*=\s*(\w+)/) {
            $conf->{vxfen_mode}=$1;
        } elsif ($line=~/scsi3_disk_policy\s*=\s*(\w+)/) {
            $conf->{scsi3_disk_policy}=$1;
        } elsif ($line=~/vxfen_mechanism\s*=\s*(\w+)/) {
            $conf->{vxfen_mechanism}=$1;
        }
    }
    $str = EDR::cmd_sys($sys,"_cmd_cat $rootpath$prod->{vxfenmode} 2> /dev/null | _cmd_grep '^cps'");
    $sysname = $sys->{hostname};
    for $line(split(/\n/,$str)) {
        ($cpserver, $port) = split(/:/, $line); 
        # Capture the name between []
        if ($cpserver =~ /\[(.*)\]/) { 
            $cpserver = $1;
        } else {
            next;
        }
        next unless ($cpserver);
        push (@{$conf->{cps}{$sysname}}, $cpserver);
        $port = EDR::despace($port);
        $port ||= "";
        if ($port eq "") {
            $conf->{"$cpserver"}{cpport}{$sysname} = "14250";
        } else {
            $conf->{"$cpserver"}{cpport}{$sysname} = $port;
        }
    }
    return $conf;
}

sub maincf_upgrade_precheck_sys {
    my ($prod,$sys,$rtn,$attrs,$line);
    my ($delete_cmc,$sfua_delete_SfuaBase,$delete_VCSweb);
    my (@grps,$grp,$unfreeze_sg);
    my ($cmd_hacf,$msg,$rootpath,$treeref_olduser,$upgradepath);
    my (@grp_syslist,$grp_syslists,$vxss_autostartlist,$vxss_add_autostartlist);
    my @extra_types;

    ($prod,$sys) = @_;
    $rootpath = Cfg::opt("rootpath") || "";
    $upgradepath= EDR::tmpdir()."/upgrade";
    $cmd_hacf = $prod->{cmd_hacf};

    # Use hacf on ABE only if the OS on PBE is same with ABE.  
    if ($rootpath) {
        my @uname = split(/\s+/,$sys->{uname});
        if ($sys->{platvers} eq $uname[2]) {
            $cmd_hacf= "$rootpath$prod->{bindir}/hacf";
            Obj::set_value($prod->{pool},"cmd_hacf",$cmd_hacf);
            Thr::setq($prod->{pool},"cmd_hacf",$cmd_hacf);
        } else {
            EDR::set_value_sys($sys, "osupgraded",1);
        }
    }

    if ($sys->system1) {

        # dump vcs configuraton
        if (!Cfg::opt("rootpath") && !$prod->{upgrade_ignore_conf} && $sys->{prod_running}) {
            $prod->haconf_makerw();
            $prod->haconf_dumpmakero();
        }

        unless ($prod->dynupgrade_pre_upgrade_sys($sys)) {
            EDR::create_flag("maincf_upgrade_precheck_done");
            return 0;
        }

        # create cmd list
        $treeref_olduser = dynupgrade_import_file2tree_sys($sys, "$rootpath$prod->{configdir}/main.cmd");

        # Change sourcefile path for types/service groups to config dir
        modify_sourcefile_sys($sys,$treeref_olduser,"GRP");
        my $sourcefiles= modify_sourcefile_sys($sys,$treeref_olduser,"TYPE");
        dynupgrade_export_tree2file_sys( $sys, "$upgradepath/old_config/main.cmd", $treeref_olduser);
        $prod->translate_file_cmd2cf_sys( $sys, "$upgradepath/old_config");

        # Copy types for VRTSvcsea and VRTSdbac to old_conf;
        my ($extra_type,$extra_type_base,$pkg,$pkgi);
        for $pkgi qw( VRTSvcsea51 VRTSdbac51) {
            $pkg = $Obj::pool{"Pkg\::$pkgi\::$sys->{padv}"};
            next unless (defined $pkg);
            push (@extra_types,@{$pkg->{extra_types}});
        }
        for $extra_type (@extra_types,@{$prod->{extra_types}}) {
            $extra_type_base = EDR::basename($extra_type);
            if (EDR::inarr($extra_type_base, @{$sourcefiles})) {
                EDR::cmd_sys($sys, "_cmd_cp $upgradepath/old_config/$extra_type_base $upgradepath/old_conf");
                EDR::set_value_sys($sys, "maincf_include,$extra_type_base", $extra_type_base);
            }
        }

        unless ($prod->check_obsolete_types_sys($sys,$treeref_olduser)) {
            EDR::create_flag("maincf_upgrade_precheck_done");
            return 0;
        }

        my $panic_system_on_dg_loss_attr = dynupgrade_get_attrvalue_of_typename($treeref_olduser,"DiskGroup","PanicSystemOnDGLoss");
        if ($panic_system_on_dg_loss_attr eq "1") {
            my @reslist=dynupgrade_get_resname_list_from_typename($treeref_olduser, "DiskGroup");
            if (@reslist) {
                $msg = Msg::new("The default value of the diskGroup agent attribute, PanicSystemOnDGLoss, will be changed from 1 to 0 after upgrade. Refer to the Veritas Cluster Server Bundled Agents Reference Guide for the implications of this change.", 40, 3383);
                EDR::push_warning_sys($sys,$msg);
            }
        }

        $prod->upgrade_cvm_cfs_types_sys($sys,$treeref_olduser);

        @grps=dynupgrade_get_grplist($treeref_olduser);
        # unfreeze service group
        unless (Cfg::opt("rootpath")) {
            for $grp(@grps) {
                $unfreeze_sg="hagrp -modify $grp Frozen 0";
                EDR::set_value_sys($sys, "maincf_upgrade,20_unfreeze_$grp", $unfreeze_sg);
            }
        }

        # Add AutoStartList to VxSS Service Group if it is not defined.
        if (EDR::inarr("VxSS",@grps)) {
            $vxss_autostartlist = dynupgrade_get_attr_from_tree($treeref_olduser,"GRP","VxSS","AutoStartList");
            unless ($vxss_autostartlist) {
                @grp_syslist = dynupgrade_get_systemlist_of_grpname($treeref_olduser,"VxSS");
                $grp_syslists = join (" ", @grp_syslist);
                $vxss_add_autostartlist = "hagrp -modify VxSS AutoStartList $grp_syslists";
                EDR::set_value_sys($sys, "maincf_upgrade,21_vxss_add_autostartlist", $vxss_add_autostartlist);
            }
        }

        # Update DNS ResRecord.
        my $dns_resrecord = dynupgrade_modify_attr_value_DNS($treeref_olduser);
        if ($dns_resrecord) {
            EDR::set_value_sys($sys, 'maincf_upgrade,30_dns_resrecord', $dns_resrecord);
        }

        #remove service group.
        #Add 99_ prefix to make sure the delete operation is added to the end.
        $sfua_delete_SfuaBase="hagrp -delete Sfua_Base";
        EDR::set_value_sys($sys, 'maincf_upgrade,99_sfua_delete_SfuaBase', $sfua_delete_SfuaBase);
        $delete_cmc="hagrp -delete CMC";
        EDR::set_value_sys($sys, 'maincf_upgrade,99_delete_CMC', $delete_cmc);

        $delete_VCSweb="hares -delete VCSweb";
        EDR::set_value_sys($sys, 'maincf_upgrade,99_delete_VCSweb', $delete_VCSweb);

        $prod->padv_maincf_upgrade_precheck_sys($sys,$treeref_olduser) if ($prod->can('padv_maincf_upgrade_precheck_sys'));

        EDR::create_flag("maincf_upgrade_precheck_done");
    } else {
        EDR::wait_for_flag("maincf_upgrade_precheck_done");
    }
    return 1;
}

sub upgrade_cvm_cfs_types_sys {
    my ($prod, $rtn, $sys, $treeref_olduser);
    my ($cfsmount_attr, $cvmvoldg_reglist, $cvmvoldg_arglist);
    my ($type_name,$type_node,$attr_name,$attr_node);
    ($prod,$sys,$treeref_olduser) = (@_);

    # Update CFSMount
    # Add AMFMountType as a attr to CFSMount in 5.1SP1
    $type_name = "CFSMount";
    if ($type_node = dynupgrade_get_node_from_tree( $treeref_olduser, "TYPE", $type_name)) {
        $attr_name = "AMFMountType";
        $attr_node = dynupgrade_get_node_from_tree( $type_node, "ATTR", $attr_name);
        # do nothing if AMFMountType already exist
        unless ($attr_node) {
            $cfsmount_attr = "haattr -add -temp CFSMount AMFMountType -string";
            EDR::set_value_sys($sys, 'maincf_upgrade,10_cfsmount_add', $cfsmount_attr);
        }
    }

    # Update CFSMount
    # Add Primary to ArgList in CFSMount in 5.1
    # Add AMFMountType to ArgList in CFSMount in 5.1SP1
    $rtn = dynupgrade_get_attrvalue_of_typename($treeref_olduser,"CFSMount","ArgList");
    if ($rtn) {
        $rtn .= " Primary" if ($rtn!~/Primary/);
        $rtn .= " AMFMountType" if ($rtn!~/AMFMountType/);
        $cfsmount_attr="hatype -modify CFSMount ArgList $rtn";
        EDR::set_value_sys($sys, 'maincf_upgrade,10_cfsmount_attr', $cfsmount_attr);
    }

    # Update CVMVolDg
    # Add CVMVolume to RegList in CVMVolDg in 5.1
    $rtn = dynupgrade_get_attrvalue_of_typename($treeref_olduser,"CVMVolDg","RegList");
    if ($rtn) {
        $rtn .= " CVMVolume" if ($rtn!~/CVMVolume/);
        $cvmvoldg_reglist="hatype -modify CVMVolDg RegList $rtn";
        EDR::set_value_sys($sys, 'maincf_upgrade,11_cvmvoldg_reglist', $cvmvoldg_reglist);
    }
    # Add CVMVolumeIoTest, CVMDGAction to ArgList in CVMVolDg in 5.1
    # Add CVMDeportOnOffline to ArgList in CVMVolDg in 5.1SP1
    $rtn = dynupgrade_get_attrvalue_of_typename($treeref_olduser,"CVMVolDg","ArgList");
    if ($rtn) {
        $rtn .= " CVMVolumeIoTest" if ($rtn!~/CVMVolumeIoTest/);
        $rtn .= " CVMDGAction" if ($rtn!~/CVMDGAction/);
        $rtn .= " CVMDeportOnOffline" if ($rtn!~/CVMDeportOnOffline/);
        $rtn .= " State" if ($rtn!~/State/);
        $cvmvoldg_arglist="hatype -modify CVMVolDg ArgList $rtn\n";
        $cvmvoldg_arglist.="haattr -add CVMVolDg CVMVolumeIoTest -keylist\n";
        $cvmvoldg_arglist.="hatype -modify CVMVolDg OnlineRetryLimit 2\n";
        $cvmvoldg_arglist.="hatype -modify CVMVolDg OnlineTimeout 400\n";
        $cvmvoldg_arglist.="haattr -add CVMVolDg CVMDGAction -string\n";
        $cvmvoldg_arglist.="haattr -add CVMVolDg CVMDeportOnOffline -integer";
        EDR::set_value_sys($sys, 'maincf_upgrade,11_cvmvoldg_arglist', $cvmvoldg_arglist);
    }

}

# Get attribute value of $attr_name for type $type_name from main.cmd
# Same as this command: hatype -display $type_name -attribute $attr_name
sub dynupgrade_get_attrvalue_of_typename {
    my ($treeref, $type_name, $attr_name)= @_;
    my $branchref= dynupgrade_get_node_from_tree( $treeref, "TYPE", $type_name);
    my $noderef = dynupgrade_get_node_from_tree ($branchref, "ATTR", $attr_name);
    return (defined $noderef->{ADD}->{ATTRVAL}) ?
        $noderef->{ADD}->{ATTRVAL} : $noderef->{MODIFY}->{ATTRVAL};
}

# Modify sourcefile 
sub modify_sourcefile_sys {
    my ($sys, $treeref, $tree_name)= @_;
    my ($key, $sourcefile, $sourcefilebase, @sourcefiles);
    for $key (keys %{$treeref->{$tree_name}}) {
        $sourcefile = dynupgrade_get_sourcefile($treeref,$tree_name,$key);
        $sourcefile =~ s/^\"(\S+)\"$/$1/;
        $sourcefilebase=EDR::basename($sourcefile);
        if ( ($sourcefile =~ /^\//) || (($sourcefile =~ /\.\//) && ($sourcefile !~ /^\.\/$sourcefilebase$/)) ){
            dynupgrade_set_sourcefile($treeref,$tree_name,$key,"\"\.\/$sourcefilebase\"");
            EDR::set_value_sys($sys, "maincf_sourcefile_$tree_name,$key", "$sourcefile");
        }
        push (@sourcefiles, $sourcefilebase) if ($tree_name eq "TYPE");
    }
    return ($tree_name eq "TYPE") ? EDR::arruniq(@sourcefiles) : "";
}

# Get sourcefile of $key in tree $tree_name
sub dynupgrade_get_sourcefile {
    my ($treeref, $tree_name, $key)= @_;
    my $branchref= dynupgrade_get_node_from_tree( $treeref, $tree_name, $key);
    return $branchref->{FILE}->{ATTRVAL};
}

# Set sourcefile of $key in tree $tree_name
sub dynupgrade_set_sourcefile {
    my ($treeref, $tree_name, $key, $source_file)= @_;
    my $branchref= dynupgrade_get_node_from_tree( $treeref, $tree_name, $key);
    $branchref->{FILE}->{ATTRVAL}=$source_file;
    return;
}

# Get group list from main.cmd
# Same as this command: hagrp -list
sub dynupgrade_get_grplist {
    my $treeref = shift;
    my $grpref = dynupgrade_get_branch_from_tree( $treeref, "GRP");
    return (keys %{$grpref});
}

# Get resources list for group $grp_name from main.cmd
# Same as this command: hagrp -resources $grp_name
sub dynupgrade_get_res_from_grp {
    my ($treeref,$grp_name) = @_;
    my @resname_list= ();
    my $branchref= dynupgrade_get_branch_from_tree( $treeref, "RES");
    my ( $resname, $noderef, $cmdref);
    
    foreach $resname ( keys %{$branchref} ) {
        # get each resource node and main command
        $noderef= dynupgrade_get_node_from_branch( $branchref, $resname);
        $cmdref= $noderef->{MAIN};
        # compare the group name with required one
        if( $cmdref->{SERVGRP} eq $grp_name) {
            push ( @resname_list, $resname);
        }
    }
    
    return @resname_list;
}

# Get attribute value of $attr_name for resource $res_name from main.cmd
# Same as this command: hares -display $res_name -attribute $attr_name
sub dynupgrade_get_attrvalue_of_resname_array {
    my ($treeref, $res_name, $attr_name)= @_;
    my $branchref= dynupgrade_get_node_from_tree( $treeref, "RES", $res_name);
    my $noderef = dynupgrade_get_node_from_tree ($branchref, "ATTR", $attr_name);
    # return an array.
    return $noderef->{MODIFY}->{ATTRVAL};
}

sub dynupgrade_get_attrvalue_of_resname {
    my $attrs=dynupgrade_get_attrvalue_of_resname_array(@_);
    return $attrs->[0];
}

# Get the resource type of $res_name from main.cmd
# Same as this command: hares -display $res_name -attribute Type
sub dynupgrade_get_restype_of_resname {
    my ($treeref, $res_name)= @_;
    my $branchref= dynupgrade_get_node_from_tree( $treeref, "RES", $res_name);
    return $branchref->{MAIN}->{RESTYPE};
}


# Get the SystemList defined for $grp_name from main.cmd
# Same as this command: hagrp -display $grp_name -attribute SystemList
sub dynupgrade_get_systemlist_of_grpname {
    my ($treeref, $grp_name)= @_;
    my ($sys_ref,@sys_list,$grp_sys,@grp_syslist);

    my $syslist_ref =  dynupgrade_get_branch_from_tree( $treeref, "SYS");
    for $sys_ref (@{$syslist_ref}) {
        push (@sys_list, $sys_ref->{CONTENT}) if ($sys_ref->{SUBCMD} eq "-add");
    }

    my $grp_syslist_ref = dynupgrade_get_attr_from_tree($treeref, "GRP", $grp_name, "SystemList");
    my $grp_syslist = $grp_syslist_ref->{MODIFY}->{ATTRVAL}->[0];
    for $grp_sys (split(/\s+/,$grp_syslist)) {
        push (@grp_syslist, $grp_sys) if (EDR::inarr($grp_sys,@sys_list));
    }
    return @grp_syslist;
}

# Rename the attribute name
sub dynupgrade_rename_attr_name {
    my ($treeref,$type_name,$old_attr_name,$new_attr_name) = @_;
    my ($attrref,$noderef,$res,@reslist);
    @reslist = dynupgrade_get_resname_list_from_typename($treeref,$type_name);
    for $res (@reslist) {
        $noderef=dynupgrade_get_node_from_tree($treeref,"RES",$res);
        $attrref=dynupgrade_get_node_from_tree($noderef,"ATTR",$old_attr_name);
        if (defined $attrref) {
            $attrref->{MODIFY}->{ATTRNAME}=$new_attr_name;
            $noderef->{ATTR}->{$new_attr_name} = $attrref;
            delete ($noderef->{ATTR}->{$old_attr_name});
        }
    }
}

sub dynupgrade_modify_attr_value_DNS {
    my ($treeref) = @_;
    my ($alias,$alias_ref,$dns_resrecord,$hostname,$hostname_ref,$noderef,$res,@reslist,$res_name,$resrecord,$resrecord_ref);
    @reslist = dynupgrade_get_resname_list_from_typename($treeref,"DNS");
    for $res (@reslist) {
        $noderef=dynupgrade_get_node_from_tree($treeref,"RES",$res);
        $alias_ref=dynupgrade_get_node_from_tree($noderef,"ATTR","Alias");
        $hostname_ref=dynupgrade_get_node_from_tree($noderef,"ATTR","Hostname");
        $resrecord_ref=dynupgrade_get_node_from_tree($noderef,"ATTR","ResRecord");
        if (defined $alias_ref && defined $hostname_ref) {
            $alias=$alias_ref->{MODIFY}->{ATTRVAL}->[0];
            $hostname=$hostname_ref->{MODIFY}->{ATTRVAL}->[0];
            $res_name=$noderef->{MAIN}->{RESNAME};
            $resrecord="";
            if (defined $resrecord_ref) {
                $resrecord=$resrecord_ref->{MODIFY}->{ATTRVAL}->[0];
                $resrecord="" if ($resrecord =~ /-delete -keys/);
            }
            $dns_resrecord.="hares -modify $res_name ResRecord $resrecord $alias $hostname\n";
        }
    }
    return $dns_resrecord;
}

sub dynupgrade_modify_attr_value {
    my ($treeref,$type_name,$attr_name,$old_attr_value,$new_attr_value) = @_;
    my ($attrref,$noderef,$res,@reslist);
    @reslist = dynupgrade_get_resname_list_from_typename($treeref,$type_name);
    for $res (@reslist) {
        $noderef=dynupgrade_get_node_from_tree($treeref,"RES",$res);
        $attrref=dynupgrade_get_node_from_tree($noderef,"ATTR",$attr_name);
        if (defined $attrref) {
            $attrref->{MODIFY}->{ATTRVAL}->[0]=$new_attr_value
                if ($attrref->{MODIFY}->{ATTRVAL}->[0] =~ /^(\")?$old_attr_value(\")?$/);
        }
    }
}

sub configure_precheck_sys {
    my ($prod,$sys,$cpic,$running,$gab,$had,$llt,$msg);
    ($prod,$sys) = @_;
    
    $cpic = Obj::cpic();
    $had = $sys->proc("had51");
    $gab = $sys->proc("gab51");
    $llt = $sys->proc("llt51");
    $prod->tsub("check_timesync_sys", $sys, $cpic->{systems}, 5) if($sys->system1 && (EDR::num_sys_keyset("stop_checks")==0));
    if($had->check_sys($sys)) {
        $running = 1;
    } elsif ($gab->check_sys($sys)) {
        $running = 1;
    } elsif ($llt->check_sys($sys)) {
        $running = 1;
    }

    if($running) {
        if($cpic->{prod} ne "SFRAC51") { 
            $msg = Msg::new("VCS is runing on $sys->{sys}. Execute $cpic->{script} -stop before configuration to stop VCS.", 40, 2243, "$sys->{sys}", "$cpic->{script}");
            #EDR::push_error_sys($sys,$msg);
            return "";
        } else {
            return 0;
        }
    }

    return 1;
}

sub start_precheck_sys {
    my ($prod,$sys,$msg,$rtn,@missing_comms_files,$maincf,$sysnum,$onenode,$sysi,$comms_required,$hacf_cmd,$filename);
    ($prod,$sys) = @_;
    @missing_comms_files = ();
    $sysnum = 0;
    $onenode = 0;
    $comms_required = 1;
    $sysi = $sys->{sys};
    # set $cfg->{vcs_allowcomms}
    $prod->tsub("set_vcs_allowcomms_sys",$sys);
    
    # check required config files. 

    #  a. check existence of main.cf in /etc/VRTSvcs/conf/config/
    if (EDR::file_sys($sys,$prod->{maincf})) {
        $maincf = EDR::readfile_sys($sys,$prod->{maincf});
    } else {
        $maincf = undef;
    }
    if (!$maincf){
        $filename = $prod->{maincf};
        $msg = Msg::new("$filename does not exist on $sysi.", 40, 3384, "$filename", "$sysi");
        EDR::push_error_sys($sys,$msg);
        return 0;
    } else {
        $sysnum = grep(/^system/, split (/\n/,$maincf));
    }

    #  b. validate main.cf by 'hacf -verify /etc/VRTSvcs/conf/config'
    $rtn = EDR::cmd_sys($sys, "$prod->{cmd_hacf} -verify $prod->{configdir}");
    if (EDR::cmdexit() != 0) {
        $hacf_cmd = "\'$prod->{cmd_hacf} -verify $prod->{configdir}\'";
        $msg = Msg::new("main.cf is not valid on $sysi:\n\t$rtn\nFix the error(s), and verify the main.cf file before starting VCS by running $hacf_cmd on $sysi", 40, 3385, "$sysi", "$rtn", "$hacf_cmd", "$sysi");
        EDR::push_error_sys($sys,$msg);
        return 0;
    }

    #  c. Prompt for reconfigure if any of /etc/llttab /etc/llthosts or /etc/gabtab is missing and
    #     1) on Linux, ONENODE is not set to yes or not set in /etc/sysconfig/vcs;
    #     2) on AIX/SunOS/HPUX, the number of systems defined in main.cf is greater than one(1).
    if ($sys->{plat} eq "Linux") {
        if (EDR::file_sys($sys,"/etc/sysconfig/vcs")) {
            $onenode = grep (/^\s*ONENODE=yes/, split (/\n/,EDR::readfile_sys($sys,"/etc/sysconfig/vcs")));
        }
        if ( $onenode > 0) {
            $comms_required = 0;
        }
    } else {
        # ONENODE is not usded on AIX/SunOS/HPUX. -onenode will be added as argument of hastart if there is only one system configured.
        if ( $sysnum == 1 ) {
            $comms_required = 0;
        }
    }
    if ($comms_required) {
        if (!EDR::file_sys($sys,$prod->{llttab})){
            push @missing_comms_files, $prod->{llttab};
            $filename = $prod->{llttab};
            $msg = Msg::new("LLT cannot be started on $sysi because $filename does not exist.", 40, 3386, "$sysi", "$filename");
            EDR::push_error_sys($sys,$msg);
        }
        if (!EDR::file_sys($sys,$prod->{llthosts})){
            push @missing_comms_files, $prod->{llthosts};
            $filename = $prod->{llthosts};
            $msg = Msg::new("LLT cannot be started on $sysi because $filename does not exist.", 40, 3386, "$sysi", "$filename");
            EDR::push_error_sys($sys,$msg);
        }
        if (!EDR::file_sys($sys,$prod->{gabtab})){
            push @missing_comms_files, $prod->{gabtab};
            $filename = $prod->{gabtab};
            $msg = Msg::new("GAB cannot be started on $sysi because $filename does not exist.", 40, 3387, "$sysi", "$filename");
            EDR::push_error_sys($sys,$msg);
        }
        if ( scalar @missing_comms_files > 0 ) {
            $msg = Msg::new("HAD cannot be started on $sysi because LLT or GAB cannot be started and VCS is not configured in single-node mode on that system.\nPlease (re)configure VCS.", 40, 3388, "$sysi");
            EDR::push_error_sys($sys,$msg);
            return 0;
        }
    }
    return 1;
}

sub stop_precheck_sys {
    my ($prod,$sys,$cpic);
    ($prod,$sys) = @_;
    $cpic = Obj::cpic();
    return 1;
}

sub install_precheck_sys {
    my ($padv,$prod,$sys,$cpic);
    ($prod,$sys) = @_;
    $cpic=Obj::cpic();
    $prod->tsub("check_timesync_sys", $sys, $cpic->{systems}, 5) if($sys->system1 && (EDR::num_sys_keyset("stop_checks")==0));    
    $prod->checknicconf_sys($sys) if ($prod->can("checknicconf_sys"));
    return 1;
}

sub precheck_sys {
    my($gab,$had,$llt,$msg,$prod,$running,$sys);
    ($prod,$sys) = @_;
    my $cpic = Obj::cpic();
    return 1;
}

sub preremove_tasks {
    my $prod = shift;
    $prod->vxfen_preremove_tasks;
}

sub vxfen_preremove_tasks {
    my (%flag,$clustername,$conf,$cpic,$cpsname,$cpssys,$prod,$sys,$sysname,$vxfenpkg);
    return unless (Cfg::opt("uninstall"));

    $prod = shift;
    $cpic = Obj::cpic();
    $vxfenpkg = Obj::pkg("VRTSvxfen51",$cpic->{padv});
    for $sys(@{$cpic->{systems}}) {
        next unless (EDR::file_sys($sys,$prod->{vxfenmode}));
        $conf = $prod->get_vxfen_config_sys($sys); 
        $sysname = $sys->{hostname};
        next unless (($conf->{vxfen_mechanism} =~ /cps/) && (defined($conf->{cps}{$sysname}))); 
        # set the first node in each cluster
        next unless $prod->get_config_sys($sys);
        $clustername = $sys->{vcs_conf}{clustername};
        next if ($flag{"$clustername"});
        $sys->{system1} = 1;
        $flag{"$clustername"} = 1;
        # create cps sys object in main thread.
        for $cpsname(@{$conf->{cps}{$sysname}}) {
            $cpssys = $vxfenpkg->create_cps_sys($cpsname);
        }
    }
}

sub upgrade_preremove_sys {
    my ($prod,$sys) = @_;
    $prod->backup_vcs_lock_files_sys($sys) if ($prod->can('backup_vcs_lock_files_sys'));
}

sub upgrade_postinstall_sys {
    my ($prod,$sys) = @_;
    $prod->register_extra_types() if($sys->system1);
    $prod->restore_vcs_lock_files_sys($sys) if ($prod->can('restore_vcs_lock_files_sys'));
    $prod->move_NFS_triggers_sys($sys) if ($prod->can('move_NFS_triggers_sys'));
}

# NFS triggers will be deleted from new implementation. The NFS triggers should be moved to sample_triggers directory.
sub move_NFS_triggers_sys{
    my ($prod,$sys) = @_;

    return 1 unless EDR::file_sys($sys, "/opt/VRTSvcs/bin/triggers/nfs_*");
    EDR::cmd_sys($sys,"_cmd_mkdir -p /opt/VRTSvcs/bin/sample_triggers;_cmd_mv -f /opt/VRTSvcs/bin/triggers/nfs_* /opt/VRTSvcs/bin/sample_triggers/");
}

sub postinstall_sys {
    my ($rootpath);
    my ($prod,$sys) = @_;
    my $cpic = Obj::cpic();
    
    # save vcs extra types list, CPI will read it during vcs configuration
    $prod->register_extra_types() if($sys->system1);
    # skip link install if it's not a fresh install for VCS 
    return if (defined(${$sys->{upgradeprod}}[0]) &&
        ${$sys->{upgradeprod}}[0] =~ /^(SFRAC|SVS|SFCFS|SFHA|VCS)/);
    # link install: disable it, if fresh installed
    $prod->vcs_disable_sys($sys);
}

sub startprocs_sys {
    my ($prod,$sys)=@_;
    my ($at,$cfg,$procs,$vxfen);
    $cfg = Obj::cfg();
    return adjust_ru_procs if(Cfg::opt("upgrade_nonkernelpkgs"));    
    $at=$sys->prod("AT50");
    $at->verify_vxatd_start_sys($sys);
    $procs = Prod::startprocs_sys($prod, $sys);
    if ($cfg->{vcs_allowcomms}) {
        if ((!EDR::file_sys($sys,"$prod->{vxfenmode}")) && (!EDR::file_sys($sys,"$prod->{vxfendg}"))) {
            $procs = EDR::arrdel($procs, "vxfen51");
        } else {
            $vxfen = $sys->proc("vxfen51");
            if ($vxfen->check_service_sys($sys)==0) {
                $procs = EDR::arrdel($procs, "vxfen51");
            }
        }
        return $procs;
    } else {
        $procs = EDR::arrdel($procs, "llt51");
        $procs = EDR::arrdel($procs, "gab51");
        $procs = EDR::arrdel($procs, "vxfen51");
        return $procs;
    }
}
  
sub get_extra_types_sys {
    my (@extypes,$pkg,$pkgi,$type);
    my ($prod,$sys) = @_;
    
    if(EDR::file_sys($sys,$prod->{extra_types_cf})) {
        my $type_list = EDR::cmd_sys($sys,"_cmd_cat $prod->{extra_types_cf} 2>/dev/null");
        my @types = split(/\n/,$type_list);
        for $type (@types) {
            next unless($type);
            if(EDR::file_sys($sys,"$prod->{configdir}/$type")) {
                push (@extypes, $type);
            } else {
                Msg::log("Couldn't find $prod->{configdir}/$type on $sys->{sys}")
            }
        }
        return EDR::arruniq(@extypes);
    } else {
        # user might run kickstart/jumpstart/nim or other way to install pkgs without CPI
        $prod->register_extra_types();
        if(EDR::file_sys($sys,$prod->{extra_types_cf})) {
            return $prod->get_extra_types_sys($sys);
        }
        return "";
    }
}

sub register_extra_types {
    my ($conf_dir,$config_dir,$cpic,$padv,$pkg,$pkgi,$prod,$rootpath,$ext,$ext_name,$extype_file,$sys,$sysi);
    $prod = shift;
    $cpic = Obj::cpic();
    my $localsys = Obj::localsys();
    my $tmpfile = EDR::tmpdir()."/extra_types_cf";
    $conf_dir = "/etc/VRTSvcs/conf";
    $config_dir = "/etc/VRTSvcs/conf/config";
    $extype_file= $prod->{extra_types_cf};
    
    if (Cfg::opt("rootpath")) {
        $rootpath = Cfg::opt("rootpath");
        $conf_dir = $rootpath.$conf_dir;
        $config_dir = $rootpath.$config_dir;
        $extype_file = $rootpath.$extype_file;
        
    }
    for $sys (@{$cpic->{systems}}) {
        $padv = $sys->{padv};
        $sysi = $sys->{sys};
        # delete old extra type filelist
        if(EDR::file_sys($sys,$extype_file)) {
            EDR::cmd_sys($sys,"_cmd_rmr $extype_file");
        }
        for $pkgi qw( VRTSvcsea51 VRTSdbac51) {
            $pkg = $Obj::pool{"Pkg\::$pkgi\::$padv"};
            next unless (defined $pkg && $pkg->version_sys($sys,1));        
            for $ext (@{$pkg->{extra_types}}) {
                $ext = $rootpath.$ext if (Cfg::opt("rootpath"));
                # cp extra types.cf to /etc/VRTSvcs/conf/config
                # cp extra types.cf to /etc/VRTSvcs/conf for dynamic upgrade.
                if (EDR::file_sys($sys,$ext)) {
                    EDR::cmd_sys($sys,"_cmd_cp $ext $conf_dir") unless (EDR::dirname($ext) eq $conf_dir);
                    EDR::cmd_sys($sys,"_cmd_cp $ext $config_dir");
                } else {
                    Msg::log("Could not find $ext on $sys->{sys}");
        }
                # tell CPI to incldue these types into main.cf
                $ext_name = EDR::basename($ext);
                EDR::appendfile($ext_name, "$tmpfile.$sysi");
        }
    }
        if(EDR::file_sys($localsys,"$tmpfile.$sysi")) {
            EDR::copy_sys($sys,"$tmpfile.$sysi",$extype_file);
        }
    }
    return 1;
}

sub include_extra_types_sys {
    my ($cmd_scripts,$cpic,$edr,$extype,$extypes,$had,$inctype,@inctypes,$maincf,$new_inc,$prod,$rtn,$sys,$sysi);
    ($prod,$sys) = (@_);
    $edr=Obj::edr();
    $cpic=Obj::cpic();
    if ($sys->system1) {
        $extypes = $prod->get_extra_types_sys($sys);
        if($extypes) {
            # Get the additional extra types to be included.
            $rtn = EDR::cmd_sys($sys, "_cmd_grep '^include' $prod->{maincf} 2> /dev/null");
            for $inctype (split(/\n/,$rtn)) {
                next unless ($inctype =~ /^include\s+(\S+)/);
                $inctype = $1;
                $inctype =~ s/"(.*)"/$1/;
                $inctype = EDR::basename($inctype);
                push (@inctypes,$inctype);
            }
            $extypes = EDR::arrdel($extypes,@inctypes);

            # no additional extra types to include.
            if ($#{$extypes} < 0) {
                EDR::create_flag("add_extypes_done");
                return;
            }

            $had = $sys->proc("had51");
            if ($had->check_sys($sys, "start")) {
                # had is running. Include the extra types by ha commands.
                $prod->haconf_makerw();
                EDR::cmd_sys($sys, "_cmd_mkdir $edr->{tmpdir}/dir_extypes");
                for $extype (@$extypes) {
                    $new_inc .= "\ninclude \"$extype\"";
                    EDR::cmd_sys($sys, "_cmd_cp $prod->{configdir}/$extype $edr->{tmpdir}/dir_extypes");
                }
                EDR::writefile_sys($sys, $new_inc, "$edr->{tmpdir}/dir_extypes/main.cf");
                EDR::cmd_sys($sys, "$prod->{cmd_hacf} -cftocmd $edr->{tmpdir}/dir_extypes");
                $cmd_scripts=EDR::readfile_sys($sys, "$edr->{tmpdir}/dir_extypes/main.cmd");
                $cmd_scripts=~s/\s*(hatype|haattr)/\n$prod->{bindir}\/$1/g;
                EDR::writefile_sys($sys, $cmd_scripts, "$edr->{tmpdir}/dir_extypes/main.sh");
                EDR::cmd_sys($sys, "_cmd_cat $edr->{tmpdir}/dir_extypes/main.sh");
                EDR::cmd_sys($sys, "_cmd_sh $edr->{tmpdir}/dir_extypes/main.sh");
                $prod->haconf_dumpmakero();
            } else {
                # had is not running. Append "include <extra types>" to the end of main.cf
                for $extype (@$extypes) {
                    $new_inc .= "\ninclude \"$extype\"";
                }
                $new_inc .= "\n";
                EDR::cmd_sys($sys, "_cmd_cp $prod->{maincf} $prod->{maincf}.$edr->{uuid}");
                EDR::appendfile_sys($sys,$new_inc,$prod->{maincf});
                $rtn = EDR::cmd_sys($sys, "$prod->{cmd_hacf} -verify $prod->{configdir}");
                if ($rtn) {
                    EDR::cmd_sys($sys, "_cmd_mv $prod->{maincf}.$edr->{uuid} $prod->{maincf}");
                } else {
                    for $sysi (@{$cpic->{systems}}) {
                        next if $sysi->system1;
                        EDR::copy_file($sys, $sysi, $prod->{maincf}, $prod->{maincf});
                    }
                }
            }
            EDR::create_flag("add_extypes_done");
        } else {
            EDR::create_flag("add_extypes_done");
        }
    } else {
        EDR::wait_for_flag("add_extypes_done");
    }
    return;
}

sub default_systemnames {
    my (@llthosts,$prod,$hosts,$host);
    $prod=shift; 
    return "" unless (-f $prod->{llthosts});
    $hosts = EDR::cmd_local("_cmd_cat $prod->{llthosts} 2>/dev/null | _cmd_awk '{print \$2}'");
    foreach $host (split(/\n/,$hosts)) {
        next unless $host;
        push(@llthosts,$host);
    }
    return join(" ", @llthosts);
}

# display preinstallation messages and confirmations
sub cli_preinstall_messages {
    #TODO:add SFRAC check here
    return;
}

sub responsefile_prestart_config {
    my ($prod)=shift;
    my ($j,$cfg,$msg,$rb);
    #initialize RB for installer resilience and responsefile config
    $cfg=Obj::cfg();
    return 1 unless ($cfg->{rootbroker});
    $prod->{rootbroker}=$cfg->{rootbroker};
    $rb=$prod->initial_rootbroker($prod->{rootbroker});
    if (!$rb) {
        $msg=Msg::new("Failed to initialize $prod->{rootbroker} during responsefile configuration. Check the ssh/rsh communication.", 40, 2246, "$prod->{rootbroker}");
        $msg->die;
    }   
}

sub responsefile_poststart_config {
    my $prod =shift;
    my ($cfg,$pkg);
    $cfg=Obj::cfg();
    if ($cfg->{fencingenabled}) {
        $pkg=Obj::pkg("VRTSvxfen51",$prod->{padv});
        $pkg->tsub("config_vxfen");
    }
}
sub responsefile_comments {
    # Each response file comment is a 4 item list
    # item 1 is the comment, previously translated in the prior line
    # item 2 a 0=optional, 1=required
    # item 3 is 0=scalar, 1=list
    # item 4 is 0=1d, 1=2d is System, other=other second dimension
    my ($cfg,$cmt,$cpic,$edr,$email,$pkg,$prod);
    $edr=Obj::edr();
    $cpic=Obj::cpic();
    $cfg=Obj::cfg();
    $cmt=Msg::new("This variable defines the name of the cluster", 40, 2930);
    $edr->{rfc}{vcs_clustername}=[$cmt->{msg},1,0,0];
    $cmt=Msg::new("This variable must be an integer between 0 and 65535 which uniquely identifies the cluster", 40, 2931);
    $edr->{rfc}{vcs_clusterid}=[$cmt->{msg},1,0,0];
    $cmt=Msg::new("This variable defines the NIC to be used for a private heartbeat link on each system.  Two LLT links are required per system (lltlink1 and lltlink2).  Up to four LLT links can be configured", 40, 2932);
    $edr->{rfc}{"vcs_lltlink#"}=[$cmt->{msg},1,0,1];
    $cmt=Msg::new("This variable defines a low-priority heartbeat link.  Typically, low-priority heartbeat link is used on a public network link to provide an additional layer of communication", 40, 3389);
    $edr->{rfc}{"vcs_lltlinklowpri#"}=[$cmt->{msg},0,0,1];
    $cmt=Msg::new("This variable defines the NIC for Cluster Manager (Web Console) to use on a system.  'ALL' can be entered as a system value if the same NIC is used on all systems", 40, 2934);
    $edr->{rfc}{vcs_csgnic}=[$cmt->{msg},0,0,1];
    $cmt=Msg::new("This variable defines the virtual IP address to be used by the Cluster Manager (Web Console)", 40, 2935);
    $edr->{rfc}{vcs_csgvip}=[$cmt->{msg},0,0,0];
    $cmt=Msg::new("This variable defines the Netmask of the virtual IP address to be used by the Cluster Manager (Web Console)", 40, 2936);
    $edr->{rfc}{vcs_csgnetmask}=[$cmt->{msg},0,0,0];
    $cmt=Msg::new("This variable defines the domain-based hostname (example: smtp.yourcompany.com) of the SMTP server to be used for web notification", 40, 2937);
    $edr->{rfc}{vcs_smtpserver}=[$cmt->{msg},0,0,0];
    $email="user\@yourcompany.com";
    $cmt=Msg::new("This variable defines a list of full email addresses (example: $email) of SMTP recipients", 40, 2938, "$email");
    $edr->{rfc}{vcs_smtprecp}=[$cmt->{msg},0,1,0];
    $cmt=Msg::new("This variable defines the minimum severity level of messages (Information, Warning, Error, SevereError) that listed SMTP recipients are to receive", 40, 2939);
    $edr->{rfc}{vcs_smtpserv}=[$cmt->{msg},0,1,0];
    $cmt=Msg::new("This variable defines the SNMP trap daemon port (default=162)", 40, 2940);
    $edr->{rfc}{vcs_snmpport}=[$cmt->{msg},0,0,0];
    $cmt=Msg::new("This variable defines a list of SNMP console system names", 40, 2941);
    $edr->{rfc}{vcs_snmpcons}=[$cmt->{msg},0,1,0];
    $cmt=Msg::new("This variable defines the minimum severity level of messages (Information, Warning, Error, SevereError) that listed SNMP consoles are to receive", 40, 2942);
    $edr->{rfc}{vcs_snmpcsev}=[$cmt->{msg},0,1,0];
    $cmt=Msg::new("This variable defines the NIC for the Virtual IP used for the Global Cluster Option.  'ALL' can be entered as a system value if the same NIC is used on all systems", 40, 2943);
    $edr->{rfc}{vcs_gconic}=[$cmt->{msg},0,0,1];
    $cmt=Msg::new("This variable defines the virtual IP address to be used by the Global Cluster Option", 40, 2944);
    $edr->{rfc}{vcs_gcovip}=[$cmt->{msg},0,0,0];
    $cmt=Msg::new("This variable defines the Netmask of the virtual IP address to be used by the Global Cluster Option)", 40, 2945);
    $edr->{rfc}{vcs_gconetmask}=[$cmt->{msg},0,0,0];
    $cmt=Msg::new("This variable defines the security of the Global Cluster Option(0=unsecure,1=secure)", 40, 2946);
    $edr->{rfc}{vcs_securegco}=[$cmt->{msg},0,0,0];
    $cmt=Msg::new("This variable defines a list of VCS usernames", 40, 2947);
    $edr->{rfc}{vcs_username}=[$cmt->{msg},0,1,0];
    $cmt=Msg::new("This variable defines each user's VCS privileges", 40, 2948);
    $edr->{rfc}{vcs_userpriv}=[$cmt->{msg},0,1,0];
    $cmt=Msg::new("This variable defines an encrypted password for each VCS user", 40, 2949);
    $edr->{rfc}{vcs_userenpw}=[$cmt->{msg},0,1,0];
    $prod=$cpic->prod;
    $cmt=Msg::new("This variable defines whether a single node $prod->{abbr} configuration should start GAB and LLT or not.", 40, 3024, "$prod->{abbr}");
    $edr->{rfc}{vcs_allowcomms}=[$cmt->{msg},1,0,0]; 
    $cmt=Msg::new("This variable defines the name of the system where the root broker is installed", 40, 2950);
    $edr->{rfc}{at_rootdomain}=[$cmt->{msg},0,1,0]; 
    #add new global variable for AT configuration
    $cmt=Msg::new("This variable defines mode to configure security cluster.(1, 2, 3)", 40, 2951);
    $edr->{rfc}{vcs_securitymenuopt}=[$cmt->{msg},0,0,0];
    $cmt=Msg::new("This variable defines location of blob file", 40, 2952);
    $edr->{rfc}{vcs_blobpath}=[$cmt->{msg},0,0,1];
    $cmt=Msg::new("This variable defines root broker port", 40, 2953);
    $edr->{rfc}{vcs_vssdefport}=[$cmt->{msg},0,0,0];
    $cmt=Msg::new("This variable defines location of root hash file", 40, 2954);
    $edr->{rfc}{vcs_roothashpath}=[$cmt->{msg},0,0,0];
    $cmt=Msg::new("This variable defines authentication broker's principle name", 40, 2955);
    $edr->{rfc}{vcs_ab_prplname}=[$cmt->{msg},0,0,1];
    $cmt=Msg::new("This variable defines authentication broker's password", 40, 2956);
    $edr->{rfc}{vcs_ab_password}=[$cmt->{msg},0,0,1];
    $cmt=Msg::new("This variable defines fencing as enabled", 40, 3390);
    $edr->{rfc}{fencingenabled}=[$cmt->{msg},1,0,0];
    if ($cfg->{fencingenabled}) {
        $pkg=Obj::pkg("VRTSvxfen51",$prod->{padv});
        $pkg->tsub("responsefile_comments_for_fencing");
    }
}

sub prestart_config_common_questions {
    my ($ayn,$cfg,$cpic,$msg,$prod,$rel,$rtn);
    $prod=shift;
    $cpic = Obj::cpic();
    $cfg = Obj::cfg();

    Msg::log("start VCS prestart_config_questions\n");
    $cfg->{vcs_allowcomms} = 1;
    if ((($cpic->{prod} eq "VCS51")||($cpic->{prod} eq "SFHA51")) && (!$cfg->{fencingenabled})) {
        if ($cpic->nsystems == 1) {
            my $addmsg = Msg::new("If you plan to run $prod->{abbr} on a single node without any need for adding cluster node online, you have an option to proceed without starting GAB and LLT. Starting GAB and LLT is recommended.", 40, 2244, "$prod->{abbr}");
            $addmsg->bold;
            Msg::n();
            $msg = Msg::new("Do you want to start GAB and LLT?", 40, 2245);
            $rtn = $msg->ayny_web('','',$addmsg);
            Msg::n();
            if ($rtn eq "N") {
                $cfg->{vcs_allowcomms} = 0;
            }       
        }
    }

    # check uuid, if uuid is not configured on systems, then it means that 5.1SP1 is not configured before.
    if ($prod->check_config() && $prod->check_uuid()) {
        $rel=Obj::rel($prod->{release}, $prod->{padv});
        $msg=Msg::new("Installer detects that $prod->{abbr} is already configured. If you re-configure $prod->{abbr}, you will lose all the current configurations in main.cf. If you do not re-configure $prod->{abbr}, please make sure all the current configurations are compatible with $rel->{titlevers}.\n\nDo you want to re-configure $prod->{abbr}?", 40, 3391, "$prod->{abbr}", "$prod->{abbr}", "$prod->{abbr}", "$rel->{titlevers}", "$prod->{abbr}");
        $ayn = $msg->aynn_web;
        Msg::n();
        if ($ayn eq "N") {
            $cfg->{donotreconfigurevcs}=1;
            # link install: enable vcs 
            for my $sys(@{$cpic->{systems}}) {
                $prod->vcs_enable_sys($sys);
            }
            return 0;
        }
    }
    Msg::title();
    $msg=Msg::new("To configure $prod->{abbr}, answer the set of questions on the next screen.\n", 40, 2248, "$prod->{abbr}");
    $msg->print;
    $msg=Msg::new("When [b] is presented after a question, 'b' may be entered to go back to the first question of the configuration set.\n", 40, 2249);
    $msg->print;
    $msg=Msg::new("When [?] is presented after a question, '?' may be entered for help or additional information about the question.\n", 40, 2250);
    $msg->print;
    $msg=Msg::new("Following each set of questions, the information you have entered will be presented for confirmation.  To repeat the set of questions and correct any previous errors, enter 'n' at the confirmation prompt.\n", 40, 2251);
    $msg->print;
    if (Cfg::opt("configure")) {
        $msg = Msg::new("No configuration changes are made to the systems until all configuration questions are completed and confirmed.", 40, 2252);
        $msg->print;
    } else {
        $msg = Msg::new("No configuration changes are made to the systems until all configuration questions are completed and $prod->{abbr} is installed successfully.", 40, 2253, "$prod->{abbr}");
        $msg->print;
    }
    Msg::prtc();
    return 1;
}

sub cli_prestart_config_questions {
    my ($cpic,$prod);
    $prod=shift;
    $cpic = Obj::cpic();
    if ($cpic->{prod}!~/^(SFCFSRAC|SFRAC)\d+/) {
        $prod->tsub("ask_fencing_enabled") unless (Cfg::opt("makeresponsefile"));
    }

    return 1 unless($prod->prestart_config_common_questions);
    $prod->tsub("config_cluster");
    return 1 if ($cpic->{prod}=~/SFCFS\d+/);
    if (Cfg::opt("demo")) {
        #default values for demo 
        my $cfg=Obj::cfg();
        $cfg->{vcs_userenpw}=[ qw(IpqIpkPmqLqqOyqKpn) ];
        $cfg->{vcs_username}=[ qw(admin) ];
        $cfg->{vcs_userpriv}=[ qw(Administrators) ];
        return 1; 
    }
    $prod->tsub("config_vip");
    $prod->tsub("config_vxss");
    $prod->tsub("add_users");
    $prod->tsub("config_smtp");
    $prod->tsub("config_snmp");
    $prod->tsub("config_gcoption");      
}

sub web_autocfg_hbnics {
    my ($msg);
    my $prod = shift;
    my $edr = Obj::edr();
    my $web = Obj::web();
    my $cfg = Obj::cfg();
    
    my $rhbn = $prod->tsub("autocfg_hbnics");
    if ($prod->num_hbnics($rhbn)< 1) {
        # auto detect failed
        $msg=Msg::new("Failed to detect and configure LLT heartbeat links. Please configure LLT manually.", 40, 3392);
        $web->tsub("web_script_form","alert",$msg->{msg});
    } else {
        my $notice;
        for my $sysi (@{$edr->{systems}}) {
            my $sys = $sysi->{sys};
            my $lmsg="";
            for my $n (1..$prod->num_hbnics($rhbn)) {
                my $key="lltlink$n";
                next unless ($rhbn->{$key}{$sys});
                $lmsg.="\nlink$n=$rhbn->{$key}{$sys}";
            }
            $notice .= Msg::new("Private Heartbeat NICs for $sys: $lmsg", 40, 3393, "$sys", "$lmsg")->{msg};
            $notice .= "\n";
            
            if ($prod->num_lopri_hbnics($rhbn) > 0) {
                $lmsg = "";
                for my $n (1..$prod->num_lopri_hbnics($rhbn)) {
                    my $key="lltlinklowpri$n";
                    $lmsg.="\nlink-lowpri$n=$rhbn->{$key}{$sys}";
                }
                if ($prod->num_lopri_hbnics($rhbn) > 1) {
                    $notice .= Msg::new("Low-Priority Heartbeat NICs for $sys: $lmsg", 40, 3394, "$sys", "$lmsg")->{msg};
                } else {
                    $notice .= Msg::new("Low-Priority Heartbeat NIC for $sys: $lmsg", 40, 3395, "$sys", "$lmsg")->{msg};
                }
                $notice .= "\n";
            }
        }
        $msg = Msg::new("Do you want to use this heartbeat configuration?", 40, 3396);
        my $answer = $msg->aynn_web("","",$notice);
        if ($answer eq "Y") {
            $web->{rhbn} = $rhbn;
            return 1;
        }
    }
    return 0;
}

sub web_prestart_config_questions {
    my $prod = shift;
    my $cfg = Obj::cfg();
    my $cpic = Obj::cpic();
    my $web = Obj::web();
    my $edr = Obj::edr();
    my ($rtvalue);

    $web->{vxss_enabled} = 0;
    if ($cpic->{prod}!~/^(SFCFSRAC|SFRAC)\d+/) {
        $prod->tsub("ask_fencing_enabled") unless (Cfg::opt("makeresponsefile"));
    }

    return 1 unless($prod->prestart_config_common_questions);
    
    while (1) {
        $rtvalue=$web->tsub("web_script_form", "vcs_select_cluster");
        if ($cfg->{autocfgllt}) {
            if (!$prod->tsub("web_autocfg_hbnics")) {
                $cfg->{autocfgllt} = 0;
                next;
            }
        } else {
            my $unique = $web->{submit}{uniqueNicsPerSys};
            for my $sys (@{$edr->{systems}}) {
                my $padv = $sys->padv;
                my $nics = $padv->tsub("systemnics_sys", $sys,1);
                $nics = [qw(NIC1 NIC2 NIC3 NIC4)] if Cfg::opt("demo");
                $rtvalue=$web->tsub("web_script_form", "vcs_select_heartbeats", "configure", $nics, $sys);
                last if($rtvalue eq "back");
                last unless $unique;
            }
            next if($rtvalue eq "back");
            my $diff = $prod->check_link_speed($edr->{systems},$web->{rhbn});
            next if (!$prod->web_check_link_speed($diff));
        }
            
        # skip VCS optional configuraition with demo option
        unless (Cfg::opt("demo") || $cpic->{prod}=~/^SFCFS\d+/) {
            # security cluster
            $rtvalue = $prod->tsub("web_config_vxss");
            next if ($rtvalue eq "back");
            $rtvalue=$web->tsub("web_script_form", "vcs_optional");
            next if($rtvalue eq "back");
        }

        $prod->set_hb_nics($web->{rhbn},$edr->{systems});
        # delete this hash key so that it do not show in responsefile
        delete $cfg->{autocfgllt};
        last; # done
    }
}

sub cli_poststart_config_questions {
    my ($prod) = @_;
    my ($cfg,$pkg);
    $cfg=Obj::cfg();
    if ($cfg->{fencingenabled}) {
        $pkg=Obj::pkg("VRTSvxfen51",$prod->{padv});
        $pkg->tsub("config_vxfen");
    }
}

sub web_poststart_config_questions {
    my ($prod) = @_;
    my ($cfg,$pkg);
    $cfg=Obj::cfg();
    if ($cfg->{fencingenabled}) {
        $pkg=Obj::pkg("VRTSvxfen51",$prod->{padv});
        $pkg->tsub("config_vxfen");
    }
}

sub licensed_sys {
    my ($prod,$sys) = @_;
    my $cpic = Obj::cpic();
    
    $prod->is_features_licensed_sys($sys);
    return $cpic->tsub("prod_licensed_sys", $sys);
}

sub is_features_licensed_sys {
    my ($prod,$sys) = @_;
    my $cpic = Obj::cpic();
    if ($cpic->tsub("feature_licensed_sys", $sys, "Global Cluster Option")) {
        Cfg::set_opt("gco");
    }
}


# if a nic is configured with IPv6 and the input IP addres is IPv6 then Set Protocol=IPv6, return 0;
sub nic_ipv6_sys {
    my ($cpic,$padv,$ips,$ip,$gcoip,$hasipv6,$cfg);
    my ($prod,$sys,$nic) = @_;
    $cpic = Obj::cpic();
    $cfg=Obj::cfg();
    $padv=$sys->{padv};
    if($gcoip){
        $gcoip="$cfg->{vcs_gcovip}";
    } else {
        $gcoip="$cfg->{vcs_csgvip}";
    }
        
    $ips=$padv->tsub("nic_ips_sys",$sys,$nic);
    foreach $ip (@{$ips}) {
        $hasipv6=1 if(EDR::ip_is_ipv6($ip)); 
    }
    return 1 if($hasipv6 && EDR::ip_is_ipv6($gcoip));
    return 0;
}


#Protocol @sys1=IPv6
#Protocol @sys2=IPv6
sub nic_protocol_attr {
    my ($prod,$gco) = @_;
    my ($edr,$sys,$sysname,$ipv6,$attr,$nic,$cfg);
    return if(EDR::plat($prod->{padv}) eq "Linux");
    $nic="vcs_csgnic";
    $nic="vcs_gconic" if ($gco);
    $edr=Obj::edr();
    $cfg=Obj::cfg();
    foreach $sys(@{$edr->{systems}}) {
        $sysname =transform_system_name($sys->{sys});
        if ($cfg->{$nic}{all}) {
            $ipv6=$prod->tsub("nic_ipv6_sys",$sys,$cfg->{$nic}{all},$gco); 
        } elsif($cfg->{$nic}{$sys->{sys}})  {
            $ipv6=$prod->tsub("nic_ipv6_sys",$sys,$cfg->{$nic}{$sys->{sys}},$gco);
        }
        $attr .="Protocol \@$sysname=IPv6\n" if ($ipv6);
    }
    return $attr;
}

sub ask_nhip {
    my ($prod,$sys)=@_;
    my ($answer,$backopt,$edr,$done,$msg,$help);
    
    $done=0;
    $backopt=1;
    $edr=Obj::edr();
	while (!$done) {
		$help=Msg::new("List of hosts on the network to which the agent pings to determine if the network connection is alive.", 40, 3397);
	    $msg=Msg::new("Enter the NetworkHosts IP addresses, separated by spaces:", 40, 3398);
		$answer=$msg->ask("",$help,$backopt);
        return $answer if ($answer eq $edr->{msg}{back});
		if (!$prod->validate_nwhosts($answer)) {
            $msg=Msg::new("$answer contains invalid IP address", 40, 3399, "$answer");
            $msg->print;
            next;
        }
		$done=1;
    }
    return $answer;
}

# Configure the Virtual IP
sub config_vip {
    my ($prod) = @_;
    my $edr = Obj::edr();
    my $cfg = Obj::cfg();
    return "" if (Cfg::opt("responsefile"));
    my($ayn,$cmsg,$done,$msg,$msg0,$help,$backopt,$netm,$vipnic,$vip,$prefix,$nwhosts,$nonhip,$nhip);
    while (!$done) {
        $backopt = 0;
        undef($nwhosts);
        Msg::title();
        $msg = Msg::new("Virtual IP can be specified in RemoteGroup resource, and can be used to connect to the cluster using Java GUI", 40, 2254);
        $help = $msg;
        $msg->bold;
        $msg = Msg::new("The following data is required to configure the Virtual IP of the Cluster:\n", 40, 2255);
        $msg->bold;
        $msg = Msg::new("\tA public NIC used by each system in the cluster", 40, 2256);
        $msg->print;
        $msg = Msg::new("\tA Virtual IP address and netmask", 40, 3400);
        $msg->print;
        if (EDR::plat($prod->{padv}) eq "HPUX") {
            $msg=Msg::new("\tOne or more NetworkHosts IP addresses for connection checking\n", 40, 3401);
            $msg->print;
            $nonhip=1;
        } else {
            Msg::n();
        }
        $msg = Msg::new("Do you want to configure the Virtual IP?", 40, 2258);
        $ayn = $msg->aynn($help,$backopt);
        return if ($ayn eq "N");

        $vipnic = $prod->ask_publicnic;
        next if ($vipnic eq $edr->{msg}{back});
        $vip = $prod->ask_vip;
        next if ($vip eq $edr->{msg}{back});
        $netm = $prod->ask_netmask($vip,$$vipnic{${$cfg->{systems}}[0]});
        next if ($netm eq $edr->{msg}{back});
        # AIX virtual nic specific configuration
        if ($prod->has_virtual_nic()) {
            $nwhosts=$prod->ask_networkhosts_csg($vipnic);
            next if ($nwhosts eq $edr->{msg}{back});
        }
        # HP specific network host configuration
        $nhip=$prod->ask_nhip() if ($nonhip);
        next if ($nhip eq $edr->{msg}{back}); 
        
        Msg::title();
        if (ip_is_ipv6($vip)) {
            $prefix=Msg::new("Prefix", 40, 2259);
        } else {
            $prefix=Msg::new("NetMask", 40, 2260);
        }
        $msg = Msg::new("Cluster Virtual IP verification:\n", 40, 2261);
        $msg->bold;
        $msg0 = $prod->display_csgnic($vipnic);
        $msg = Msg::new("\tIP: $vip\n", 40, 2262, "$vip");
        $msg0 .= $msg->{msg};
        $msg = Msg::new("\t$prefix->{msg}: $netm\n", 40, 2263, "$prefix->{msg}", "$netm");
        $msg0 .= $msg->{msg};
        $msg0.=$prod->display_csgnwhosts($nwhosts) if ($nwhosts);
        if ($nhip) {
            $msg = Msg::new("\tNetworkHosts: $nhip\n", 40, 3402, "$nhip");
            $msg0 .= $msg->{msg};
        }        
        Msg::print($msg0);
        $backopt = 0;
        $msg = Msg::new("Is this information correct?", 40, 1102);
        $ayn = $msg->ayny;
        $done = 1 if ($ayn eq "Y");
    }
    $prod->store_nic($vipnic,"vcs_csgnic");
    $cfg->{vcs_csgvip} = $vip;
    $cfg->{vcs_csgnetmask} = $netm;
    $cfg->{vcs_networkhosts} = $nhip if ($nhip);
    $prod->store_nwhosts_csg($nwhosts) if ($nwhosts);
    $msg = Msg::new("Cluster Virtual IP configuration:\n", 40, 2264);
    $cmsg = $msg->{msg};
    push(@{$prod->{configmsg}},$cmsg,$msg0)
}



# Configure the Global Cluster Option
# Copies CSGVIP most likely if configured
sub config_gcoption {
    my ($prod) = @_;    
    my $edr = Obj::edr();
    my $cfg = Obj::cfg();
    return "" unless (Cfg::opt("gco"));
    return "" if (Cfg::opt("responsefile"));

    my($ayn,$cmsg,$done,$msg,$msg0,$help,$backopt,$netm,$rgconic,$vip,$prefix,$nwhosts,$sys0,$nonhip,$nhip);
    
    while (!$done) {
        $backopt = 0;
        undef($nwhosts);
        Msg::title();
        $msg = Msg::new("The following data is required to configure the Global Cluster Option:\n", 40, 2265);
        $msg->bold;
        $msg = Msg::new("\tA public NIC used by each system in the cluster", 40, 2256);
        $msg->print;
        $msg = Msg::new("\tA Virtual IP address and netmask", 40, 3400);
        $msg->print;
        if ((EDR::plat($prod->{padv}) eq "HPUX") && !$cfg->{vcs_csgvip} && !$cfg->{vcs_smtpserver} && !$cfg->{vcs_snmpport}) {
            $msg=Msg::new("\tOne or more NetworkHosts IP addresses for connection checking\n", 40, 3401);
            $msg->print;
            $nonhip=1;
        } else {
            Msg::n();
        } 
        
        $help = Msg::new("If Global Cluster Option is configured, then the connection between the global clusters will be available to the user.", 40, 2267);
        $msg = Msg::new("Do you want to configure the Global Cluster Option?", 40, 2268);
        $ayn = $msg->aynn($help,$backopt);
        Msg::n();
        return if ($ayn eq "N");
        
        Msg::n();
        $backopt = 1;
        if ($cfg->{vcs_csgvip}) {
            $vip = $prod->ask_vip($cfg->{vcs_csgvip},1);
            next if ($vip eq $edr->{msg}{back});
            if ($vip eq $cfg->{vcs_csgvip}) {
                $rgconic = $prod->{rcsgnic};
                $netm = $cfg->{vcs_csgnetmask};
            } else {
                $rgconic = $prod->ask_publicnic(2);
                next if ($rgconic eq $edr->{msg}{back});
                $netm = $prod->ask_netmask($vip,$$rgconic{${$cfg->{systems}}[0]});
                next if ($netm eq $edr->{msg}{back});
            }
        } else {
            $rgconic = $prod->ask_publicnic(2);
            next if ($rgconic eq $edr->{msg}{back});
            $vip = $prod->ask_vip("",1);
            next if ($vip eq $edr->{msg}{back});
            $netm = $prod->ask_netmask($vip,$$rgconic{${$cfg->{systems}}[0]});
            next if ($netm eq $edr->{msg}{back});
        }
        # AIX virtual nic specific configuration
        if ($prod->has_virtual_nic()) {
            $nwhosts=$prod->ask_networkhosts_gco($rgconic);
            next if ($nwhosts eq $edr->{msg}{back});
        }
        # HP specific network host configuration
        $nhip=$prod->ask_nhip() if ($nonhip);
        next if ($nhip eq $edr->{msg}{back});
        
        Msg::title();
        if ( ip_is_ipv6($vip)) {
            $prefix=Msg::new("Prefix", 40, 2259);
        } else {
            $prefix=Msg::new("NetMask", 40, 2260);
        }
        $msg = Msg::new("Global Cluster Option configuration verification:\n", 40, 2269);
        $msg->bold;
        $msg0 = $prod->display_csgnic($rgconic);
        $msg = Msg::new("\tIP: $vip\n", 40, 2262, "$vip");
        $msg0 .= $msg->{msg};
        $msg = Msg::new("\t$prefix->{msg}: $netm\n", 40, 2263, "$prefix->{msg}", "$netm");
        $msg0 .= $msg->{msg};
        if ($nwhosts) {
            $sys0 = ${$edr->{systems}}[0];
            $msg = Msg::new("\tNetworkhosts: $nwhosts->{$sys0->{sys}}\n", 40, 2270, "$nwhosts->{$sys0->{sys}}");
            $msg0 .= $msg->{msg};
        }
        if ($nhip) {
            $msg = Msg::new("\tNetworkHosts: $nhip\n", 40, 3402, "$nhip");
            $msg0 .= $msg->{msg};
        }
        Msg::print($msg0);

        if ($vip eq $cfg->{vcs_csgvip}) {
            $msg = Msg::new("Matching the Cluster Virtual IP configuration\n", 40, 2271);
            $msg->print;
        }
        $backopt = 0;
        $msg = Msg::new("Is this information correct?", 40, 1102);
        $ayn = $msg->ayny;
        $done = 1 if ($ayn eq "Y");
    }

    $prod->store_nic($rgconic,"vcs_gconic");
    $cfg->{vcs_gcovip} = $vip;
    $cfg->{vcs_gconetmask} = $netm;
    $cfg->{vcs_gconwhosts} = $nwhosts->{$sys0->{sys}} if ($nwhosts);
    $cfg->{vcs_networkhosts} = $nhip if ($nhip);
    $msg = Msg::new("Global Cluster Option configuration:\n", 40, 2272);
    $cmsg = $msg->{msg};
    push(@{$prod->{configmsg}},$cmsg,$msg0);
}

sub store_nwhosts_csg {
    my ($prod,$rnwhosts) = @_;
    my ($sys,$cpic,$sys0,$cfg);
    
    $cpic = Obj::cpic();
    $cfg = Obj::cfg();
    for $sys (@{$cpic->{systems}}) { 
        $cfg->{vcs_csgnwhosts}{$sys->{sys}} = $rnwhosts->{$sys->{sys}};
    }
}

sub has_virtual_nic {
    my ($prod)=@_;
    my ($cpic,$sys,$ret,$padv);
    #Only Aix has sub virtualnics_sys for now.
    $padv = Obj::padv($prod->{padv});
    return 0 unless $padv->can("virtualnics_sys");
    $cpic=Obj::cpic();
    for $sys (@{$cpic->{systems}}) { 
        $sys->{vionics}=$sys->padv->virtualnics_sys($sys);
        $ret||=1 if ($sys->{vionics});
    }
    return $ret;
}

sub ask_networkhosts_gco {
    my ($prod,$rnic)=@_;
    my ($sys,$cpic,$sys0,%nwhosts,$msg,$help,$backopt,$nic,$done,$answer,$edr);
    
    return 0 if (!$rnic);
    $cpic=Obj::cpic();
    $edr=Obj::edr();
    $backopt=1;    
    $sys0=${$cpic->{systems}}[0];        
        
    for $sys (@{$cpic->{systems}}) {
        $nic=$$rnic{$sys->{sys}};
        if (EDR::inarr($nic, @{$sys->{vionics}})) {
            while (!$done) {
                $msg=Msg::new("Enter the NetworkHosts for the Global Cluster Option :", 40, 2273);
                $help=Msg::new("You have a virtual device for your Global Cluster Option, hence NetworkHosts attribute is needed for a virtual device.", 40, 2274);
                $answer=$msg->ask("",$help,$backopt);
                return $answer if ($answer eq $edr->{msg}{back});
                if ($prod->validate_nwhosts($answer)) {
                    $done=1;
                    $nwhosts{$sys0->{sys}}=$answer;
                } else {
                    $msg=Msg::new("The NetworkHosts, $answer, is not a vaild ip address", 40, 2275, "$answer");
                    $msg->print;
                }
            } 
        }
        last if ($nwhosts{$sys0->{sys}}); 
    }   
    return \%nwhosts;
}

sub ask_networkhosts_csg {
    my ($prod,$rnic)=@_;
    my ($aayn,$sys,$cpic,$sys0,%nwhosts,$msg,$help,$backopt,$nic,$all,$answer,$edr,$firstanswer,$done);
    
    return 0 if (!$rnic);
    $cpic=Obj::cpic();
    $edr=Obj::edr();
    $backopt=1;    
    $sys0=${$cpic->{systems}}[0];        
    $firstanswer=1;    
    for $sys (@{$cpic->{systems}}) {
        $nic=$$rnic{$sys->{sys}};
        if ($all) {
            if (EDR::inarr($nic, @{$sys->{vionics}})) {
                $nwhosts{$sys->{sys}}=$answer;
                next;
            }
        } 
        if (EDR::inarr($nic, @{$sys->{vionics}})) {
            $done=0;
            while (!$done) {
                $msg=Msg::new("$nic on $sys->{sys} is a virtual device. Enter its NetworkHosts:", 40, 2276, "$nic", "$sys->{sys}");
                $help=Msg::new("You must configure the NetworkHosts attribute for a virtual device. NetworkHosts is a list of hosts on the network that are pingable to determine if the network connection is alive. Enter the IP address of the host instead of the host name to prevent the monitor from timing out.", 40, 2277);
                $answer=$msg->ask("",$help,$backopt);
                return $answer if ($answer eq $edr->{msg}{back});
                if ($prod->validate_nwhosts($answer)) {
                    $done=1;
                    $nwhosts{$sys->{sys}}=$answer;
                    if ($firstanswer==1) {
                        $firstanswer=0;
                        $msg=Msg::new("Do you want to apply the same NetworkHosts ($answer) for all systems?", 40, 2278, "$answer");
                        $help=Msg::new("The NetworkHosts will only be applied on systems which have virtual nics", 40, 2279);
                        $aayn=$msg->ayny($help,$backopt);
                        return $aayn if ($aayn eq $edr->{msg}{back});
                        $all=1 if ($aayn eq "Y");
                    }
                } else {
                    $msg=Msg::new("The NetworkHosts ($answer) is not a vaild ip address", 40, 2280, "$answer");
                    $msg->print;
                }
            } 
        }
    }   
    return \%nwhosts;
}

sub validate_nwhosts {
    my ($prod,$nwhosts)=@_;
    my ($ip);
    
    for $ip (split(" ",$nwhosts)) {
        return 0 if (!EDR::isip($ip));
    }
    return 1;
}
# ask for the netmask on a public nic
sub ask_netmask {
    my ($prod,$vip,$nic) = @_;
    my ($msg,$help,$question,$answer,$edr,$cfg,$def,$netm,$backopt,$prefix);

    $edr = Obj::edr();
    $cfg = Obj::cfg();
    $backopt = 1;
    $def = EDR::defaultnetmask_sys(${$edr->{systems}}[0],$vip,$nic);
    if(ip_is_ipv6($vip)) {
        $prefix=Msg::new("Prefix", 40, 2259);
    } else {
        $prefix=Msg::new("NetMask", 40, 2260);
    }
    $help = Msg::new("The $prefix->{msg} that is configured for the physical address of this NIC (default selection) is typically used for logical addresses configured on the same NIC", 40, 2281, "$prefix->{msg}");
    $question = Msg::new("Enter the $prefix->{msg} for IP $vip:", 40, 2282, "$prefix->{msg}", "$vip");
    while (1) {
        $netm = $question->ask($def,$help,$backopt);
        return $netm if ($netm eq $edr->{msg}{back});
        if (ip_is_ipv6($vip)){
            unless ((EDR::isint($netm) == 1) && (($netm > 0 && $netm <= 128) || ($netm == 0 && (EDR::plat($prod->{padv}) =~ /^Linux|^SunOS/)))){ 
                $msg = Msg::new("$netm is not a valid prefix", 40, 2283, "$netm");
                $msg->print;
                next;
            }
        } else {       
            if (!EDR::isip($netm)) {
                $msg = Msg::new("$netm is not a valid netmask", 40, 2284, "$netm");
                $msg->print;
                next;
            }
        }
        return $netm;
    }
}

# ask for a virtual ip for cluster manager or the global cluster option
sub ask_vip {
    my ($prod,$def,$gco) = @_;
    my ($answer,$help,$msg,$edr,$backopt,$question);

    $backopt = 1;
    $edr = Obj::edr();
    $help = Msg::new("A virtual address is an IP address that is configured on a public NIC interface on one system in a cluster at any given time to perform specific services that must remain highly available.", 40, 2285);
    if($gco){
        $question = Msg::new("Enter the Virtual IP address for the Global Cluster Option:", 40, 2286);
    }else{    
        $question = Msg::new("Enter the Virtual IP address for the Cluster:", 40, 2287);
    }
    while(1) {
        $answer = $question->ask($def,$help,$backopt);
        return $answer if ($answer eq $edr->{msg}{back});
        if (!EDR::isip($answer)) {
            $msg = Msg::new("$answer is not a valid IP address", 40, 1509, "$answer");
            $msg->print;
        } else {
            return $answer;
        }
    }
}

sub get_severity_msgs {
    my ($msg, $prod, $sev_msgs);

    $prod = shift;
    $msg = Msg::new("Information", 40, 3711);
    $sev_msgs->{"Information"} = $msg->msg;
    $msg = Msg::new("Warning", 40, 3712);
    $sev_msgs->{"Warning"} = $msg->msg;
    $msg = Msg::new("Error", 40, 1894);
    $sev_msgs->{"Error"} = $msg->msg;
    $msg = Msg::new("SevereError", 40, 3713);
    $sev_msgs->{"SevereError"} = $msg->msg;
    return $sev_msgs;    
}

# configure snmp notification:
# the snmp trap port, snmp consoles, and severity for each
sub config_snmp {
    my ($prod) = @_;
    my ($edr,$cfg,@conss,@sevs,$ayn,$cmsg,$cons,$done,$msg,$msg0,$help,$n,$backopt,$port,$rcsgnic,$sev,$nwhosts,$nonhip,$nhip);
    return if (Cfg::opt(qw(installonly responsefile)));

    $edr = Obj::edr();
    $cfg = Obj::cfg();
    my $sev_msgs = $prod->get_severity_msgs;
    while (!$done) {
        undef(@conss);
        undef(@sevs);
        undef($nwhosts);
        Msg::title();
        $msg = Msg::new("The following information is required to configure SNMP notification:\n", 40, 2288);
        $msg->bold;
        $msg = Msg::new("\tSystem names of SNMP consoles to receive VCS trap messages", 40, 2289);
        $msg->print;
        $msg = Msg::new("\tSNMP trap daemon port numbers for each console", 40, 2290);
        $msg->print;
        $msg = Msg::new("\tA minimum severity level of messages to send to each console", 40, 2291);
        $msg->print;
        if ((EDR::plat($prod->{padv}) eq "HPUX") && !$cfg->{vcs_csgvip} && !$cfg->{vcs_smtpserver}) {
            $msg=Msg::new("\tOne or more NetworkHosts IP addresses for connection checking\n", 40, 3401);
            $msg->print;
            $nonhip=1;
        } else {
            Msg::n();
        } 
        
        $help = Msg::new("If SNMP is configured, VCS trap messages can be delivered to various systems", 40, 2292);
        $msg = Msg::new("Do you want to configure SNMP notification?", 40, 2293);
        $ayn = $msg->aynn($help,0); 
        Msg::n();
        return if ($ayn eq "N");

        Msg::n();
        $backopt = 1;
        $rcsgnic = $prod->ask_publicnic(1);
        next if ($rcsgnic eq $edr->{msg}{back});
        
        # AIX virtual nic specific configuration
        if ($prod->has_virtual_nic()) {
            if (!$prod->has_same_csg($rcsgnic)) {
                $nwhosts=$prod->ask_networkhosts_csg($rcsgnic);
                next if ($nwhosts eq $edr->{msg}{back});                
            }
        }
         # HP specific network host configuration
        $nhip=$prod->ask_nhip() if ($nonhip);
        next if ($nhip eq $edr->{msg}{back});
        
        $port = $prod->ask_snmpport;
        next if ($port eq $edr->{msg}{back});
        
        while (($#conss<0) || ($ayn eq "Y")) {
            $cons = lc($prod->ask_snmpconsole);
            last if ($cons eq $edr->{msg}{back});
            $sev = $prod->ask_severity($cons,1);
            last if ($sev eq $edr->{msg}{back});
            
            $n = EDR::arrpos($cons,@conss);
            if ($n>=0) {
                $msg = Msg::new("$cons was previously entered. Changing priority to $sev_msgs->{$sev}.", 40, 2294, "$cons", "$sev_msgs->{$sev}");
                $msg->print;
                $sevs[$n] = $sev;
            } else {
                push(@conss,$cons);
                push(@sevs,$sev);
            }
            
            $msg = Msg::new("Would you like to add another SNMP console?", 40, 2295);
            $ayn = $msg->aynn(0,$backopt);
        }
        next if ($ayn eq $edr->{msg}{back} || $cons eq $edr->{msg}{back} || $sev eq $edr->{msg}{back});
        
        Msg::title();
        $msg = Msg::new("SNMP notification verification:\n", 40, 2296);
        $msg->bold;
        $msg0 = $prod->display_csgnic($rcsgnic);
        $msg = Msg::new("\tSNMP port: $port\n", 40, 2297, "$port");
        $msg0 .= $msg->{msg};
        foreach $n(0..$#conss) { 
            $msg = Msg::new("\tConsole: $conss[$n] receives SNMP traps for $sev_msgs->{$sevs[$n]} or higher events\n", 40, 2298, "$conss[$n]", "$sev_msgs->{$sevs[$n]}");
            $msg0 .= $msg->{msg};
        }
        if ($nhip) {
            $msg = Msg::new("\tNetworkHosts: $nhip\n", 40, 3402, "$nhip");
            $msg0 .= $msg->{msg};
        }
        $msg0.=$prod->display_csgnwhosts($nwhosts) if ($nwhosts);
        Msg::print($msg0);
        $backopt = "";
        $msg = Msg::new("Is this information correct?", 40, 1102);
        $ayn = $msg->ayny;
        $done = 1 if ($ayn eq "Y");
    }
    
    $prod->store_nic($rcsgnic,"vcs_csgnic");
    $cfg->{vcs_snmpport} = $port;
    $cfg->{vcs_snmpcons} = [];
    $cfg->{vcs_snmpcsev} = [];
    foreach $n(0..$#conss) { 
        push(@{$cfg->{vcs_snmpcons}},$conss[$n]);
        push(@{$cfg->{vcs_snmpcsev}},$sevs[$n]);
    }
    $cfg->{vcs_networkhosts} = $nhip if ($nhip);
    $prod->store_nwhosts_csg($nwhosts) if ($nwhosts);
    $msg = Msg::new("SNMP notification configuration:\n", 40, 2299);
    $cmsg = $msg->{msg};
    push(@{$prod->{configmsg}},$cmsg,$msg0);
}


# ask for an snmp console server
sub ask_snmpconsole {
    my ($prod) = @_;
    my ($cons,$ping,$help,$msg,$backopt,$edr,$def);

    $backopt = 1;
    $edr = Obj::edr();
    $help = Msg::new("The console designated will receive notification of $prod->{abbr} events via SNMP trap messages", 40, 2300, "$prod->{abbr}");
    while (1) {
        $msg = Msg::new("Enter the SNMP console system name:", 40, 2301);
        $cons = $msg->ask($def,$help,$backopt);
        return $cons if ($cons eq $edr->{msg}{back});

        #check the ping state of console system.
        $ping = $edr->localpadv->ping($cons);
        if ($ping) {
            $msg = Msg::new("Cannot ping $cons", 40, 2302, "$cons");
            $msg->print;
        } else {
            return $cons;
        }
    }
}

# ask for the snmp port, default is 162
sub ask_snmpport {
    my ($prod) = @_;
    my ($ayn,$port,$def,$backopt,$msg,$help,$edr);
    
    $def = 162;
    $backopt = 1;
    $edr = Obj::edr();
    $help = Msg::new("SNMP traps are sent to a specific port on each console server.  The default port number for SNMP trap messages is 162.", 40, 2303);
    while (1) {
        $msg = Msg::new("Enter the SNMP trap daemon port:", 40, 2304); 
        $port = $msg->ask($def, $help, $backopt);
        return $port if ($port eq $edr->{msg}{back});
        
        if (($port>65535) || ($port<0) || ($port=~/\D/)) {
            $msg = Msg::new("$port is an invalid port number", 40, 2305, "$port");
            $msg->print;
        } elsif (($port!=162) && ($port<=1024)) {
            $msg = Msg::new("Are you sure you want to use $port as the SNMP trap port?", 40, 2306, "$port");
            $ayn = $msg->aynn;
            return $port if ($ayn eq "Y");
        } else {
            return $port;
        }
    }
}

# Configure SMTP Notification:
# A SMTP mail server, recipients, and severity for each
sub config_smtp {
    my ($prod) = @_;
    my (@recps,@sevs,$help,$ayn,$edr,$cfg,$cmsg,$done,$backopt,$msg,$n,$rcsgnic,$recp,$server,$sev,$msg0,$nwhosts,$nonhip,$nhip);
    return if (Cfg::opt(qw(installonly responsefile)));

    $edr = Obj::edr();
    $cfg = Obj::cfg();
    my $sev_msgs = $prod->get_severity_msgs;

    $done = "";
    while (!$done) {
        undef(@recps);
        undef(@sevs);
        undef($nwhosts);
        $backopt = "";
        
        Msg::title();
        $msg = Msg::new("The following information is required to configure SMTP notification:\n", 40, 2307);
        $msg->bold;
        $msg = Msg::new("\tThe domain-based hostname of the SMTP server", 40, 2308);
        $msg->print;
        $msg = Msg::new("\tThe email address of each SMTP recipient", 40, 2309);
        $msg->print;
        $msg = Msg::new("\tA minimum severity level of messages to send to each recipient", 40, 2310);
        $msg->print;
        if ((EDR::plat($prod->{padv}) eq "HPUX") && !$cfg->{vcs_csgvip}) {
            $msg=Msg::new("\tOne or more NetworkHosts IP addresses for connection checking\n", 40, 3401);
            $msg->print;
            $nonhip=1;
        } else {
            Msg::n();
        } 
        
        $help = Msg::new("If SMTP is configured, status and failure messages about cluster can be delivered to users", 40, 2311);
        $msg = Msg::new("Do you want to configure SMTP notification?", 40, 2312);
        $ayn = $msg->aynn($help);
        return if ($ayn eq "N");

        Msg::n();
        $backopt = 1;
        $rcsgnic = $prod->ask_publicnic(1);
        next if ($rcsgnic eq $edr->{msg}{back});
        
        # AIX virtual nic specific configuration
        if ($prod->has_virtual_nic()) {
            if (!$prod->has_same_csg($rcsgnic)) {
                $nwhosts=$prod->ask_networkhosts_csg($rcsgnic);
                next if ($nwhosts eq $edr->{msg}{back});                
            }
        }
        # HP specific network host configuration
        $nhip=$prod->ask_nhip() if ($nonhip);
        next if ($nhip eq $edr->{msg}{back});
        
        $server = $prod->ask_smtpserver($server);
        next if ($server eq $edr->{msg}{back});

        while (($#recps<0) || ($ayn eq "Y")) {
            $recp = lc($prod->ask_smtprecp($recp));
            last if ( $recp eq $edr->{msg}{back} );
            $sev = $prod->ask_severity($recp);
            last if ( $sev eq $edr->{msg}{back} );
            $n = EDR::arrpos($recp,@recps);
            if ($n>=0) {
                $msg = Msg::new("$recp was previously entered. Changing priority to $sev_msgs->{$sev}.", 40, 2294, "$recp", "$sev_msgs->{$sev}");
                $msg->print;
                $sevs[$n] = $sev;
            } else {
                push(@recps,$recp);
                push(@sevs,$sev);
            }

            $msg = Msg::new("Would you like to add another SMTP recipient?", 40, 2313);
            $ayn = $msg->aynn(0,$backopt);
        }
        next if ($ayn eq $edr->{msg}{back} || $recp eq $edr->{msg}{back} || $sev eq $edr->{msg}{back} );
        
        Msg::title();
        $msg = Msg::new("SMTP notification verification:\n", 40, 2314);
        $msg->bold;
        $msg0 = $prod->display_csgnic($rcsgnic);
        $msg = Msg::new("\tSMTP Address: $server\n", 40, 2315, "$server");
        $msg0 .= $msg->{msg};
        foreach $n(0..$#recps) { 
            $msg = Msg::new("\tRecipient: $recps[$n] receives email for $sev_msgs->{$sevs[$n]} or higher events\n", 40, 2316, "$recps[$n]", "$sev_msgs->{$sevs[$n]}");
            $msg0 .= $msg->{msg};
        }
        if ($nhip) {
            $msg = Msg::new("\tNetworkHosts: $nhip\n", 40, 3402, "$nhip");
            $msg0 .= $msg->{msg};
        }
        $msg0.=$prod->display_csgnwhosts($nwhosts) if ($nwhosts);
        Msg::print($msg0);
        $backopt = "";
        $msg = Msg::new("Is this information correct?", 40, 1102);
        $ayn = $msg->ayny();
        $done = 1 if ($ayn eq "Y");
    }
    
    $prod->store_nic($rcsgnic,"vcs_csgnic");
    $cfg->{vcs_smtpserver} = $server;
    $cfg->{vcs_smtprecp} = [];
    $cfg->{vcs_smtprsev} = [];
    foreach $n (0..$#recps) { 
        push(@{$cfg->{vcs_smtprecp}}, $recps[$n]);
        push(@{$cfg->{vcs_smtprsev}}, $sevs[$n]);
    }
    $cfg->{vcs_networkhosts} = $nhip if ($nhip);
    $prod->store_nwhosts_csg($nwhosts) if ($nwhosts);
    $cmsg = Msg::new("SMTP notification configuration:\n", 40, 2317);
    push(@{$prod->{configmsg}},$cmsg,$msg0);
}

sub has_same_csg {
    my ($prod,$rnic)=@_;
    my ($cpic,$cfg,$sys,$sys0);
    
    $cpic=Obj::cpic();
    $cfg=Obj::cfg();
    if ( EDR::hashvaleq($rnic)) {
        $sys0 = ${$cpic->{systems}}[0];
        return 0 if ($cfg->{vcs_csgnic}{all} ne $$rnic{$sys0->{sys}});
    } else {
        for $sys (@{$cpic->{systems}}) { 
            return 0 if ($cfg->{vcs_csgnic}{$sys->{sys}} ne $rnic->{$sys->{sys}}); 
        }
    }
    return 1;
}
# store a selected nic in a hash
sub store_nic {
    my ($prod,$rnic,$key) = @_;
    my ($sys,$edr,$sys0,$cfg);
    
    return if (!$rnic);
    $edr = Obj::edr();
    $cfg = Obj::cfg();
    if ( EDR::hashvaleq($rnic)) {
        $sys0 = ${$edr->{systems}}[0];
        $cfg->{$key}{all} = $$rnic{$sys0->{sys}};
        foreach $sys (@{$edr->{systems}}) { 
            undef($cfg->{$key}{$sys->{sys}}); 
        }
    } else {
        foreach $sys(@{$edr->{systems}}) { 
            $cfg->{$key}{$sys->{sys}} = $$rnic{$sys->{sys}}; 
        }
        undef($cfg->{$key}{all}); 
    }
    $prod->{"r$key"} = $rnic;
}

sub display_csgnwhosts {
    my ($prod,$rnwhosts)=@_;
    my ($cpic,$msg,$sys,$nmsg);
    
    return "" if (!$rnwhosts);
    $cpic=Obj::cpic();
    for $sys (@{$cpic->{systems}}) { 
        if ($rnwhosts->{$sys->{sys}}) {
            $msg=Msg::new("\tNetworkHosts ($sys->{sys}): $rnwhosts->{$sys->{sys}}\n", 40, 2318, "$sys->{sys}", "$rnwhosts->{$sys->{sys}}");
            $nmsg.=$msg->{msg};
        }
    }
    return $nmsg;
}

# display csgnic information for confirmation
sub display_csgnic {
    my ($prod,$rcsgnic) = @_;
    my ($nmsg,$sys,$sys0,$msg,$edr);
    return "" if (!$rcsgnic);

    $edr = Obj::edr();
    if ( EDR::hashvaleq($rcsgnic) ) {
        $sys0 = ${$edr->{systems}}[0];
        $msg = Msg::new("\tNIC: $$rcsgnic{$sys0->{sys}}\n", 40, 2319, "$$rcsgnic{$sys0->{sys}}");
        $nmsg .= $msg->{msg};
    } else {
        for $sys (@{$edr->{systems}}) { 
            $msg = Msg::new("\tNIC ($sys->{sys}): $$rcsgnic{$sys->{sys}}\n", 40, 2320, "$sys->{sys}", "$$rcsgnic{$sys->{sys}}");
            $nmsg .= $msg->{msg};
        }
    }
    return $nmsg;
}


# ask for severity for SMTP/SNMP notification
sub ask_severity {
    my ($prod,$rc,$snmp) = @_;
    my (@sevs,$sev,$msg,$edr,$help,$def,$backopt);

    $edr = Obj::edr();
    $backopt = 1;
    @sevs = ("Information","Warning","Error","SevereError");
    $help = Msg::new("Information messages are events exhibiting normal behavior.  Warning messages draw attention to deviation from normal behavior.  Error messages draw attention to faults.  SevereError messages are critical errors that can lead to loss of data and/or corruption.", 40, 2321);
    if ($snmp){
        $msg = Msg::new("Enter the minimum severity of events for which SNMP traps should be sent to $rc  [I=Information, W=Warning, E=Error, S=SevereError]:", 40, 2322, "$rc");
    }else{
        $msg = Msg::new("Enter the minimum severity of events for which mail should be sent to $rc  [I=Information, W=Warning, E=Error, S=SevereError]:", 40, 2323, "$rc");
    }
    while (!EDR::inarr($sev,@sevs)) { 
        $sev = $msg->ask($def,$help,$backopt);
        return $sev if ($sev eq $edr->{msg}{back});
        $sev = "Information" if ($sev =~ /^I/i);
        $sev = "Warning" if ($sev =~ /^W/i);
        $sev = "Error" if ($sev =~ /^E/i);
        $sev = "SevereError" if ($sev =~ /^S/i);
    }
    return $sev;
}


# ask for a smtp recipient
sub ask_smtprecp {
    my ($prod,$def) = @_;
    my ($email,$help,$msg,$answer,$edr,$backopt);
    $edr = Obj::edr();
    $help = Msg::new("The SMTP recipient will receive notification of $prod->{abbr} events via email at the designated address", 40, 2324, "$prod->{abbr}");
    # vxgettext is sensitive with @
    $email="user\@yourcompany.com";
    $msg = Msg::new("Enter the full email address of the SMTP recipient\n(example: $email):", 40, 2325, "$email");    
    $backopt = 1;
    while (1) {
        $answer = $msg->ask($def,$help,$backopt);
        return $answer if ( $answer eq $edr->{msg}{back} || $prod->verify_emailadd($answer));
    }
}

# verify an email address has proper format
sub verify_emailadd {
    my ($prod,$recp) = @_;
    my(@aa,@sa,$msg);
 
    @aa = split(/@/,$recp);
    @sa = split(/\./,$aa[1]);
    if ( ($#aa!=1) || ($#sa<1) ) {
        $msg = Msg::new("$recp is not a valid domain-based email address", 40, 2326, "$recp");
        $msg->print;
        return 0;
    } 
    return 1;
}

# ask for one public nic
# $ntfrgco="" means first time in for cluster service group
# $ntfrgco=1 means potential second time in for notifier
# $ntfrgco=2 means potential second time in for global cluster option
sub ask_publicnic {
    my ($prod, $ntfrgco) = @_;
    my ($edr,$def,$help,$msg,%csgnic,$all,$aayn,$ayn,$en,$nic,$nicl,$rpn,$sys,$backopt,$padv);
    return "" if (($prod->{rcsgnic}) && ($ntfrgco == 1));

    $edr = Obj::edr();
    $backopt = 1;
    foreach $sys(@{$edr->{systems}}) {
        if ($all) { 
            $csgnic{$sys->{sys}} = $nic; 
            next; 
        }
        $padv=$sys->padv;
        $rpn = $padv->publicnics_sys($sys);
        $rpn=EDR::arruniq(@$rpn);
        if ($#$rpn<0) {
            $msg = Msg::new("No active NIC devices have been discovered on $sys->{sys}", 40, 2327, "$sys->{sys}");
            $msg->warning();
        } else {
            $nicl = join(" ",@$rpn);
            $msg = Msg::new("Active NIC devices discovered on $sys->{sys}: $nicl", 40, 2328, "$sys->{sys}", "$nicl");
            $msg->print();
        }

        $nic = "";
        while (!$nic) {
            #ask for the active NIC for VCS Notifier
            $def = $$rpn[0];
            $help = Msg::new("The NIC selected is typically the network card on which the system's public IP address is active.", 40, 2329);
            if ($ntfrgco == 2) {
                $msg = Msg::new("Enter the NIC for Global Cluster Option to use on $sys->{sys}:", 40, 2330, "$sys->{sys}");
            } elsif ($ntfrgco == 1){   
                $msg = Msg::new("Enter the NIC for the $prod->{abbr} Notifier to use on $sys->{sys}:", 40, 2331, "$prod->{abbr}", "$sys->{sys}");
            } else {
                $msg = Msg::new("Enter the NIC for Virtual IP of the Cluster to use on $sys->{sys}:", 40, 2332, "$sys->{sys}");
            }
            $en = $msg->ask($def,$help,$backopt);
            return $en if ($en eq $edr->{msg}{back});
            $en = EDR::despace($en);

            if ($en eq "") {
            } elsif ($en =~/\s+/) {
                $msg = Msg::new("Only one NIC name is needed at one time", 40, 2333);
                $msg->print;
            } elsif (!EDR::nic_sys($sys, $en)) {
                $msg = Msg::new("$en is not a valid NIC name", 40, 2334, "$en");
                $msg->print;
            } elsif ((EDR::inarr($sys,@{$edr->{systems}})) && 
                (!EDR::inarr($en,@$rpn))) {
                $msg = Msg::new("$en does not appear to be an active NIC", 40, 2335, "$en");
                $msg->print;
                $help = Msg::new("The NIC selected is typically the network card on which the systems public IP address is active. This NIC may or may not be the same on each system within a cluster.", 40, 2336);
                $msg = Msg::new("Are you sure you want to use NIC $en ?", 40, 2337, "$en");
                $ayn = $msg->aynn($help,$backopt);
                return $ayn if ($ayn eq $edr->{msg}{back});
                $nic = $en if ($ayn eq "Y");
            } else {
                $nic=$en;
            }
        }
        $csgnic{$sys->{sys}} = $nic;

        #ask the nic for all systems.
        if ($sys eq ${$edr->{systems}}[0]) {
            $help = Msg::new("The NIC selected is typically the network card on which the systems public IP address is active. This NIC device may or may not be the same on each system within a cluster.", 40, 2338);
            $msg = Msg::new("Is $nic to be the public NIC used by all systems?", 40, 2339, "$nic");
            $aayn = $msg->ayny($help,$backopt);
            return $aayn if ($aayn eq $edr->{msg}{back});
        }
        $all=1 if ($aayn eq "Y");
    }
    return \%csgnic;
}

# ask for an SMTP mail server
sub ask_smtpserver {
    my ($prod,$def) = @_;
    my ($ask,$help,$msg,$edr,$backopt);
    $edr = Obj::edr();
    $help = Msg::new("The SMTP server is a mail server on your network", 40, 2340);
    $msg = Msg::new("Enter the domain-based hostname of the SMTP server\n(example: smtp.yourcompany.com):", 40, 2341);
    $backopt = 1;
    while(1) {
        $ask = $msg->ask($def,$help,$backopt);
        return $ask if ($ask eq $edr->{msg}{back} || $prod->verify_smtpserver($ask) );
    }
}


# verify an smtp server is a complete domain address and can be pinged
sub verify_smtpserver {
    my ($prod,$server) = @_;
    my(@f,$msg,$edr);

    $edr = Obj::edr();
    @f = split(/\./,$server);    
    if ( $#f<2 ) {
        $msg = Msg::new("$server does not include a domain", 40, 2342, "$server");
        $msg->print;
    } elsif ($edr->localpadv->tsub("ping", $server)) {
        $msg = Msg::new("Cannot ping $server", 40, 2302, "$server");
        $msg->print;
    } else {
        return 1;
    }
    return 0;
}

sub set_vcsencrypt {
	my ($prod,$edr);
	$prod=shift;
	$edr = Obj::edr();
	#check for vcsencrypt binary
    if ($edr->{fromdisk}){
        $prod->{vcsencrypt} = "$prod->{bindir}/vcsencrypt";
    }else{
    #TODO: we need to change the vcsencrypt file according to local padv when a2a build is ready. 
        $prod->{vcsencrypt} = "$edr->{mediapath}/scripts/vcsencrypt";
    }
    return 1 if (-x "$prod->{vcsencrypt}");
    
    Msg::log("Cannot find vcsencrypt binary - skipping add_users");
    return 0;
}

sub add_users {
    my ($prod) = @_;
    my($edr,$cfg,$padv,$msg,$backopt,@epws,@pris,@users,@pws,$ayn,$ayn_more,$done,$epw,$pri,$user,$n);
    return "" if (Cfg::opt("responsefile"));
    $edr = Obj::edr();
    $cfg = Obj::cfg();
    $padv = Obj::padv($prod->{padv});
    return 1 if (($cfg->{at_rootdomain}) || ($prod->{vxssmenuopt}) || ($cfg->{opt}{installonly}));

    return 0 unless ($prod->set_vcsencrypt);
    
    #ask for users info.(name/passwd/privilege) 
    while (!$done) {
        Msg::title();
        undef(@users);
        undef(@epws);
        undef(@pris);
        $backopt = "";
        
        $msg = Msg::new("The following information is required to add $prod->{abbr} users:\n", 40, 2344, "$prod->{abbr}");
        $msg->bold;
        $msg = Msg::new("\tA user name", 40, 2345);
        $msg->print;
        $msg = Msg::new("\tA password for the user", 40, 2346);
        $msg->print;
        $msg = Msg::new("\tUser privileges (Administrator, Operator, or Guest)\n", 40, 2347);
        $msg->print;
        
        $msg = Msg::new("Do you want to set the username and/or password for the Admin user\n(default username = 'admin', password='password')?", 40, 2348);
        $ayn = $msg->aynn;
        if ($ayn eq "Y") {
            $prod->{setadminpwd} = 1;
            $backopt = 1;
            $user = $prod->ask_username("admin");
            next if ($user eq $edr->{msg}{back});
            $backopt = "";
            $epw=$prod->encrypt_password;
            Msg::n();
        } else {
            $user = "admin";
            $epw = $prod->encrypt_password("password");
        }
        push(@users,$user);
        push(@epws,$epw);
        push(@pris,"Administrators");
        
        $msg = Msg::new("Do you want to add another user to the cluster?", 40, 2349);
        $ayn = $msg->aynn;
        while (($#users<0) || ($ayn eq "Y")) {
            $backopt = 1;
            $user = $prod->ask_username;
            last if ($user eq $edr->{msg}{back});

            # check if the user has been entered beforehand so as to provide more info previously
            if( EDR::inarr($user, @users) ) {
                $ayn_more= "N";
                if( $user eq "admin") {
                    $msg = Msg::new("$user was previously entered. Do you want to update its password?", 40, 2350, "$user");
                    $ayn_more= $msg->aynn;
                } else {
                    $msg = Msg::new("$user was previously entered. Do you want to update its password and privilege?", 40, 2351, "$user");
                    $ayn_more= $msg->aynn;
                }
                next if( $ayn_more eq "N");
            }

            $epw = $prod->encrypt_password;
            $backopt = 1;
            $pri = $prod->ask_userprivilege($user);
            $n = EDR::arrpos($user,@users);
            if ($n>=0) {
                $epws[$n]=$epw;
                $pris[$n]=$pri unless ($user eq "admin");
            } else {
                push(@users,$user);
                push(@epws,$epw);
                push(@pris,$pri);
            }
            $msg = Msg::new("Would you like to add another user?", 40, 2352);
            $ayn = $msg->aynn();
            Msg::n();
        }
        next if ($user eq $edr->{msg}{back});
        $backopt = "";
        Msg::title();
        $msg = Msg::new("$prod->{abbr} User verification:\n", 40, 2353, "$prod->{abbr}");
        $msg->bold();

        # find the max length of all names
        my $lenmax= 0;
        foreach my $name (@users) {
            $lenmax = (length($name) > $lenmax) ? length($name) : $lenmax;
        }

        # display names line by line with more spaces for the shorter names
        foreach $n (0..$#users) {
            my $lenname = length($users[$n]);
            $msg = Msg::new("\tUser: $users[$n]", 40, 2354, "$users[$n]");
            my $line = $msg->{msg};
            for my $ispaces ($lenname..$lenmax+4) {
                $line .= " ";
            }
            $msg = Msg::new("Privilege: $pris[$n]", 40, 3714, "$pris[$n]");
            $line .= $msg->{msg};
            Msg::print($line);
        }
        Msg::n();
        $msg = Msg::new("\tPasswords are not displayed", 40, 3715);
        $msg->printn;
        $msg = Msg::new("Is this information correct?", 40, 1102);
        $ayn = $msg->ayny();
        $done = 1 if ($ayn eq "Y");
    }

    $cfg->{vcs_username} = [];
    $cfg->{vcs_userpriv} = [];
    @{$cfg->{vcs_userenpw}} = @epws;
    foreach $n(0..$#users) {
        push(@{$cfg->{vcs_username}},$users[$n]);
        push(@{$cfg->{vcs_userpriv}},$pris[$n]);
    }
}

# ask for user name
sub ask_username {
    my($prod,$defanswer) = @_;
    my ($help, $msg, $answer,$edr,$backopt);

    $edr = Obj::edr();
    $help = Msg::new("Username can have the following characters: alphabets[a-z][A-Z], numbers[0-9], underscore[_], or minus[-].", 40, 2357);
    $msg = Msg::new("Enter the user name:", 40, 2358);
    $backopt = 1;
    while (1) {
        $answer = $msg->ask($defanswer, $help, $backopt);
        return $answer if ( $answer eq $edr->{msg}{back} || $prod->verify_username( $answer));
    }
}

sub verify_username {
    my ($prod,$name) = @_;
    my ($msg);
    
    $name =~ s/[A-Za-z0-9_-]//g;
    if ($name) {
        $msg = Msg::new("User name cannot use the characters: $name", 40, 2359, "$name");
        $msg->print;
        return 0;
    }else{
        return 1;
    }
}

sub encrypt_password {
    my ($prod,$passwd) = @_;
    my ($epw,$sys,$edr,$nolog);

    $edr = Obj::edr();
    $sys = ${$edr->{systems}}[0];
    # get password until effective  
    $passwd = $prod->ask_userpassword if(!$passwd);

    #Install from localsys or from hardisk
    if ($sys->{islocal} || $edr->{fromdisk}){
        #Here we use system call which will not log the password in clear text.
        system("$prod->{vcsencrypt} -vcs $passwd 1> $edr->{tmpdir}/userpasswd");
        $epw = EDR::despace(EDR::cmd_local("_cmd_cat $edr->{tmpdir}/userpasswd"));
    }else{
        EDR::copy_sys($sys, $prod->{vcsencrypt},"$edr->{tmpdir}/vcsencrypt");
        # Do not log the password
        $edr->{donotlog} = 1;
        $epw = EDR::despace(EDR::cmd_sys($sys, "$edr->{tmpdir}/vcsencrypt -vcs $passwd"));
        $edr->{donotlog} = 0;
    }
    return "$epw";
}

# ask for user password
sub ask_userpassword {
    my ($prod) = @_;
    my ($passwd,$passwd2,$msg,$edr);

    $edr = Obj::edr();
    while (1) {
        $msg = Msg::new("Enter the password: ", 40, 2360);
        print "$edr->{tput}{bs}$msg->{msg}$edr->{tput}{be}";
        EDR::cmd_local("stty -echo");
        $passwd = <STDIN>;
        print("\n");
        $passwd = EDR::despace($passwd);
        if( $passwd eq "") {
            $msg = Msg::new("Empty passwords are not allowed.", 40, 2361);
            $msg->print;
            next;
        } elsif ($passwd=~/\P{IsASCII}/) {
            $msg = Msg::new("Only ASCII characters are allowed.", 40, 2362);
            $msg->print;
            next;
        }
        EDR::cmd_local("stty echo");
        $msg = Msg::new("Enter again:", 40, 2363);
        print "$edr->{tput}{bs}$msg->{msg}$edr->{tput}{be}";
        EDR::cmd_local("stty -echo");
        $passwd2 = <STDIN>;
        EDR::cmd_local("stty echo");
        print("\n");
        $passwd2 = EDR::despace($passwd2);
        return $passwd if ($passwd eq $passwd2);

        $msg = Msg::new("Passwords do not match", 40, 2364);
        $msg->warning();
    }
}


sub ask_userprivilege {
    my ($prod,$user)=(@_);
    my ($edr,@pris,$pri,$msg,$help,$def,$backopt);
    
    @pris = ("Administrators","Operators","Guests");
    $def = "";
    $backopt = 1;
    $edr = Obj::edr();
    $help = Msg::new("Administrator users have full privileges to make changes to the cluster.\nOperator users have limited administrative privileges on the cluster.\nGuest users have read-only access to the cluster.", 40, 2365);
    $msg = Msg::new("Enter the privilege for user $user (A=Administrator, O=Operator, G=Guest):", 40, 2366, "$user");
    while (!EDR::inarr($pri,@pris)) {
         $pri = $msg->ask($def,$help,$backopt);
         return $pri if ($pri eq $edr->{msg}{back});
         $pri = "Administrators" if ($pri =~ /^A$/i || $pri =~/^[Aa]dministrator$/);
         $pri = "Operators" if ($pri =~ /^O$/i || $pri =~/^[Oo]perator$/);
         $pri = "Guests" if ($pri =~ /^G$/i || $pri =~/^[Gg]uest$/);
    }
    return $pri;
}

sub check_config {
    my $prod = shift;
    my ($at,$cfg,$clusterid,$clustername,$confi,$cpic,$llt_gab_configured,$mode,$rootpath,$sys0,$sysi);
    $cpic = Obj::cpic();
    $cfg = Obj::cfg();
    $at=Obj::prod("AT50",$prod->{padv});
    return 0 if (Cfg::opt("responsefile") && !defined($cfg->{donotreconfigurevcs}) );
    $rootpath = Cfg::opt("rootpath") || "";
    for $sysi (@{$cpic->{systems}}) {
        $confi = $prod->get_config_sys($sysi);
        if ($confi) {
            return 0 if ((defined $clustername) && ($clustername ne $confi->{clustername}));
            return 0 if ($confi->{secureclus} && !$at->check_vxssmode_sys($sysi)); 
            $clustername = $confi->{clustername};
            return 0 if ((defined $clusterid) && ($clusterid ne $confi->{clusterid})); 
            $clusterid = $confi->{clusterid};
        } else {
            return 0;
        }
    }
    # for single node cluster, check if llt/gab is configured before.
    if ($#{$cpic->{systems}} == 0) {
        $sys0 = ${$cpic->{systems}}[0];
        if (EDR::file_sys($sys0,"$rootpath$prod->{llthosts}")
                && EDR::file_sys($sys0,"$rootpath$prod->{llttab}")
                && EDR::file_sys($sys0,"$rootpath$prod->{gabtab}")) {
            $llt_gab_configured = 1;
        }
        if (($cpic->{prod}=~/^(SVS|SFCFS|SFRAC)/) || ($cfg->{vcs_allowcomms})) {
            return 0 unless $llt_gab_configured;
        }
        if (defined $cfg->{vcs_allowcomms} && ($cfg->{vcs_allowcomms} == 0)) {
            if (EDR::file_sys($sys0,"$rootpath$prod->{llthosts}")
                || EDR::file_sys($sys0,"$rootpath$prod->{llttab}")
                || EDR::file_sys($sys0,"$rootpath$prod->{gabtab}")) {
                return 0;
            }
        }
    }

    return 1;
}

sub get_config_sys {
    my (%conf,$key,$line,@lines,$llttab,$maincf,$n,$nlow,$num,$str,$system,@values);
    my ($prod,$sys) = @_;
    my $rootpath = Cfg::opt("rootpath") || "";
    
    return 0 unless (EDR::file_sys($sys,"$rootpath$prod->{maincf}"));
    return $sys->{vcs_conf} if (defined($sys->{vcs_conf}));
    
    $maincf = "$rootpath$prod->{maincf}";
    $str = EDR::cmd_sys($sys, "_cmd_grep '^cluster' $maincf 2> /dev/null");
    # get cluster name
    if ($str =~ /^cluster\s+(\S+)\s*/) {
        $conf{clustername} = $1;
    } else {
        return 0;
    }
    
    # get secure cluster flag
    $str=EDR::cmd_sys($sys, "_cmd_grep 'SecureClus' $maincf 2> /dev/null");
    if ($str=~/SecureClus\s*=\s*(\d)/) {
        $conf{secureclus} = $1;
    } else {
        $conf{secureclus} = 0;
    }
    
    # get cluster systems
    $conf{systems} = [];
    $num = 0;
    $str = EDR::cmd_sys($sys, "_cmd_grep '^system' $maincf 2> /dev/null");
    for $system(split(/\n/,$str)) {
        next unless ($system =~ /^system\s+(\S+)\s*/);
        $system = $1;
        $system =~ s/"(.*)"/$1/; # remove double quote
        push(@{$conf{systems}},$system);
        $num ++;
    }
    return 0 if ($num == 0); # no system defined
    
    if ($num > 1) {
        # multi-node cluster, llt and gab should also be configured
        unless (EDR::file_sys($sys,"$rootpath$prod->{llthosts}")
                && EDR::file_sys($sys,"$rootpath$prod->{llttab}")
                && EDR::file_sys($sys,"$rootpath$prod->{gabtab}")) {
            return 0;
        }
    }

    if (EDR::file_sys($sys,"$rootpath$prod->{llttab}")) {
        $llttab = EDR::cmd_sys($sys,"_cmd_cat $rootpath$prod->{llttab} 2>/dev/null");
        # get cluster id
        if ($llttab =~ /\n\s*set-cluster\s+(\d+)\s*/) {
            $conf{clusterid} = $1;
        } else {
            return 0;
        }
        # get heartbeat nics line by line in $llttab
        $n = 0;
        $nlow = 0;
        @lines = split(/\n/, $llttab);
        for $line (@lines) {
            if ($line =~ /^\s*link\s+.*/) {
                $n++;
                @values = split(/\s+/,$line);
                if ( $values[4] =~ /udp6?/ ) {
                    $key = "lltlink$n";
                    $conf{$key}{$sys->{sys}} = $prod->convert_linkip2nic_sys($sys, $values[7]);
                    $conf{lltoverudp} = 1;
                    $key = "udplink$n"."_address";
                    $conf{$key}{$sys->{sys}} = $values[7];
                    $key = "udplink$n"."_port";
                    $conf{$key}{$sys->{sys}} = $values[5];
                } else {
                    $key = "lltlink$n";
                    $conf{$key}{$sys->{sys}} = $prod->convert_linkdev2nic_sys($sys, $values[2]);
                }
                    
            } elsif ($line =~ /^\s*link-lowpri\s+.*/) {
                $nlow++;
                @values = split(/\s+/,$line);
                if ( $values[4] =~ /udp6?/ ) {
                    $key = "lltlinklowpri$nlow";
                    $conf{$key}{$sys->{sys}} = $prod->convert_linkip2nic_sys($sys, $values[7]);
                    $conf{lltoverudp} = 1;
                    $key = "udplinklowpri$nlow"."_address";
                    $conf{$key}{$sys->{sys}} = $values[7];
                    $key = "udplinklowpri$nlow"."_port";
                    $conf{$key}{$sys->{sys}} = $values[5];
                } else {
                    $key = "lltlinklowpri$nlow";
                    $conf{$key}{$sys->{sys}} = $prod->convert_linkdev2nic_sys($sys, $values[2]);
                }
            }
        }
    } else {
        $conf{onenode} = 1 if ($num == 1);
    }   
        
    $sys->{vcs_conf} = \%conf;
    Obj::set_value($sys->{pool},"cluster_systems","list",@{$conf{systems}});
    Thr::setq($sys->{pool},"cluster_systems","list",@{$conf{systems}});
    return \%conf;
}

sub configure_sys {
    my ($cfg,$cpic,$edr,$hostname,$msg,$prod,$rtn,$str,$sys,$sysi);
    ($prod,$sys) = @_;

    $edr = Obj::edr();
    $cfg = Obj::cfg();
    $cpic = Obj::cpic();
    my $vxfen=$sys->proc("vxfen51");
    my $vxatd=$sys->proc("vxatd50");
   
    if (!$cfg->{donotreconfigurevcs}) {
        # for SVS/SFCFS/SFCFSHA/SFCFSRAC/SFRAC, start vxfen in disabled mode initially.
        if ($cpic->{prod} =~ /^(SVS|SFCFS|SFRAC)/) {
            EDR::cmd_sys($sys, "_cmd_cp /etc/vxfen.d/vxfenmode_disabled $prod->{vxfenmode}");
        } else {
            # for VCS/SFHA, do not start vxfen initially
            EDR::set_value_sys($sys,'donotstartprocs',"push","vxfen51");
            $str = EDR::datetime();
            EDR::cmd_sys($sys, "_cmd_mv $prod->{vxfenmode} $prod->{vxfenmode}-$str") if (EDR::file_sys($sys, "$prod->{vxfenmode}"));
            EDR::cmd_sys($sys, "_cmd_mv $prod->{vxfendg} $prod->{vxfendg}-$str") if (EDR::file_sys($sys, "$prod->{vxfendg}"));
            $vxfen->disable_sys($sys);
            if ($vxfen->can("disable_service_sys")) {
                $vxfen->disable_service_sys($sys);
            }
        }
    } else {
        if ($cpic->{prod} =~ /^(SVS|SFCFS|SFRAC)/) {
            if ((!EDR::file_sys($sys, $prod->{vxfenmode})) && (!EDR::file_sys($sys, $prod->{vxfendg}))) {
                EDR::cmd_sys($sys, "_cmd_cp /etc/vxfen.d/vxfenmode_disabled $prod->{vxfenmode}");
                $prod->vxfen_enable_sys($sys);
            }
        }
    }

    #return if ($cfg->{donotreconfigurevcs});
    if ($cfg->{donotreconfigurevcs}) {
        $prod->include_extra_types_sys($sys);
        return;
    }
        
    # Only perform following tasks on first node of the cluster
    if ($sys->system1) {
        #for secure cluster configuration
        $prod->{vxssmenuopt}=$cfg->{vcs_securitymenuopt} if($cfg->{vcs_securitymenuopt});
        if ($prod->{vxssmenuopt}) {
            $prod->{vssdefport}=$cfg->{vcs_vssdefport} if ($cfg->{vcs_vssdefport});
            $prod->{roothashpath}=$cfg->{vcs_roothashpath} if ($cfg->{vcs_roothashpath});
            $rtn=$prod->tsub("setup_vxsscredentials");
            if(!$rtn) {
                EDR::create_flag("main_cf_done");
                return "";
            }
            $vxatd->{donotstart}=0;
            Obj::set_value($vxatd->{pool},'donotstart',0);
            Thr::setq($vxatd->{pool},'donotstart',0); 
            # At this point, the installer has started vxatd and created required VCS domains/principals/credentials.
        }
    
        # uuid configuration
        $rtn = $prod->config_uuid();
        if ($rtn == -1) {
            $msg = Msg::new("Cannot find uuidconfig.pl for uuid configuration. Create uuid manually before starting VCS", 40, 2367);
            EDR::push_warning_sys($sys,$msg);
        }
        
        # generate configuration files
        Msg::log("Creating $prod->{name} configuration files");
        if ($cfg->{vcs_allowcomms}) {
            $prod->update_llttab();
            $prod->update_llthosts();
            $prod->update_gabtab();
        }
        $prod->update_maincf($sys);
        EDR::create_flag("main_cf_done");
    } else {
        EDR::wait_for_flag("main_cf_done");
    }
    
    return "" unless (-f "$edr->{tmpdir}/main.cf");
    $sysi=$sys->{sys};
    $hostname =transform_system_name($sysi);
    EDR::writefile("$hostname\n","$edr->{tmpdir}/sysname.$sysi");
    EDR::copy_sys($sys, "$edr->{tmpdir}/sysname.$sysi","$prod->{confdir}/sysname");
    EDR::copy_sys($sys, "$edr->{tmpdir}/main.cf", $prod->{maincf});
    Msg::log("Copying configuration files to $sysi");
    if ($cfg->{vcs_allowcomms}) {
        EDR::copy_sys($sys, "$edr->{tmpdir}/llttab.$sysi",$prod->{llttab});
        EDR::copy_sys($sys, "$edr->{tmpdir}/llthosts",$prod->{llthosts});
        EDR::copy_sys($sys, "$edr->{tmpdir}/gabtab",$prod->{gabtab});
        $prod->set_onenode_cluster_sys($sys,0) if ($prod->can("set_onenode_cluster_sys"));
    } else {
        # Linux: set ONENODE=yes in /etc/sysconfig/vcs
        # SunOS: import manifest vcs-onenode.xml
        $prod->set_onenode_cluster_sys($sys,1) if ($prod->can("set_onenode_cluster_sys"));
        # remove old config files if exist
        if (EDR::file_sys($sys,$prod->{llttab})) {
           EDR::cmd_sys($sys,"_cmd_mv $prod->{llttab} $prod->{llttab}.old"); 
        }
        if (EDR::file_sys($sys,$prod->{llthosts})) {
           EDR::cmd_sys($sys,"_cmd_mv $prod->{llthosts} $prod->{llthosts}.old"); 
        }
        if (EDR::file_sys($sys,$prod->{gabtab})) {
           EDR::cmd_sys($sys,"_cmd_mv $prod->{gabtab} $prod->{gabtab}.old"); 
        }
    }

    EDR::cmd_sys($sys, "_cmd_cp $prod->{confdir}/types.cf $prod->{configdir}");
    EDR::cmd_sys($sys, "_cmd_rmr $prod->{configdir}/.stale");
    # on AIX, update /etc/pse.conf file
    $prod->tsub("update_pseconf_sys", $sys) if $prod->can("update_pseconf_sys");

    # link install: configuration finished, enable it
    $prod->vcs_enable_sys($sys);
    
    #create .secure file for CmdServer
    if ($cfg->{vcs_securitymenuopt}) {
        EDR::cmd_sys($sys, "_cmd_touch $prod->{secure}");    
    } else {
        EDR::cmd_sys($sys, "_cmd_rmr $prod->{secure}");    
    }
}

# configure cluster information:
# Cluster name, ID, and heartbeat links
sub config_cluster {
    my($ayn,$backopt,$cfg,$cpic,$diff,$done,$edr,$help,$prod,$sysi,$sys);
    my($address,$cid,$cname,$key,$lmsg,$multinode,$msg,$n,$netmask,$rhbn,$vswif_error,$vswif_config,$output,$port,$config_file,$cpicprod);

    $prod = shift;
    $cfg = Obj::cfg();
    $cpic = Obj::cpic();
    $edr = Obj::edr();
    
    $backopt = 1;
    # by default, set vcs_allowcomms to 1 if it isn't defined
    # this happens if other products (eg. sfcfs sfrac) calls vcs configure_sys to setup clusters
    # but foget to set this attribute to 1 in order to let cpi generate llt/gab config files  
    $cfg->{vcs_allowcomms} = 1 unless (defined($cfg->{vcs_allowcomms}));
    $multinode = 1 if ($cfg->{vcs_allowcomms});
    while (!$done) {
        Msg::title();
        if ($multinode) {
            if ($cpic->{prod} =~ /VCS\d+/) {
                $msg = Msg::new("To configure VCS the following information is required:\n", 40, 2369);
            } else {
                $cpicprod=$cpic->prod;
                $msg = Msg::new("To configure VCS for $cpicprod->{abbr} the following information is required:\n", 40, 2370, "$cpicprod->{abbr}");
            }
            $msg->bold;
            $msg = Msg::new("\tA unique cluster name", 40, 3403);
            $msg->print;
            $msg = Msg::new("\tA unique cluster ID number between 0-65535", 40, 3404);
            $msg->print;
            if ($cfg->{fencingenabled}) {
                $msg = Msg::new("\tOne or more NICs per system used for heartbeat links\n", 40, 3405);
            } else {
                $msg = Msg::new("\tTwo or more NICs per system used for heartbeat links\n", 40, 3406);
            }
            $msg->print;
            if ($cpic->{prod} !~ /SFRAC\d+/) {
                $msg = Msg::new("\tOne or more heartbeat links are configured as private links", 40, 2375);
                $msg->print;
                $msg = Msg::new("\tYou can configure one heartbeat link as a low-priority link\n", 40, 3407);
                $msg->print;
            }
            $msg = Msg::new("All systems are being configured to create one cluster.\n", 40, 3408);
            $msg->print;
        } else {
            $msg = Msg::new("To configure a single-node VCS cluster, a unique cluster name is required:\n", 40, 2378);
            $msg->bold;
        }

        $cname=$prod->tsub("ask_clustername",$cname);
        $cname||=$cfg->{vcs_clustername}; # for installer resilience
        if ($multinode) {
            $cid=$prod->tsub("ask_clusterid",$cid);
            $cid||=$cfg->{vcs_clusterid}; # for installer resilience
            next if ($cid eq $edr->{msg}{back});
            $rhbn=$prod->tsub("hb_config_option");
            last if (Cfg::opt("responsefile"));
            next if ($rhbn eq $edr->{msg}{back});
            if (!$cfg->{autocfgllt}) {
                # check media speed for private nics on each system
                $diff = $prod->check_link_speed($edr->{systems},$rhbn);
                if ($diff == 1) {
                    Msg::n();
                    $msg = Msg::new("The private NICs do not have same media speed.\n", 40, 2958);
                    $msg->warning();
                    $msg = Msg::new("It is recommended that the media speed be same for all the private NICs. Consult your Operating System manual for information on how to set the Media Speed.\n", 40, 3409);
                    $msg->bold;
                }                    
                if ($diff == 2) {
                    Msg::n();
                    $msg = Msg::new("$edr->{script} can not detect media speed for the selected private NICs properly. Consult your Operating System manual for information on how to set the Media Speed.\n", 40, 2959, "$edr->{script}");
                    $msg->warning();
                }
                if ($diff) {    
                    $msg = Msg::new("Do you want to continue with current heartbeat configuration?", 40, 2382);
                    $ayn = $msg->ayny();
                    next if ($ayn ne "Y");

                    if ($prod->num_hbnics($rhbn) > 1) {
                        for $n (2..$prod->num_hbnics($rhbn)) {
                            $key="lltlink$n"."_low";
                            if ($rhbn->{$key} != $rhbn->{lltlink1_low}) {
                                $msg = Msg::new("Do you want to configure lower speed NICs as low-priority links?", 40, 3410);
                                $help = Msg::new("The performance of LLT would be decreased if NICs with lower media speed were used as high priority private links. The overall performance of LLT will be the same as the high priority link with the lowest media speed.", 40, 3411);
                                $ayn = $msg->ayny($help);
                                if ( $ayn eq "Y" ) {
                                    $rhbn = $prod->tsub("set_lowpri_for_slow_links",$rhbn);
                                }
                                last;
                            }
                        }
                    }
                }
            }
            Msg::title();
            $msg = Msg::new("Cluster information verification:\n", 40, 2383);
            $msg->bold;
            $msg=Msg::new("\tCluster Name: \t   $cname", 40, 2384, "$cname");
            $msg->print;
            $msg=Msg::new("\tCluster ID Number: $cid", 40, 2385, "$cid");
            $msg->print;
            foreach $sysi(@{$edr->{systems}}) {
                $sys=$sysi->{sys};
                $lmsg="";
                for $n (1..$prod->num_hbnics($rhbn)) {
                    $key="lltlink$n";
                    next unless ($rhbn->{$key}{$sys});
                    if ($cfg->{lltoverudp}) {
                        $address=$rhbn->{"udplink$n"."_address"}{$sys};
                        $netmask=$rhbn->{"udplink$n"."_netmask"}{$sys};
                        $port=$rhbn->{"udplink$n"."_port"}{$sys};
                        if (ip_is_ipv6($address)) {
                            $msg=Msg::new("\n\t\tlink$n=$rhbn->{$key}{$sys} over UDP6\n\t\t  ip $address prefix $netmask port $port", 40, 3716, "$n", "$rhbn->{$key}{$sys}", "$address", "$netmask", "$port");
                        } else {
                            $msg=Msg::new("\n\t\tlink$n=$rhbn->{$key}{$sys} over UDP\n\t\t  ip $address netmask $netmask port $port", 40, 3717, "$n", "$rhbn->{$key}{$sys}", "$address", "$netmask", "$port");
                        }
                        $lmsg.=$msg->{msg};
                    } else {
                        $msg=Msg::new("\n\t\tlink$n=$rhbn->{$key}{$sys}", 40, 3718, "$n", "$rhbn->{$key}{$sys}");
                        $lmsg.=$msg->{msg};
                    }
                }
                $msg=Msg::new("\tPrivate Heartbeat NICs for $sys: $lmsg", 40, 2386, "$sys", "$lmsg");
                $msg->print;
                if ($prod->num_lopri_hbnics($rhbn) > 0) {
                    $lmsg = "";
                    for $n (1..$prod->num_lopri_hbnics($rhbn)) {
                        $key="lltlinklowpri$n";
                        if ($cfg->{lltoverudp}) {
                            $address=$rhbn->{"udplinklowpri$n"."_address"}{$sys};
                            $netmask=$rhbn->{"udplinklowpri$n"."_netmask"}{$sys};
                            $port=$rhbn->{"udplinklowpri$n"."_port"}{$sys};
                            if (ip_is_ipv6($address)) {
                                $msg=Msg::new("\n\t\tlink-lowpri$n=$rhbn->{$key}{$sys} over UDP6\n\t\t  ip $address prefix $netmask port $port", 40, 3719, "$n", "$rhbn->{$key}{$sys}", "$address", "$netmask", "$port");
                            } else {
                                $msg=Msg::new("\n\t\tlink-lowpri$n=$rhbn->{$key}{$sys} over UDP\n\t\t  ip $address netmask $netmask port $port", 40, 3720, "$n", "$rhbn->{$key}{$sys}", "$address", "$netmask", "$port");
                            }
                            $lmsg.=$msg->{msg};
                        } else {
                            $msg=Msg::new("\n\t\tlink-lowpri$n=$rhbn->{$key}{$sys}", 40, 3721, "$n", "$rhbn->{$key}{$sys}");
                            $lmsg.=$msg->{msg};
                        }
                    }
                    if ($prod->num_lopri_hbnics($rhbn) > 1) {
                        $msg=Msg::new("\tLow-Priority Heartbeat NICs for $sys: $lmsg", 40, 3412, "$sys", "$lmsg");
                    } else {
                        $msg=Msg::new("\tLow-Priority Heartbeat NIC for $sys: $lmsg", 40, 3413, "$sys", "$lmsg");
                    }
                    $msg->print;
                }
            }
        } else {
            $msg=Msg::new("\tCluster Name: $cname", 40, 2388, "$cname");
            $msg->print;
        }

        Msg::n();
        $msg = Msg::new("Is this information correct?", 40, 1102);
        $help =Msg::new("Verification of the input", 40, 2389);
        $ayn = $msg->ayny($help,$backopt);
        $done=1 if ($ayn eq "Y");
    }
    unless (Cfg::opt("responsefile")) {
        if ($multinode) {
            $cfg->{vcs_clusterid}=$cid;
            $prod->set_hb_nics($rhbn,$edr->{systems});
        }
        $cfg->{vcs_clustername}=$cname;
    }
    delete($cfg->{autocfgllt});
}

sub set_hb_nics {
    my ($prod,$rhbn,$rsystems) = @_;
    my @systems = @$rsystems;
    
    for my $sys (@systems) {
        $prod->set_hb_nics_sys($rhbn,$sys);
    }
}

sub set_hb_nics_sys {
    my ($prod,$rhbn,$sysi) = @_;
    my $cfg = Obj::cfg();
    my ($n,$sys);
    
    if (ref($sysi) eq "Sys") {
        $sys = $sysi->{sys};
    } else {
        $sys = $sysi;
    }
    for $n (1..$prod->num_hbnics($rhbn)) {
        $cfg->{"vcs_lltlink$n"}{$sys}=$rhbn->{"lltlink$n"}{$sys};
        if ($cfg->{lltoverudp}) {
            $cfg->{"vcs_udplink$n"."_address"}{$sys}=$rhbn->{"udplink$n"."_address"}{$sys};
            $cfg->{"vcs_udplink$n"."_netmask"}{$sys}=$rhbn->{"udplink$n"."_netmask"}{$sys};
            $cfg->{"vcs_udplink$n"."_port"}{$sys}=$rhbn->{"udplink$n"."_port"}{$sys};
        }
    }
    for $n (1..$prod->num_lopri_hbnics($rhbn)) {
        $cfg->{"vcs_lltlinklowpri$n"}{$sys}=$rhbn->{"lltlinklowpri$n"}{$sys};
        if ($cfg->{lltoverudp}) {
            $cfg->{"vcs_udplinklowpri$n"."_address"}{$sys}=$rhbn->{"udplinklowpri$n"."_address"}{$sys};
            $cfg->{"vcs_udplinklowpri$n"."_netmask"}{$sys}=$rhbn->{"udplinklowpri$n"."_netmask"}{$sys};
            $cfg->{"vcs_udplinklowpri$n"."_port"}{$sys}=$rhbn->{"udplinklowpri$n"."_port"}{$sys};                    
        }
    }
}

sub check_uuidconfig_pl {
    my ($edr,$localsys,$prod,$result);
    $prod = shift;
    $edr = Obj::edr();
    $localsys = Obj::localsys();
    
    # check existance of uuidconfig.pl
    if($edr->{fromdisk}) {
        unless(EDR::file_sys($localsys,$prod->{uuidconfig})) {
            return 0;
        }
    } elsif (EDR::file_sys($localsys,"$edr->{mediapath}/scripts/uuidconfig.pl")) {
        $prod->{uuidconfig} = "$edr->{mediapath}/scripts/uuidconfig.pl";
    } elsif (EDR::file_sys($localsys,"$edr->{mediapath}/cluster_server/scripts/uuidconfig.pl")) {
        $prod->{uuidconfig} = "$edr->{mediapath}/cluster_server/scripts/uuidconfig.pl";
    } else {
        return 0;
    }
    return 1;
}

#check if all systems have the same uuid
sub check_uuid {
    my ($prod,@uuid_keys,$uuid_hash);
    $prod = shift;
    $uuid_hash=$prod->get_uuid();
    @uuid_keys=keys(%{$uuid_hash});
    return 1 if ($#uuid_keys == 0) && ($uuid_keys[0]!~/NO_UUID/);
    return 0;
}

sub config_uuid {
    my ($prod,$result);
    $prod = shift;
    return -1 unless ($prod->check_uuidconfig_pl());
    $result=$prod->check_configure_uuid();
    return $result;
}

#First,to check existing UUID,and then configure UUID according to the case of existing UUID.
sub check_configure_uuid {
    my ($prod,$rtn,$sys,$uuid_hash,$uuid_source,$uuid,$syslist,$nouuid,$uuid_count,@tmp);
    $prod = shift;
    $uuid_hash=$prod->get_uuid();
    $nouuid=0;
    $uuid_count=0;
    while (($uuid,$syslist) = each %{$uuid_hash}) {
        if($uuid=~/NO_UUID/){
            $nouuid=1;
         }else{
            $uuid_count=$uuid_count+1;
            $uuid_source=$syslist;
         }
    }
    # no uuid or more than 1 different uuids
    if($uuid_count!=1){
        $rtn = $prod->create_uuid();
    }elsif($nouuid){
        chomp($uuid_source);
        $syslist=EDR::despace($uuid_source);
        @tmp=split(/\s+/,$syslist);
        $sys=$tmp[0] unless($#tmp<0);
        $rtn = $prod->copy_uuid($sys,$uuid_hash->{NO_UUID});
    }else{
        # All sys have same uuids.
        $rtn=1;
    }
    return $rtn;
}

sub create_uuid {
    my ($cpic,$prod,$rtn,$sys,$sys1);
    $prod = shift;
    $cpic = Obj::cpic();
    $sys1=$cpic->{systems}->[0];
    $rtn = $prod->create_uuid_sys($sys1);
    unless($rtn){
        Msg::log("UUID could not be created successfully on the cluster");
        return 0;
    }
    foreach $sys (@{$cpic->{systems}}) {
        next if ($sys->system1);
        $prod->copy_uuid_2sys($sys1,$sys);
    }
    return 1;
}

sub get_uuid {
    my ($cpic,$prod,$sys,$uuids,%uuid_hash);
    $prod = shift;
    $cpic = Obj::cpic();
    for $sys (@{$cpic->{systems}}) {
        $uuids=$prod->get_uuid_sys($sys);
        $uuids||="NO_UUID";
        if(exists($uuid_hash{$uuids})){
            $uuid_hash{$uuids}=$uuid_hash{$uuids}." ".$sys->{sys};
        } else {
            $uuid_hash{$uuids}=$sys->{sys};
        }
    }
    return \%uuid_hash;
}

sub get_uuid_sys {
    my ($cpic,$prod,$sys,$uuid);
    ($prod,$sys) =@_;
    my $rootpath=Cfg::opt("rootpath") || "";
    $uuid=EDR::cmd_sys($sys,"_cmd_cat $rootpath$prod->{uuidfile}") if (EDR::file_sys($sys,"$rootpath$prod->{uuidfile}"));
    return $uuid;
}

sub copy_uuid_2sys{
    my ($cmd,$edr,$prod,$rootpath,$sys_src,$sys_dest,$uuidpath,$uuid,$uuidfile);
    ($prod,$sys_src,$sys_dest)=@_;
    return 0 unless(($sys_src) && ($sys_dest));
    $uuid=$prod->get_uuid_sys($sys_src);
    return $prod->set_uuid_2sys($sys_dest,$uuid);
}

sub set_uuid_2sys{
    my ($cmd,$edr,$prod,$rootpath,$sys_dest,$uuidpath,$uuid,$uuidfile);
    ($prod,$sys_dest,$uuid)=@_;
    return 0 unless($sys_dest);
    $uuidfile=$prod->{uuidfile};
    $rootpath = Cfg::opt("rootpath") || "";
    $uuidfile=$rootpath.$uuidfile;
    $uuidpath=$uuidfile;
    $uuidpath=~s/\/clusuuid//g;
    EDR::cmd_sys($sys_dest,"_cmd_mkdir -p $uuidpath");
    EDR::cmd_sys($sys_dest,"_cmd_echo $uuid > $uuidfile");
    if($edr->{cmdexit}){
        Msg::log("FAILED to set UUID to $sys_dest->{sys}");
        return 0;    
    }
    return 1;
}

sub create_uuid_sys {        
    my ($cpic,$edr,$prod,$sys,$syslist,$cmd,$rsh,$hostname);
    ($prod,$sys)=@_;        
    $edr = Obj::edr();          
    $cpic = Obj::cpic();        
    $hostname = ($sys->{islocal}) ? $sys->{hostname} : $sys->{sys};
    my $rootpath=Cfg::opt("rootpath") || "";
    $rsh=$sys->{rsh};       
    my $export_rootpath = ($rootpath) ?
        "INSTALL_ROOT_PATH=$rootpath; export INSTALL_ROOT_PATH;" : "";
    my $cmd_uuidconfig = $export_rootpath.$prod->{uuidconfig};
    if($rsh =~/rsh/) {          
        $cmd=$cmd_uuidconfig." -cpi -rsh -clus -configure -force $hostname";         
    }else {         
        $cmd=$cmd_uuidconfig." -cpi -clus -configure -force $hostname";          
    }       
    EDR::cmd_local($cmd);       
    if($edr->{cmdexit}){        
        Msg::log("UUID could not be created successfully on the cluster");         
        return 0;       
    }       
    return 1;       
 }       
     

sub copy_uuid {
    my ($prod,$sys_src,$src_obj,$sys_dest,$sys,@syslist,@sysnamelist);
    ($prod,$sys_src,$sys_dest)=@_;
    
    return 0 if(($sys_src eq  "") || ($sys_dest eq ""));
    $src_obj=($Obj::pool{"Sys::$sys_src"}) ? Obj::sys($sys_src) : EDR::tsub("Sys","new",$sys_src);
    @sysnamelist=split(/\s+/,$sys_dest);
    foreach my $sysname (@sysnamelist){
        $sys=($Obj::pool{"Sys::$sysname"}) ? Obj::sys($sysname) : EDR::tsub("Sys","new",$sysname);
        push(@syslist,$sys);
    }
    foreach $sys (@syslist){
        $prod->copy_uuid_2sys($src_obj,$sys);
    }
    return 1;
}

sub ask_clusterid {
    my ($prod,$def_id,$edr,$msg,$help,$backopt,$answer,$w_msg);
    ($prod,$def_id)=@_;
    my $cfg=Obj::cfg();
    return "" if (Cfg::opt("responsefile"));
    $edr = Obj::edr();
    $def_id = "" unless ($def_id);
    $msg = Msg::new("Enter a unique cluster ID number between 0-65535:", 40, 3414);
    $w_msg=Msg::new("The cluster ID number is wrong. Re-enter the cluster ID", 40, 3415); 
    $help = Msg::new("Each $prod->{abbr} cluster has a unique identifier that is an integer value between 0 and 65535. System will panic if it is configured with a Cluster identifier that is already being used by another cluster.", 40, 2394, "$prod->{abbr}");
    $backopt = 1;
    while (1) {
        $answer = $msg->ask($def_id,$help,$backopt);
        return "$answer" if ($answer eq $edr->{msg}{back});
        $answer=~s/^0+//;
        $answer||=0;
        if ((EDR::isint($answer) == 1) && ($answer>=0) && ($answer<=65535)){
           $cfg->{vcs_clusterid}=$answer;
           return "$answer"; 
        } else {
           $w_msg->print;
        }
    }
}

# ask for cluster name
sub ask_clustername {
    my ($prod,$def_name,$edr,$msg,$help,$backopt,$answer);
    ($prod,$def_name)=@_;
    my $cfg=Obj::cfg();
    return "" if (Cfg::opt("responsefile"));
    $msg = Msg::new("Enter the unique cluster name:", 40, 2395);
    $help = Msg::new("Each $prod->{abbr} cluster has a name which must begin with a letter of an alphabet. Only letters, numbers, or the characters - and _ are allowed in a cluster name.", 40, 2396, "$prod->{abbr}");
    $backopt = 0;
    while (1) {
        $answer = $msg->ask($def_name,$help,$backopt);
        if ($prod->verify_clustername($answer)) {
            $cfg->{vcs_clustername}=$answer;
            return $answer;
        }
    }
}

sub ask_bonded_nic {
    my ($prod,$nic,$msg,$ayn);
    ($prod,$nic)=@_;
    $msg=Msg::new("Is $nic a bonded NIC?", 40, 2397, "$nic");
    $ayn=$msg->aynn();
    return 1 if ($ayn eq "Y");
    return 0;    
}

sub hb_config_option {
    my($edr,$cfg,$help,$items,$msg,$option,$prod,$backopt,$defaultopt,$autocfg_failed,$rhbn);
    ($prod,$autocfg_failed) = @_;
    $edr = Obj::edr();
    $cfg = Obj::cfg();
    $msg = Msg::new("Configure heartbeat links using LLT over Ethernet", 40, 3416);
    push(@{$items},$msg->{msg});
    $msg = Msg::new("Configure heartbeat links using LLT over UDP", 40, 3417);
    push(@{$items},$msg->{msg});
    if (!Cfg::opt("makeresponsefile") && scalar(@{$edr->{systems}}) > 1 && !$autocfg_failed) {
        $msg = Msg::new("Automatically detect configuration for LLT over Ethernet", 40, 3418);
        push(@{$items},$msg->{msg});
    }
    if ( ! $autocfg_failed) {
        Msg::title();
    }
    $msg = Msg::new("How would you like to configure heartbeat links?", 40, 3419);
    $help = Msg::new("Configuring heartbeat links using LLT over Ethernet is the recommended option.\nLLT over UDP is slower than LLT over Ethernet. Use LLT over UDP only when the hardware configuration makes it necessary.", 40, 3420);
    $backopt = 1;
    if ($cfg->{lltoverudp}) {
        $defaultopt = '2';
    } elsif ($cfg->{autocfgllt} && !$autocfg_failed) {
        $defaultopt = '3';
    } else {
        $defaultopt = '1';
    }
    $option = $msg->menu($items, $defaultopt, $help, $backopt);
    if ($option eq "2") {
        $cfg->{lltoverudp} = 1;
        $cfg->{autocfgllt} = 0;
        return $prod->tsub("ask_hbnics");
    } elsif ($option eq "3") {
        $cfg->{lltoverudp} = 0;
        $cfg->{autocfgllt} = 1;
        # On Linux systems NICs need to be activated before DLPI can work correctly on them.
        if (EDR::plat($prod->{padv}) eq "Linux") {
            $msg=Msg::new("On Linux systems, only activated NICs could be detected and configured automatically.", 40, 3421);
            $msg->print;
            Msg::prtc();
        }
        $rhbn = $prod->tsub("autocfg_hbnics");
        if ($prod->num_hbnics($rhbn)< 1) {
            $msg=Msg::new("Failed to detect and configure LLT heartbeat links. Please configure LLT manually.", 40, 3392);
            $msg->n();
            $msg->bold();
            $msg->n();
            return $prod->tsub("hb_config_option","autocfg_failed");
        } else {
            return $rhbn;
        }
    } elsif ($option eq $edr->{msg}{back}) {
        return $option;
    } else {
        $cfg->{lltoverudp} = 0;
        $cfg->{autocfgllt} = 0;
        return $prod->tsub("ask_hbnics");
    }
}

# ask for all heartbeat links
sub ask_hbnics {
    my($cfg,$cpic,$edr,$padv,$prod,$sys,$sysi,$msg);
    my($all,$ayn,%hbn,@en,$dsn,$hb,$hb2,$hb3,$hb4,$hbl,$ip,$port,$rpn,$rsn,$udp_port,$used_port);
    return "" if (Cfg::opt("responsefile"));
    $prod=shift;
    $edr=Obj::edr();
    $cfg=Obj::cfg();
    $cpic=Obj::cpic();
    $used_port = [];
    foreach $sys(@{$edr->{systems}}) {
        $sysi=$sys->{sys};
        $padv=$sys->padv;
        if ($all) {
            $hbn{lltlink1}{$sysi}=$en[1];
            $hbn{lltlink2}{$sysi}=$en[2] if ($en[2]);
            $hbn{lltlink3}{$sysi}=$en[3] if ($en[3]);
            $hbn{lltlink4}{$sysi}=$en[4] if ($en[4]);
            $hbn{lltlinklowpri1}{$sysi}=$en[$prod->{max_hipri_links}+1] if ($en[$prod->{max_hipri_links}+1]);
            $cfg->{$sysi}{bonded_nics}=$cfg->{${$edr->{systems}}[0]->{sys}}{bonded_nics};
        } else {
            undef(@en);            
            $rsn=$rpn=[];
            if (EDR::inarr($sys,@{$edr->{systems}})) {
                Msg::n();
                $msg=Msg::new("Discovering NICs on $sysi", 40, 2398, "$sysi");
                $msg->left;
                $padv=$sys->padv;
                $rsn=$padv->systemnics_sys($sys,1);
                $rpn=$padv->publicnics_sys($sys);
                $rpn = [qw()] , $rsn = [ qw(NIC1 NIC2 NIC3 NIC4) ] if (Cfg::opt("demo"));
                EDR::arruniq(@$rsn);
                $dsn=join(" ",@$rsn);
                if ($#$rsn<0) {
                    $msg=Msg::new("No NICs discovered", 40, 2399);
                    $msg->right;
                } else {
                    $msg=Msg::new("Discovered $dsn", 40, 2400, "$dsn");
                    $msg->right;
                    #$msg=Msg::new("\nTo use aggregated interfaces for private heartbeat, enter the name of an aggregated interface. \nTo use a NIC for private heartbeat, enter a NIC which is not part of an aggregated interface.\n");
                    #$msg->print;
                }
                Msg::n();
            }
            # link 1
            $hb = $prod->ask_hbnic_sys($sys,1,$rsn,$rpn);
            return $hb if ($hb eq $edr->{msg}{back});
            $hbn{lltlink1}{$sysi} = $en[1] = $hb;
            if ($padv->is_bonded_nic_sys($sys,$hb)){
                push(@{$cfg->{$sys}{bonded_nics}},$hb);
            }
        }
        if ($cfg->{lltoverudp}) {
            $ip = $prod->ask_nic_ip_sys($sys,$en[1],1);
            return $ip if ($ip eq $edr->{msg}{back});
            $hbn{udplink1_address}{$sysi}=$ip->{address};
            $hbn{udplink1_netmask}{$sysi}=$ip->{netmask};
            if ($used_port->[1]) {
                $port = $used_port->[1];
                $msg=Msg::new("The UDP Port for this link: $port", 40, 3422, "$port");
                $msg->print;
                $hbn{udplink1_port}{$sysi} = $used_port->[1];
            } else {
                $udp_port = $prod->ask_udp_port_sys($sys,1,$used_port);
                return $udp_port if ($udp_port eq $edr->{msg}{back});
                $hbn{udplink1_port}{$sysi} = $used_port->[1] = $udp_port;
            }
        }
        # link 2
        unless ($all) {
            $hb2 = $prod->ask_second_hb;
            return $hb2 if ($hb2 eq $edr->{msg}{back});
            if ($hb2 eq "Y") {
                $hb = $prod->ask_hbnic_sys($sys,2,$rsn,$rpn,\@en);
                return $hb if ($hb eq $edr->{msg}{back});
                $hbn{lltlink2}{$sysi} = $en[2] = $hb;
                if ($padv->is_bonded_nic_sys($sys,$hb)){
                    push(@{$cfg->{$sysi}{bonded_nics}},$hb);
                }
            }
        }
        if ($cfg->{lltoverudp} && ($hb2 eq "Y")) {
            $ip = $prod->ask_nic_ip_sys($sys,$en[2],2);
            return $ip if ($ip eq $edr->{msg}{back});
            $hbn{udplink2_address}{$sysi}=$ip->{address};
            $hbn{udplink2_netmask}{$sysi}=$ip->{netmask};
            if ($used_port->[2]) {
                $port = $used_port->[2];
                $msg=Msg::new("The UDP Port for this link: $port", 40, 3422, "$port");
                $msg->print;
                $hbn{udplink2_port}{$sysi} = $used_port->[2];
            } else {
                $udp_port = $prod->ask_udp_port_sys($sys,2,$used_port);
                return $udp_port if ($udp_port eq $edr->{msg}{back});
                $hbn{udplink2_port}{$sysi} = $used_port->[2] = $udp_port;
            }
        }
        # link 3
        unless ($all) {
            if (($hb2 eq "Y") && ($#$rsn>2)) {
                $hb3=$prod->ask_third_hb;
                return $hb3 if ($hb3 eq $edr->{msg}{back});
            }
            if ($hb3 eq "Y") {
                $hb = $prod->ask_hbnic_sys($sys,3,$rsn,$rpn,\@en);
                return $hb if ($hb eq $edr->{msg}{back});
                $hbn{lltlink3}{$sysi} = $en[3] = $hb;
                if ($padv->is_bonded_nic_sys($sys,$hb)){
                    push(@{$cfg->{$sysi}{bonded_nics}},$hb);
                }
            }
        }
        if ($cfg->{lltoverudp} && ($hb3 eq "Y")) {
            $ip = $prod->ask_nic_ip_sys($sys,$en[3],3);
            return $ip if ($hb2 eq $edr->{msg}{back});
            $hbn{udplink3_address}{$sysi}=$ip->{address};
            $hbn{udplink3_netmask}{$sysi}=$ip->{netmask};
            if ($used_port->[3]) {
                $port = $used_port->[3];
                $msg=Msg::new("The UDP Port for this link: $port", 40, 3422, "$port");
                $msg->print;
                $hbn{udplink3_port}{$sysi} = $used_port->[3];
            } else {
                $udp_port = $prod->ask_udp_port_sys($sys,3,$used_port);
                return $udp_port if ($udp_port eq $edr->{msg}{back});
                $hbn{udplink3_port}{$sysi} = $used_port->[3] = $udp_port;
            }
        }
        # link 4
        unless ($all) {
            if (($hb3 eq "Y") && ($#$rsn>3)) {
                $hb4=$prod->ask_fourth_hb;
                return $hb4 if ($hb4 eq $edr->{msg}{back});
            }
            if ($hb4 eq "Y") {
                $hb = $prod->ask_hbnic_sys($sys,4,$rsn,$rpn,\@en);
                return $hb if ($hb eq $edr->{msg}{back});
                $hbn{lltlink4}{$sysi} = $en[4] = $hb;
                if ($padv->is_bonded_nic_sys($sys,$hb)){
                    push(@{$cfg->{$sysi}{bonded_nics}},$hb);
                }
            }
        }
        if ($cfg->{lltoverudp} && ($hb4 eq "Y")) {
            $ip = $prod->ask_nic_ip_sys($sys,$en[4],4);
            return $ip if ($ip eq $edr->{msg}{back});
            $hbn{udplink4_address}{$sysi}=$ip->{address};
            $hbn{udplink4_netmask}{$sysi}=$ip->{netmask};
            if ($used_port->[4]) {
                $port = $used_port->[4];
                $msg=Msg::new("The UDP Port for this link: $port", 40, 3422, "$port");
                $msg->print;
                $hbn{udplink4_port}{$sysi} = $used_port->[4];
            } else {
                $udp_port = $prod->ask_udp_port_sys($sys,4,$used_port);
                return $udp_port if ($udp_port eq $edr->{msg}{back});
                $hbn{udplink4_port}{$sysi} = $used_port->[4] = $udp_port;
            }
        }
        # link lowpri
        unless ($all) {
            if ($cfg->{fencingenabled} || ($hb2 eq "Y")) {
                $hbl = $prod->ask_lowpri_hb if ($cpic->{prod} !~ /SFRAC\d+/);
                return $hbl if ($hbl eq $edr->{msg}{back});
            } else {
                $hbl = "Y" if ($hb2 eq "N");
            }
            if ($hbl eq "Y") {
                $hb = $prod->ask_hbnic_sys($sys,"lowpri",$rsn,$rpn,\@en);
                return $hb if ($hb eq $edr->{msg}{back});
                $hbn{lltlinklowpri1}{$sysi} = $en[$prod->{max_hipri_links}+1] = $hb;
                if ($padv->is_bonded_nic_sys($sys,$hb)){
                    push(@{$cfg->{$sysi}{bonded_nics}},$hb);
                }
            }            
        }
        if ($cfg->{lltoverudp} && ($hbl eq "Y")) {
            $ip = $prod->ask_nic_ip_sys($sys,$en[$prod->{max_hipri_links}+1],"lowpri");
            return $ip if ($ip eq $edr->{msg}{back});
            $hbn{udplinklowpri1_address}{$sysi}=$ip->{address};
            $hbn{udplinklowpri1_netmask}{$sysi}=$ip->{netmask};
            if ($used_port->[$prod->{max_hipri_links}+1]) {
                $port = $used_port->[$prod->{max_hipri_links}+1];
                $msg=Msg::new("The UDP Port for this link: $port", 40, 3422, "$port");
                $msg->print;
                $hbn{udplinklowpri1_port}{$sysi} = $used_port->[$prod->{max_hipri_links}+1];
            } else {
                $udp_port = $prod->ask_udp_port_sys($sys,"lowpri",$used_port);
                return $udp_port if ($udp_port eq $edr->{msg}{back});
                $hbn{udplinklowpri1_port}{$sysi} = $used_port->[$prod->{max_hipri_links}+1] = $udp_port;
            }
        }

        if ($sys == ${$edr->{systems}}[0] && $#{$edr->{systems}} > 0 && !$all) {
            $ayn = $prod->ask_common_nics;
            return $ayn if ($ayn eq $edr->{msg}{back});
            $all = 1 if ($ayn eq "Y");
        }
    }
    return \%hbn;
}

# $active_nics (for Linux only, ignored on other Plats):
#  0 do not active NICs
#  1 active/deactive NICs
sub autocfg_hbnics {
    my ($prod, $active_nics, $a1,$all_connected, $b1,$cmd, $cmd_out, $connected_nic, $connected_nics, $cpic,$dev,$dev_dir,$dev_l,$dev_n,$done_msg,$edr, $gatewaynics_sys0, $key_high, $key_low, $last_sap, $link, $link_index, $link_num,$max_links, $msg, $n, $n_hipri, $n_lowpri,$n0, $nic, $nic_num, $non_nic_num, $padv, $plat,@resultfiles, $retry_count, $rhbn, $rnics, $rpids, $sap, $selected_speed, @sorted_links_index, @sorted_resultfiles, $speed_value, $sys, $sys0, $sysi, $sysi0, $tmpdir,$used_nics,$web,$mac,%sap_mac,$dlpi);
    ($prod, $active_nics) = @_;
    $edr = Obj::edr();
    $cpic = Obj::cpic();
    Msg::title();
    $web = Obj::web();
    $web->set_progress_steps(2+@{$edr->{systems}});
    $web->{stage} = Msg::new("Configuring LLT links", 40, 3423);
    $web->display_bold($web->{stage}) if (Cfg::opt(qw(redirect)));
        
    #get system nics on all systems.
    #create temp dir on all nodes and copy dlpiping binary into it.
    #so all dlpiping processes started by CPI could be identified by dirname.
    $tmpdir = EDR::tmpdir()."/dlpitest_".$edr->{localsys}->{sys}."_".$$;
    $plat=EDR::plat($prod->{padv});
    for $sys (@{$edr->{systems}}) {
        $sysi = $sys->{sys};
        $padv = $sys->padv;
        if (!EDR::file_sys($sys,"/opt/VRTSllt/dlpiping")) {
            Msg::log("/opt/VRTSllt/dlpiping command was not found on $sysi. VRTSllt package was not installed successfully.");
        }
        $rnics->{$sysi} = $padv->tsub("systemnics_sys",$sys,1);
        $cmd = "_cmd_rmr $tmpdir >/dev/null 2>&1 ;";
        $cmd .= "_cmd_mkdir -p $tmpdir >/dev/null 2>&1 ;";
        $cmd .= "_cmd_cp /opt/VRTSllt/dlpiping $tmpdir/dlpiping_cpi >/dev/null 2>&1 ;";
        EDR::cmd_sys($sys, $cmd);
        # on AIX, load dlpi if it has not been loaded yet.
        if ($plat eq "AIX") {
            $dlpi=EDR::cmd_sys($sys, "_cmd_strload -q -d /usr/lib/drivers/pse/dlpi | _cmd_awk '{print \$2}'");
            chomp($dlpi);
            if ($dlpi eq "no") {
                EDR::cmd_sys($sys, "_cmd_strload -f /etc/dlpi.conf");
            }
        }
    }
    
    # start dlpiping_cpi -s in background for each NIC on the first sys.
    $sys = $edr->{systems}->[0];
    $rpids = EDR::pids_sys($sys, "dlpiping_cpi");
    if (scalar @$rpids > 0) {
        $cmd = "_cmd_kill -s 9 ". join (" ", @$rpids) . " ; " ;
        EDR::cmd_sys($sys,$cmd);
    }
    $sysi = $sys->{sys};
    $msg = Msg::new("Checking system NICs on $sysi", 40, 3424, "$sysi");
    $web->tsub("display_left",$msg);
    $last_sap = 52000; # 52000 is magical initial sap value for dlpiping test.
    if ($plat ne "Linux") {
        if ($plat eq "AIX") {
            $dev_dir = "/dev/dlpi";
        } else {
            $dev_dir = "/dev";
        }
    }
    Msg::log("Found NICs on $sysi: @{$rnics->{$sysi}}");
    $non_nic_num = 0;
    $cmd = "cd $tmpdir; ";
    for $n (0..$#{$rnics->{$sysi}}) {
        $nic = $rnics->{$sysi}->[$n];
        $last_sap++;
        if ($nic =~ /sit\d+/ || $nic =~ /vsw\d+/){ # skip sit# on Linux, vsw# on Solaris
            $non_nic_num++;
            $sap_mac{$last_sap} = "00:00:00:00:00:00";
        } else {
            if ($plat eq "Linux") {
                $dev = $nic;
                $mac=EDR::cmd_sys($sys,"_cmd_ifconfig $nic | _cmd_grep 'HWaddr'");
                if ($mac=~/HWaddr\s+([0-9A-Fa-f:]+)[\s\n\$]/) {
                    $mac=$1;
                } else {
                    Msg::log("Failed get MAC address of $dev on $sysi.");
                    $mac="FF:FF:FF:FF:FF:FF";
                }
            } else {
                if ( $nic =~ /^(.*[A-Za-z])(\d+)$/) {
                    $dev = "$dev_dir/$1:$2";
                } else {
                    $dev = "$dev_dir/$nic";
                }
                $mac=EDR::cmd_sys($sys,"/opt/VRTSllt/getmac $dev | _cmd_grep '$dev'");
                $mac =~ s/^\s*//; #remove leading spaces if any.
                (undef, $mac) = split /\s/, $mac;
                chomp $mac;
            }
            if ($mac) {
                $sap_mac{$last_sap} = $mac;
            } else {
                Msg::log("Failed get MAC address of $dev on $sysi.");
                $sap_mac{$last_sap} = "FF:FF:FF:FF:FF:FF";
            }
            $cmd .= "(./dlpiping_cpi -s -v -d $last_sap $dev >/dev/null 2>&1 &);";
            if (length($cmd) > 1024) {
                EDR::cmd_sys($sys,$cmd);
                $cmd = "cd $tmpdir;";
            }
        }
    }
    EDR::cmd_sys($sys,$cmd);
    $nic_num = $#{$rnics->{$sysi}} + 1 - $non_nic_num;
    if ( $nic_num <= 1 ) {
        $msg=Msg::new("$nic_num NIC found", 40, 3425, "$nic_num");
    } else {
        $msg=Msg::new("$nic_num NICs found", 40, 3426, "$nic_num");
    }
    $web->tsub("display_right", $msg);
    
    # try dlpiping_cpi -c for all nics on all other nodes
    for $sys (@{$edr->{systems}}) {
        next if ($sys->system1);
        $sysi = $sys->{sys};
        $msg = Msg::new("Checking system NICs on $sysi", 40, 3424, "$sysi");
        $web->tsub("display_left",$msg);
        Msg::log("Found NICs on $sysi: @{$rnics->{$sysi}}");
        $cmd = "cd $tmpdir;";
        $non_nic_num = 0;
        for $n (0..$#{$rnics->{$sysi}}) {
            $nic = $rnics->{$sysi}->[$n];
            if ($nic =~ /sit\d+/ || $nic =~ /vsw\d+/){ # skip sit# on Linux, vsw# on Solaris
                $non_nic_num++;
            } else {
                if ($plat eq "Linux") {
                    $dev = $nic;
                } else {
                    $dev_l=$dev_n=$nic;
                    $dev_l=~s/(.*[A-Za-z])\d*/$1/g;
                    $dev_n=~s/.*[A-Za-z](\d*)$/$1/g;
                    $dev = "$dev_dir/$dev_l:$dev_n";
                    if ( $nic =~ /^(.*[A-Za-z])(\d+)$/) {
                        $dev = "$dev_dir/$1:$2";
                    } else {
                        $dev = "$dev_dir/$nic";
                    }
                    # make sure this NIC is detected.
                    EDR::cmd_sys($sys,"/opt/VRTSllt/getmac $dev");
                }
                for $sap (52001..$last_sap) {
                    $n0 = $sap-52001;
                    $cmd .= "(./dlpiping_cpi -c -t 10 -d $sap $dev $sap_mac{$sap} > result-$n"."-"."$n0"."- 2>&1 &);";
                    if (length($cmd) > 1024) {
                        EDR::cmd_sys($sys,$cmd);
                        $cmd = "cd $tmpdir;";
                    }
                }
            }
        }
        EDR::cmd_sys($sys,$cmd);
        sleep 3;
        $rpids = EDR::pids_sys($sys, "dlpiping_cpi");
        $retry_count = 10;
        while (scalar @$rpids > 0 && $retry_count > 0) {
            sleep 3;
            $retry_count--;
            $rpids = EDR::pids_sys($sys, "dlpiping_cpi");
        }
        $nic_num = $#{$rnics->{$sysi}} + 1 - $non_nic_num;
        if ( $nic_num <= 1 ) {
            $msg=Msg::new("$nic_num NIC found", 40, 3425, "$nic_num");
        } else {
            $msg=Msg::new("$nic_num NICs found", 40, 3426, "$nic_num");
        }
        $web->tsub("display_right", $msg);
    }


    # sleep a while and check result files.
    for $sys (@{$edr->{systems}}) {
        $sysi = $sys->{sys};
        next if ($sys->system1);
        # get content of each result file for debugging
        EDR::cmd_sys($sys, " ; for i in `_cmd_ls $tmpdir/result-*`;do echo \$i;_cmd_cat \$i;done 2>/dev/null");
        $cmd_out = EDR::cmd_sys($sys, "_cmd_grep -l 'is alive' $tmpdir/result-* 2>/dev/null");
        @resultfiles = split /\n/, $cmd_out;
        @sorted_resultfiles = sort {
            my $a1 = $a;
            $a1 =~ s/.*result-(\d+)-.*/$1/;
            my $b1 = $b;
            $b1 =~ s/.*result-(\d+)-.*/$1/;
            $a1 <=> $b1;
        } @resultfiles;
        $connected_nics->{$sysi} = \@sorted_resultfiles;
        Msg::log("Found connected links on $sysi: @sorted_resultfiles");
    }

    # Cleanup...
    for $sys (@{$edr->{systems}}) {
        $rpids = EDR::pids_sys($sys, "dlpiping_cpi");
        $cmd = "";
        if (scalar @$rpids > 0) {
            $cmd .= "_cmd_kill -s 9 ". join (" ", @$rpids) . " ; " ;
        }
        $cmd .= "_cmd_rmr $tmpdir >/dev/null 2>&1 ;" ;
        EDR::cmd_sys($sys, $cmd);
    }

    $msg = Msg::new("Checking network links", 40, 3427);
    $web->tsub("display_left",$msg);

    # parse results and create $rhbn
    $sys0 = $edr->{systems}->[0];
    $sysi0 = $sys0->{sys};
    $link_num = 0;
    for $n (0..$#{$rnics->{$sysi0}}) {
        $link->{$sysi0} = $n;
        $all_connected = 1;
        for $sys (@{$edr->{systems}}) {
            next if ($sys->system1);
            $sysi = $sys->{sys};
            $all_connected = 0;
            for $connected_nic (@{$connected_nics->{$sysi}}) {
                if ( $connected_nic =~ /result-(\d+)-$n-/) {
                    if (!EDR::inarr($1, @{$used_nics->{$sysi}})) {
                        $all_connected = 1;
                        $link->{$sysi} = $1;
                        last;
                    }
                }
            }
            if (!$all_connected) {
                last;
            }
        }

        if ($all_connected) {
            $link_num++;
            for $sys (@{$edr->{systems}}) {
                $sysi = $sys->{sys};
                $rhbn->{"link$link_num"}{$sysi} = $rnics->{$sysi}->[$link->{$sysi}];
                push @{$used_nics->{$sysi}}, $link->{$sysi};

                $key_high = "link$link_num"."_high";
                $key_low = "link$link_num"."_low";
                $selected_speed = $sys->padv()->nic_speed_sys($sys,$rhbn->{"link$link_num"}{$sysi});

                if ($selected_speed =~ /\D*(\d+)\s*[Gg]/) {
                    $speed_value = $1;
                    $speed_value *= 1000;
                } elsif ($selected_speed =~ /\D*(\d+)\D*/) {
                    $speed_value = $1;
                } else {
                    $speed_value = 0;
                }
                if ( ! defined($rhbn->{$key_high})) {
                    $rhbn->{$key_high} = $speed_value;
                } elsif ( $rhbn->{$key_high} < $speed_value ) {
                    $rhbn->{$key_high} = $speed_value;
                }
                if ( !defined($rhbn->{$key_low})) {
                    $rhbn->{$key_low} = $speed_value;
                } elsif ( $rhbn->{$key_low} > $speed_value ) {
                    $rhbn->{$key_low} = $speed_value;
                }
            }
        }
    }
    if ( $link_num == 1) {
        $msg=Msg::new("1 link found", 40, 3428);
        $web->tsub("display_right",$msg);
    } else {
        $msg=Msg::new("$link_num links found", 40, 3429, "$link_num");
        $web->tsub("display_right",$msg);
    }

    # sort links from high speed to low.
    @sorted_links_index= sort {
        $a1 = $rhbn->{"link$a"."_low"};
        $b1 = $rhbn->{"link$b"."_low"};
        if ($b1>$a1) {
            return 1;
        } elsif ($b1 == $a1) {
            return ($a<$b)?-1:1;
        } else {
            return -1;
        }
    } (1..$link_num);
    
    $msg = Msg::new("Setting link priority", 40, 3430);
    $web->tsub("display_left",$msg);

    # set priority of connected links.
    $n_hipri = 1;
    $n_lowpri = 1;
    $n = 0;
    $gatewaynics_sys0 = $sys0->padv->gatewaynics_sys($sys0);
    $max_links = $prod->{max_lltlinks} > $link_num ? $link_num : $prod->{max_lltlinks};
    while (($n<$link_num)&&($n_hipri + $n_lowpri < $prod->{max_lltlinks} + 2)) {
        $link_index=$sorted_links_index[$n];
        if (($cpic->{prod} =~ /SFRAC\d+/) && EDR::inarr($rhbn->{"link$link_index"}{$sysi0},@{$gatewaynics_sys0})) {
            # Do not use public NICs as heartbeat links for SFRAC
            Msg::log("Ignore public link over ".$rhbn->{"link$link_index"}{$sysi0}." on $sysi0 for SFRAC product");
        } elsif ($n_hipri>$prod->{max_hipri_links} || EDR::inarr($rhbn->{"link$link_index"}{$sysi0},@{$gatewaynics_sys0})) {
            # set lowpri if already had enough hipri ones, or, the NICs are connected to public network through gateway.
            for $sys(@{$edr->{systems}}) {
                $sysi = $sys->{sys};
                $rhbn->{"lltlinklowpri$n_lowpri"}{$sysi} = $rhbn->{"link$link_index"}{$sysi};
            }
            $rhbn->{"lltlinklowpri$n_lowpri"."_low"} = $rhbn->{"link$link_index"."_low"};
            $rhbn->{"lltlinklowpri$n_lowpri"."_high"} = $rhbn->{"link$link_index"."_high"};
            $n_lowpri++;
        } else {
            for $sys(@{$edr->{systems}}) {
                $sysi = $sys->{sys};
                $rhbn->{"lltlink$n_hipri"}{$sysi} = $rhbn->{"link$link_index"}{$sysi};
            }
            $rhbn->{"lltlink$n_hipri"."_low"} = $rhbn->{"link$link_index"."_low"};
            $rhbn->{"lltlink$n_hipri"."_high"} = $rhbn->{"link$link_index"."_high"};
            $n_hipri++;
        }
        $n++;
    }
    $rhbn = $prod->tsub("set_lowpri_for_slow_links",$rhbn);
    if ( $link_num < 1) {
        $msg = Msg::new("Skipped", 40, 1865);
        $web->tsub("display_right",$msg);
    } else {
        if ( $n_hipri == 1) {
            $msg = Msg::new("No high priority links found", 40, 3431);
            $web->tsub("display_right",$msg);
        } else {
            $web->tsub("display_right");
        }
    }
    sleep(3);
    return $rhbn;
}

#####################################
# Args:
#  $prod - Prod object
#  $sys  - Sys object
#  $rhbn - Reference to Heartbeat NICs hash for all systems
#  $rsns - Reference to system NICs hash for all systems
#  $rpns - Reference to public NICs hash for all systems
#  $min_hipri_links - Minimum number of hi-pri links required
#  $max_links - Maximum number of heartbeat links supported
#  $is_first_node - Whether this is the first node in cluster. The numbers of hipri/lowpri links are determined on the first node.
#     0 - no
#     1 - yes
#  $ask_for_nic_names - Whether ask user for the NIC names for each link. Alwasys ask for nic names on the first node.
#     0 - do not ask
#     1 - ask
#    -1 - do not ask anyway.
#  $is_udp - Whether it is an LLT link over UDP
#     0 - no
#     1 - yes
#     2 - ask. Used for mix configuration of LLT over UDP and LLT over Ethernet (future development).
#  $sysi0 - the name of the first node in cluster, used to retrieving its configuration info.
#  $no_lowpri_links - do not configure Low-Priority links. For SFRAC.
######################################
sub ask_hbnics_sys{
    my ($prod, $sys, $rhbn, $rsns, $rpns, $min_hipri_links, $max_links, $is_first_node, $ask_for_nic_names, $is_udp, $sysi0, $no_lowpri_links) = @_;
    my ($ask_nic,$cfg,$cpic,$edr,$sysi,$padv,@configured_nics,@configured_low_nics,@configured_udp_ports,$backopt, $help, $ip, $msg, $n,$nic,$port, $ayn, $selected_nic, $default_nic, $en, $pn, $sn, $dsn, $link, $min_lowpri_links,$udp);
    $edr = Obj::edr();
    $cfg = Obj::cfg();
    $cpic = Obj::cpic();
    $sysi = $sys->{sys};
    $padv = $sys->padv();
    if ($ask_for_nic_names) {
        Msg::n();
        $msg=Msg::new("Discovering NICs on $sysi", 40, 2398, "$sysi");
        $msg->left;
        $dsn=join(" ",@{$rsns->{$sysi}});
        if ($#{$rsns->{$sysi}}<0) {
            $msg=Msg::new("No NICs discovered", 40, 2399);
            $msg->right;
        } else {
            $msg=Msg::new("Discovered $dsn", 40, 2400, "$dsn");
            $msg->right;
            #$msg=Msg::new("\nTo use aggregated interfaces for private heartbeat, enter the name of an aggregated interface. \nTo use a NIC for private heartbeat, enter a NIC which is not part of an aggregated interface.\n");
            #$msg->print;
        }
        Msg::n();
    }

    #ask for hipri nics
    for $link (1..$max_links) {
        if ($link > $min_hipri_links && $link > $prod->{max_hipri_links}) {
            last;
        }
        if ($is_first_node && $link>$min_hipri_links) {
            $ayn = $prod->ask_nth_hb($link);
            return $ayn if ($ayn eq $edr->{msg}{back});
            last if ($ayn ne "Y");
        }
        if ((!$is_first_node) && $link>$prod->num_hbnics($rhbn)) {
            last;
        }

        #ask for nic name
        $ask_nic = 0;
        if ( $is_first_node ) {
            $ask_nic = 1;
            $default_nic = $prod->default_hbnic($rsns->{$sysi},$rpns->{$sysi},\@configured_nics);
        } elsif ($ask_for_nic_names > 0) {
            $ask_nic = 1;
            $default_nic = $rhbn->{"lltlink$link"}{$sysi0};
            if (!EDR::inarr($default_nic,@{$rsns->{$sysi}})) {
                $default_nic = $prod->default_hbnic($rsns->{$sysi},$rpns->{$sysi},\@configured_nics);
            }
        } elsif ($ask_for_nic_names < 0) {
            $ask_nic = 0;
            $selected_nic = $rhbn->{"lltlink$link"}{$sysi0};
        } elsif (!EDR::inarr($rhbn->{"lltlink$link"}{$sysi0},@{$rsns->{$sysi}})) {
            $ask_nic = 1;
            $default_nic = $rhbn->{"lltlink$link"}{$sysi0};
            $msg=Msg::new("$default_nic was specified for this private heartbeat link on $sysi0, but it is not a NIC discovered on $sysi", 40, 3432, "$default_nic", "$sysi0", "$sysi");
            $msg->print;
            Msg::n();
            $msg=Msg::new("Discovering NICs on $sysi", 40, 2398, "$sysi");
            $msg->left;
            $dsn=join(" ",@{$rsns->{$sysi}});
            if ($#{$rsns->{$sysi}}<0) {
                $msg=Msg::new("No NICs discovered", 40, 2399);
                $msg->right;
            } else {
                $msg=Msg::new("Discovered $dsn", 40, 2400, "$dsn");
                $msg->right;
                #$msg=Msg::new("\nTo use aggregated interfaces for private heartbeat, enter the name of an aggregated interface. \nTo use a NIC for private heartbeat, enter a NIC which is not part of an aggregated interface.\n");
                #$msg->print;
            }
            Msg::n();
            $default_nic = $prod->default_hbnic($rsns->{$sysi},$rpns->{$sysi},\@configured_nics);
        } elsif (EDR::inarr($rhbn->{"lltlink$link"}{$sysi0},@configured_nics)) {
            $ask_nic = 1;
            $default_nic = $prod->default_hbnic($rsns->{$sysi},$rpns->{$sysi},\@configured_nics);
            $n = EDR::arrpos($rhbn->{"lltlink$link"}{$sysi0}, @configured_nics);
            $nic = $rhbn->{"lltlink$link"}{$sysi0};
            $msg=Msg::new("$nic is already being used for another private heartbeat link", 40, 3433, "$nic");
            $msg->print;
        } else {
            $ask_nic = 0;
            $selected_nic = $rhbn->{"lltlink$link"}{$sysi0};
        }

        if ( $ask_nic ) {
            $selected_nic = $prod->ask_hbnic_sys($sys,$link,$rsns->{$sysi},$rpns->{$sysi},\@configured_nics,$default_nic);
            return $selected_nic if ($selected_nic eq $edr->{msg}{back});
        }
        $rhbn->{"lltlink$link"}{$sysi} = $selected_nic;
        $configured_nics[$link] = $selected_nic;
        
        # use udp for this link?
        $udp = 0;
        if ( $is_udp == 1) {
            $udp = 1;
        } elsif ( $is_udp == 2) {
            if ( $is_first_node) {
                $backopt = 1;
                $msg=Msg::new("Do you want to configure this private heartbeat link over UDP?", 40, 3434);
                $help = Msg::new("Configuring heartbeat link using LLT over Ethernet is the recommended option.\nLLT over UDP is slower than LLT over Ethernet. Use LLT over UDP only when the hardware configuration makes it necessary", 40, 3435);
                $ayn=$msg->aynn($help,$backopt);
                return $ayn if ($ayn eq $edr->{msg}{back});
                $udp = 1 if ($ayn eq "Y");
            }
            else {
                $udp = 1 if ($rhbn->{"lltlink$link"."_udp"});
            }
        }
        # ask for udp information.
        if ($udp) {
            $rhbn->{"lltlink$link"."_udp"} = 1;
            # ask for IP address
            $ip = $prod->ask_nic_ip_sys($sys,$rhbn->{"lltlink$link"}{$sysi},$link);
            return $ip if ($ip eq $edr->{msg}{back});
            $rhbn->{"udplink$link"."_address"}{$sysi} = $ip->{address};
            $rhbn->{"udplink$link"."_netmask"}{$sysi} = $ip->{netmask};
            # ask for port on the first node only
            if ($is_first_node) {
                $port = $prod->ask_udp_port_sys($sys,$link,\@configured_udp_ports);
                return $port if ($port eq $edr->{msg}{back});
                $rhbn->{"udplink$link"."_port"}{$sysi} = $port;
                push(@configured_udp_ports, $port);
            } else {
                $port = $rhbn->{"udplink$link"."_port"}{$sysi0};
                $msg=Msg::new("The UDP Port for this link: $port", 40, 3422, "$port");
                $msg->print;
                $rhbn->{"udplink$link"."_port"}{$sysi} = $rhbn->{"udplink$link"."_port"}{$sysi0};
            }
        }
    }

    if (!$no_lowpri_links) {
        # At least one lowpri link is required if:
        #   a) Only one hipri link is configured and,
        #   b) vxfen is not enabled.
        # Else lowpri links are optional.
        if ( $prod->num_hbnics($rhbn) == 1 && !$cfg->{fencingenabled} ) {
            $min_lowpri_links = 1;
        } else {
            $min_lowpri_links = 0;
        }
        
        #ask for lowpri nics
        for $link (1..$max_links-$prod->num_hbnics($rhbn)) {
            if ($is_first_node && $link > $min_lowpri_links) {
                $ayn = $prod->ask_lowpri_hb($link);
                return $ayn if ($ayn eq $edr->{msg}{back});
                last if ($ayn ne "Y");
            }
            if ((!$is_first_node) && $link>$prod->num_lopri_hbnics($rhbn)) {
                last;
            }
            #ask for nic name
            $ask_nic = 0;
            if ( $is_first_node ) {
                $ask_nic = 1;
                $default_nic = undef;
                for (@{$rpns->{$sysi}} ) {
                    if (EDR::inarr($_, @configured_nics)) {
                        next;
                    } else {
                        $default_nic = $_;
                        last;
                    }
                }
                if ( ! $default_nic) {
                    $default_nic = $prod->default_hbnic($rsns->{$sysi},$rpns->{$sysi},\@configured_nics);
                }
            } elsif ($ask_for_nic_names > 0) {
                $ask_nic = 1;
                $default_nic = $rhbn->{"lltlinklowpri$link"}{$sysi0};
                if (!EDR::inarr($default_nic,@{$rsns->{$sysi}})) {
                    $default_nic = undef;
                    for (@{$rpns->{$sysi}} ) {
                        if (EDR::inarr($_, @configured_nics)) {
                            next;
                        } else {
                            $default_nic = $_;
                            last;
                        }
                    }
                    if ( ! $default_nic) {
                        $default_nic = $prod->default_hbnic($rsns->{$sysi},$rpns->{$sysi},\@configured_nics);
                    }
                }
            } elsif ($ask_for_nic_names < 0) {
                $ask_nic = 0;
                $selected_nic = $rhbn->{"lltlinklowpri$link"}{$sysi0};
            } elsif (!EDR::inarr($rhbn->{"lltlinklowpri$link"}{$sysi0},@{$rsns->{$sysi}})) {
                $ask_nic = 1;
                $default_nic = $rhbn->{"lltlinklowpri$link"}{$sysi0};
                $msg=Msg::new("$default_nic was specified for this low-priority heartbeat link on $sysi0, but it is not a NIC discovered on $sysi", 40, 3436, "$default_nic", "$sysi0", "$sysi");
                $msg->print;
                $msg=Msg::new("Discovering NICs on $sysi", 40, 2398, "$sysi");
                $msg->left;
                $dsn=join(" ",@{$rsns->{$sysi}});
                if ($#{$rsns->{$sysi}}<0) {
                    $msg=Msg::new("No NICs discovered", 40, 2399);
                    $msg->right;
                } else {
                    $msg=Msg::new("Discovered $dsn", 40, 2400, "$dsn");
                    $msg->right;
                    #$msg=Msg::new("\nTo use aggregated interfaces for private heartbeat, enter the name of an aggregated interface. \nTo use a NIC for private heartbeat, enter a NIC which is not part of an aggregated interface.\n");
                    #$msg->print;
                }
                Msg::n();
                $default_nic = undef;
                for (@{$rpns->{$sysi}} ) {
                    if (EDR::inarr($_, @configured_nics)) {
                        next;
                    } else {
                        $default_nic = $_;
                        last;
                    }
                }
                if ( ! $default_nic) {
                    $default_nic = $prod->default_hbnic($rsns->{$sysi},$rpns->{$sysi},\@configured_nics);
                }
            } elsif (EDR::inarr($rhbn->{"lltlinklowpri$link"}{$sysi0},@configured_nics)) {
                $ask_nic = 1;
                $n = EDR::arrpos($rhbn->{"lltlinklowpri$link"}{$sysi0}, @configured_nics);
                $nic = $rhbn->{"lltlinklowpri$link"}{$sysi0};
                if ($n<=8) {
                    $msg=Msg::new("$nic is already being used for a private heartbeat link", 40, 3437, "$nic");
                } else {
                    $msg=Msg::new("$nic is already being used for another low-priority heartbeat link", 40, 3438, "$nic");
                }
                $msg->print;
                $default_nic = undef;
                for (@{$rpns->{$sysi}} ) {
                    if (EDR::inarr($_, @configured_nics)) {
                        next;
                    } else {
                        $default_nic = $_;
                        last;
                    }
                }
                if ( ! $default_nic) {
                    $default_nic = $prod->default_hbnic($rsns->{$sysi},$rpns->{$sysi},\@configured_nics);
                }
            } else {
                $ask_nic = 0;
                $selected_nic = $rhbn->{"lltlinklowpri$link"}{$sysi0};
            }

            if ( $ask_nic ) {
                $selected_nic = $prod->ask_hbnic_sys($sys,"lowpri$link",$rsns->{$sysi},$rpns->{$sysi},\@configured_nics,$default_nic);
                return $selected_nic if ($selected_nic eq $edr->{msg}{back});
            }
            $rhbn->{"lltlinklowpri$link"}{$sysi} = $selected_nic;
            $configured_nics[$link+8] = $selected_nic;
        
            # use udp for this link?
            $udp = 0;
            if ( $is_udp == 1) {
                $udp = 1;
            } elsif ( $is_udp == 2) {
                if ( $is_first_node) {
                    $backopt = 1;
                    $msg=Msg::new("Do you want to configure this low-priority heartbeat link over UDP?", 40, 3439);
                    $help = Msg::new("Configuring heartbeat link using LLT over Ethernet is the recommended option.\nLLT over UDP is slower than LLT over Ethernet. Use LLT over UDP only when the hardware configuration makes it necessary", 40, 3435);
                    $ayn=$msg->aynn($help,$backopt);
                    return $ayn if ($ayn eq $edr->{msg}{back});
                    $udp = 1 if ($ayn eq "Y");
                } else {
                    $udp = 1 if ($rhbn->{"lltlinklowpri$link"."_udp"});
                }
            }
            # ask for udp information.
            if ($udp) {
                $rhbn->{"lltlinklowpri$link"."_udp"} = 1;
                # ask for IP address
                $ip = $prod->ask_nic_ip_sys($sys,$rhbn->{"lltlinklowpri$link"}{$sysi},"lowpri$link");
                return $ip if ($ip eq $edr->{msg}{back});
                $rhbn->{"udplinklowpri$link"."_address"}{$sysi} = $ip->{address};
                $rhbn->{"udplinklowpri$link"."_netmask"}{$sysi} = $ip->{netmask};
            
                # ask for port
                if ($is_first_node) {
                    $port = $prod->ask_udp_port_sys($sys,"lowpri$link",\@configured_udp_ports);
                    return $port if ($port eq $edr->{msg}{back});
                    $rhbn->{"udplinklowpri$link"."_port"}{$sysi} = $port;
                    push(@configured_udp_ports, $port);
                } else {
                    $port = $rhbn->{"udplinklowpri$link"."_port"}{$sysi0};
                    $msg=Msg::new("The UDP Port for this link: $port", 40, 3422, "$port");
                    $msg->print;
                    $rhbn->{"udplinklowpri$link"."_port"}{$sysi} = $rhbn->{"udplinklowpri$link"."_port"}{$sysi0};
                }
            }
        }
    }
    
    return $rhbn;
}

# ask for all heartbeat links for added nodes
sub ask_hbnics_addnode {
    my($cfg,$cpic,$edr,$padv,$prod,$sys,$sysi,$sysi0,$msg);
    my($all,$ayn,%hbn,@en,@enlow,$dsn,$hb,$hb2,$hb3,$hb4,$hbl,$ip,$rpns,$rsns,$udp_port,$used_port,$rhbn,$min_nics,$min_nics_sys,$num_high, $num_low);
    my ($min_hipri_links, $max_links, $is_first_node, $ask_for_nic_names, $is_udp);
    return "" if (Cfg::opt("responsefile"));
    ($prod,$rhbn,$sysi0)=@_;
    $edr=Obj::edr();
    $cfg=Obj::cfg();
    $cpic=Obj::cpic();

    if ( ! $sysi0) {
        $sysi0 = ${$cfg->{clustersystems}}[0] if ($cfg->{clustersystems});
    }

    # get nics for all systems first in order to find out the maximum links could be setup
    $min_nics = -1;
    foreach $sys(@{$edr->{systems}}) {
        $sysi=$sys->{sys};
        $padv=$sys->padv;
        $rsns->{$sysi}=$padv->systemnics_sys($sys,1);
        $rpns->{$sysi}=$padv->publicnics_sys($sys);
        $rpns->{$sysi} = [qw(NIC1)] , $rsns->{$sysi} = [ qw(NIC1 NIC2 NIC3 NIC4) ] if (Cfg::opt("demo"));
        $rsns->{$sysi}=EDR::arruniq(@{$rsns->{$sysi}});
        $rpns->{$sysi}=EDR::arruniq(@{$rpns->{$sysi}});
        if ($min_nics < 0 || $min_nics > @{$rsns->{$sysi}}) {
            $min_nics = @{$rsns->{$sysi}};
            $min_nics_sys = $sysi;
        }
    }

    $msg = Msg::new("Each node to be added to the cluster should have the same LLT heartbeat configuration as the existing node(s).\n", 40, 3440);
    $msg->bold();
    $num_high = $prod->num_hbnics($rhbn);
    $num_low = $prod->num_lopri_hbnics($rhbn);
    if ($num_low>0) {
        $msg = Msg::new("To connect to the existing node(s) properly, each new node is required to be configured with $num_high private and $num_low low-priority heartbeat links.", 40, 3441, "$num_high", "$num_low");
    } else {
        $msg = Msg::new("To connect to the existing node(s) properly, each new node is required to be configured with $num_high private heartbeat link(s).", 40, 3442, "$num_high");
    }
    $msg->bold();
    
    $min_hipri_links = 1;
    if ( $cpic->{prod} =~ /SFRAC/ || $cpic->{prod} =~ /(SVS|SFCFS)/) {
        $min_hipri_links = 2;
    }
    $max_links = $prod->{max_lltlinks};
    $is_first_node = 0;
    $ask_for_nic_names = 1;
    if ( $cfg->{lltoverudp} ) {
        $is_udp = 1;
    } else {
        $is_udp = 0;
    }
    
    foreach $sys(@{$edr->{systems}}) {
        $sysi=$sys->{sys};
        if ($sys == ${$edr->{systems}}[0]){
            $ayn = $prod->ask_hbnics_sys($sys, $rhbn, $rsns,$rpns, $min_hipri_links, $max_links, $is_first_node, $ask_for_nic_names, $is_udp, $sysi0 );
            return $ayn if ($ayn eq $edr->{msg}{back});
            if ($#{$edr->{systems}} > 0 && $ask_for_nic_names) {
                $ayn = $prod->ask_common_nics;
                return $ayn if ($ayn eq $edr->{msg}{back});
                $ask_for_nic_names = 0 if ($ayn eq "Y");
            }
        }else {
            $ayn = $prod->ask_hbnics_sys($sys, $rhbn, $rsns,$rpns, $min_hipri_links, $max_links, 0, $ask_for_nic_names, $is_udp, ${$edr->{systems}}[0]->{sys});
            return $ayn if ($ayn eq $edr->{msg}{back});
        }
    }
    return $rhbn;
}

sub ask_nic_ip_sys {
    my ($edr,$help,$msg,$padv,$prod,$question,$sys,$sysi);
    my ($address,$answer,$id,%ip,$ips,$nic,$netmask,@pool);
    ($prod,$sys,$nic,$id) = @_;
    $edr = Obj::edr();
    $padv = $sys->padv();
    $sysi = $sys->{sys};
    while (1) {        
        $ips=$padv->tsub("nic_ips_sys",$sys,$nic);
        @pool = @$ips if(@$ips);
        $address = shift(@pool);
        $help = Msg::new("Configuring a heartbeat link using LLT over UDP requires a permanent IP address assigned to the selected heartbeat NIC", 40, 3443);
        while ($address) {
            if ($id =~ /lowpri/) {
                if ($id eq "lowpri" || $id eq "lowpri1") {
                    $question = Msg::new("Do you want to use address $address for the low-priority heartbeat link on $sysi:", 40, 3444, "$address", "$sysi");
                } elsif ($id eq "lowpri2") {
                    $question = Msg::new("Do you want to use address $address for the second low-priority heartbeat link on $sysi:", 40, 3445, "$address", "$sysi");
                } elsif ($id eq "lowpri3") {
                    $question = Msg::new("Do you want to use address $address for the third low-priority heartbeat link on $sysi:", 40, 3446, "$address", "$sysi");
                } elsif ($id eq "lowpri4") {
                    $question = Msg::new("Do you want to use address $address for the fourth low-priority heartbeat link on $sysi:", 40, 3447, "$address", "$sysi");
                } elsif ($id eq "lowpri5") {
                    $question = Msg::new("Do you want to use address $address for the fifth low-priority heartbeat link on $sysi:", 40, 3448, "$address", "$sysi");
                } elsif ($id eq "lowpri6") {
                    $question = Msg::new("Do you want to use address $address for the sixth low-priority heartbeat link on $sysi:", 40, 3449, "$address", "$sysi");
                } elsif ($id eq "lowpri7") {
                    $question = Msg::new("Do you want to use address $address for the seventh low-priority heartbeat link on $sysi:", 40, 3450, "$address", "$sysi");
                } elsif ($id eq "lowpri8") {
                    $question = Msg::new("Do you want to use address $address for the eighth low-priority heartbeat link on $sysi:", 40, 3451, "$address", "$sysi");
                }
            }elsif ($id == 1) {
                $question = Msg::new("Do you want to use address $address for the first private heartbeat link on $sysi:", 40, 3452, "$address", "$sysi");
            } elsif ($id == 2) {
                $question = Msg::new("Do you want to use address $address for the second private heartbeat link on $sysi:", 40, 3453, "$address", "$sysi");
            } elsif ($id == 3) {
                $question = Msg::new("Do you want to use address $address for the third private heartbeat link on $sysi:", 40, 3454, "$address", "$sysi");
            } elsif ($id == 4) {
                $question = Msg::new("Do you want to use address $address for the fourth private heartbeat link on $sysi:", 40, 3455, "$address", "$sysi");
            } elsif ($id == 5) {
                $question = Msg::new("Do you want to use address $address for the fifth private heartbeat link on $sysi:", 40, 3456, "$address", "$sysi");
            } elsif ($id == 6) {
                $question = Msg::new("Do you want to use address $address for the sixth private heartbeat link on $sysi:", 40, 3457, "$address", "$sysi");
            } elsif ($id == 7) {
                $question = Msg::new("Do you want to use address $address for the seventh private heartbeat link on $sysi:", 40, 3458, "$address", "$sysi");
            } elsif ($id == 8) {
                $question = Msg::new("Do you want to use address $address for the eighth private heartbeat link on $sysi:", 40, 3459, "$address", "$sysi");
            }
            $answer = $question->ayny($help,1);
            return $answer if ($answer eq $edr->{msg}{back});
            if ($answer eq "Y") {
                $ip{address} = $address;
                $ip{netmask} = EDR::defaultnetmask_sys($sys,$address,$nic);
                return \%ip;
            } else {
                $address = shift(@pool);
            }
        }
        if (@$ips) {
            $msg = Msg::new("A permanent IP address must be selected for this heartbeat link.", 40, 3460);
            $msg->bold();
        } else {
            $msg = Msg::new("$edr->{script} did not detect any IP address on $nic", 40, 3461, "$edr->{script}", "$nic");
            $msg->bold();
        }
        $msg = Msg::new("Configuring a heartbeat link using LLT over UDP requires a permanent IP address assigned to the selected heartbeat NIC. Make sure $nic has an permanent IP address. Press y to re-detect, or press n to choose another NIC\n", 40, 3462, "$nic");
        $msg->print();
        $question = Msg::new("Do you want to re-detect $nic on $sysi?", 40, 3463, "$nic", "$sysi");
        $answer = $question->ayny();
        return $edr->{msg}{back} if ($answer eq "N");
    }
}

sub ask_udp_port_sys {
    my ($def_port,$edr,$help,$id,$msg,$port,$prod,$question,$sys,$sysi,$used_port);
    ($prod,$sys,$id,$used_port) = @_;
    $edr = Obj::edr();
    $sysi = $sys->{sys};
    $help = Msg::new("Please input an available 16-bit integer from the range that follows:\n  Use available ports in the private range 49152 to 65535.\n  Do not use the following ports:\n    Ports from the range of well-known ports, 0 to 1023.\n    Ports from the range of registered ports, 1024 to 49151.\nYou can use the netstat command to list the UDP ports currently in use.", 40, 3464);
    if ($id =~ /lowpri/) {
        if ($id eq "lowpri" || $id eq "lowpri1") {
            $question = Msg::new("Enter the UDP port for the low-priority heartbeat link on $sysi:", 40, 3465, "$sysi");
            $def_port = "50004"; # for capability, will be increased automatically if it has been used. 
        } elsif ($id eq "lowpri2") {
            $question = Msg::new("Enter the UDP port for the second low-priority heartbeat link on $sysi:", 40, 3466, "$sysi");
            $def_port = "50009";
        } elsif ($id eq "lowpri3") {
            $question = Msg::new("Enter the UDP port for the third low-priority heartbeat link on $sysi:", 40, 3467, "$sysi");
            $def_port = "50010";
        } elsif ($id eq "lowpri4") {
            $question = Msg::new("Enter the UDP port for the fourth low-priority heartbeat link on $sysi:", 40, 3468, "$sysi");
            $def_port = "50011";
        } elsif ($id eq "lowpri5") {
            $question = Msg::new("Enter the UDP port for the fifth low-priority heartbeat link on $sysi:", 40, 3469, "$sysi");
            $def_port = "50012";
        } elsif ($id eq "lowpri6") {
            $question = Msg::new("Enter the UDP port for the sixth low-priority heartbeat link on $sysi:", 40, 3470, "$sysi");
            $def_port = "50013";
        } elsif ($id eq "lowpri7") {
            $question = Msg::new("Enter the UDP port for the seventh low-priority heartbeat link on $sysi:", 40, 3471, "$sysi");
            $def_port = "50014";
        } elsif ($id eq "lowpri8") {
            $question = Msg::new("Enter the UDP port for the eighth low-priority heartbeat link on $sysi:", 40, 3472, "$sysi");
            $def_port = "50015";
        }
    } elsif ($id == 1) {
        $question = Msg::new("Enter the UDP port for the first private heartbeat link on $sysi:", 40, 3473, "$sysi");
        $def_port = "50000";
    } elsif ($id == 2) {
        $question = Msg::new("Enter the UDP port for the second private heartbeat link on $sysi:", 40, 3474, "$sysi");
        $def_port = "50001";
    } elsif ($id == 3) {
        $question = Msg::new("Enter the UDP port for the third private heartbeat link on $sysi:", 40, 3475, "$sysi");
        $def_port = "50002";
    } elsif ($id == 4) {
        $question = Msg::new("Enter the UDP port for the fourth private heartbeat link on $sysi:", 40, 3476, "$sysi");
        $def_port = "50003";
    } elsif ($id == 5) {
        $question = Msg::new("Enter the UDP port for the fifth private heartbeat link on $sysi:", 40, 3477, "$sysi");
        $def_port = "50004";
    } elsif ($id == 6) {
        $question = Msg::new("Enter the UDP port for the sixth private heartbeat link on $sysi:", 40, 3478, "$sysi");
        $def_port = "50005";
    } elsif ($id == 7) {
        $question = Msg::new("Enter the UDP port for the seventh private heartbeat link on $sysi:", 40, 3479, "$sysi");
        $def_port = "50006";
    } elsif ($id == 8) {
        $question = Msg::new("Enter the UDP port for the eighth private heartbeat link on $sysi:", 40, 3480, "$sysi");
        $def_port = "50007";
    }
    while (EDR::inarr($def_port,@$used_port)) {
        $def_port++;
    }
    while (1) {
        $port = $question->ask($def_port,$help,1);
        return $port if ($port eq $edr->{msg}{back});
        if (EDR::isint($port) && ($port>0)&& ($port < 65536)) {
            if (EDR::inarr($port,@$used_port)) {
                $msg = Msg::new("UDP port $port is already used by another private heartbeat link on $sysi", 40, 3481, "$port", "$sysi");
                $msg->print();
                next;
            } else {
                last;
            }
        } else {
            $msg = Msg::new("$port is not a valid UDP port", 40, 3482, "$port");
            $msg->print();
        }
    }
    return $port;
}

sub default_hbnic {
    my ($nic);
    my ($prod,$rsn,$rpn,$ren) = @_;
    my $cfg = Obj::cfg();
    if ($cfg->{lltoverudp}) {
        for $nic (@$rsn) {
            next if ($nic eq $$rpn[0]);
            next if ($ren && EDR::inarr($nic,@$ren));
            return $nic;
        }
        
    } else {
        for $nic (@$rsn) {
            next if (EDR::inarr($nic,@$rpn));
            next if ($ren && EDR::inarr($nic,@$ren));
            return $nic;
        }        
    }
    return undef;
}

# ask for a heartbeat nic
sub ask_hbnic_sys {
    my($ayn,$cfg,$default_nic,$en,$fst,$max_hipri_links,$nic,$pn,$ren,$rpn,$rsn,$sn,$edr,$prod,$padv,$sysi,$sys,$def,$help,$backopt,$msg);
    ($prod,$sysi,$fst,$rsn,$rpn,$ren,$default_nic)=(@_);
    $sys=$sysi->{sys};
    $padv=$sysi->padv;
    $edr=Obj::edr();
    $cfg=Obj::cfg();
    $backopt=1;
    if ($fst =~ /lowpri/) {
        $help=Msg::new("The NIC selected as a low-priority heartbeat link is typically the network card on which the system's public IP address is active", 40, 3483);
    } else {
        $help=Msg::new("The NIC selected as a private heartbeat link is typically not plumbed or configured with an IP address. NIC devices used for each heartbeat link must have a private connection using network switches, hubs, or crossover cables.\n\nEnter any NIC at the prompt if a NIC device on your system is not properly discovered.", 40, 3484);
    }
    while (!$nic) {
        if ($default_nic) {
            $def = $default_nic;
        } else {
            $def=$prod->default_hbnic($rsn,$rpn,$ren);
        }
        if ($fst =~ /lowpri/) {
            $def = $default_nic||$rpn->[0];
            if ($fst eq "lowpri" || $fst eq "lowpri1") {
                $msg=Msg::new("Enter the NIC for the low-priority heartbeat link on $sys:", 40, 3485, "$sys");
                $en=$msg->ask($def,$help,$backopt);
            } elsif ( $fst eq "lowpri2" ) {
                $msg=Msg::new("Enter the NIC for the second low-priority heartbeat link on $sys:", 40, 3486, "$sys");
                $en=$msg->ask($def,$help,$backopt);
            } elsif ( $fst eq "lowpri3" ) {
                $msg=Msg::new("Enter the NIC for the third low-priority heartbeat link on $sys:", 40, 3487, "$sys");
                $en=$msg->ask($def,$help,$backopt);
            } elsif ( $fst eq "lowpri4" ) {
                $msg=Msg::new("Enter the NIC for the fourth low-priority heartbeat link on $sys:", 40, 3488, "$sys");
                $en=$msg->ask($def,$help,$backopt);
            } elsif ( $fst eq "lowpri5" ) {
                $msg=Msg::new("Enter the NIC for the fifth low-priority heartbeat link on $sys:", 40, 3489, "$sys");
                $en=$msg->ask($def,$help,$backopt);
            } elsif ( $fst eq "lowpri6" ) {
                $msg=Msg::new("Enter the NIC for the sixth low-priority heartbeat link on $sys:", 40, 3490, "$sys");
                $en=$msg->ask($def,$help,$backopt);
            } elsif ( $fst eq "lowpri7" ) {
                $msg=Msg::new("Enter the NIC for the seventh low-priority heartbeat link on $sys:", 40, 3491, "$sys");
                $en=$msg->ask($def,$help,$backopt);
            } elsif ( $fst eq "lowpri8" ) {
                $msg=Msg::new("Enter the NIC for the eighth low-priority heartbeat link on $sys:", 40, 3492, "$sys");
                $en=$msg->ask($def,$help,$backopt);
            }
        } elsif ($fst==1) {
            $msg=Msg::new("Enter the NIC for the first private heartbeat link on $sys:", 40, 2405, "$sys");
            $en=$msg->ask($def,$help,$backopt);
        } elsif ($fst==2) {
            $msg=Msg::new("Enter the NIC for the second private heartbeat link on $sys:", 40, 2406, "$sys");
            $en=$msg->ask($def,$help,$backopt);
        } elsif ($fst==3) {
            $msg=Msg::new("Enter the NIC for the third private heartbeat link on $sys:", 40, 2407, "$sys");
            $en=$msg->ask($def,$help,$backopt);
        } elsif ($fst==4) {
            $msg=Msg::new("Enter the NIC for the fourth private heartbeat link on $sys:", 40, 2408, "$sys");
            $en=$msg->ask($def,$help,$backopt);
        } elsif ($fst==5) {
            $msg=Msg::new("Enter the NIC for the fifth private heartbeat link on $sys:", 40, 3493, "$sys");
            $en=$msg->ask($def,$help,$backopt);
        } elsif ($fst==6) {
            $msg=Msg::new("Enter the NIC for the sixth private heartbeat link on $sys:", 40, 3494, "$sys");
            $en=$msg->ask($def,$help,$backopt);
        } elsif ($fst==7) {
            $msg=Msg::new("Enter the NIC for the seventh private heartbeat link on $sys:", 40, 3495, "$sys");
            $en=$msg->ask($def,$help,$backopt);
        } elsif ($fst==8) {
            $msg=Msg::new("Enter the NIC for the eighth private heartbeat link on $sys:", 40, 3496, "$sys");
            $en=$msg->ask($def,$help,$backopt);
        }
        return $en if ($en eq $edr->{msg}{back});
        $en=EDR::despace($en);
        $pn=EDR::arrpos($en,@$rpn);
        $sn=EDR::arrpos($en,@$rsn);
        $max_hipri_links = $prod->{max_hipri_links};
        if ($en eq "") {
        } elsif ($en =~/\s+/) {
            $msg=Msg::new("Only one NIC name is needed at a time", 40, 2409);
            $msg->print;
        #} elsif ($padv->nic_sys($sysi, $en)) {
        } elsif (EDR::isnic($en) == 0) {
            $msg=Msg::new("$en is not a valid NIC name", 40, 2334, "$en");
            $msg->print;
        } elsif ($en eq $ren->[$max_hipri_links+1]) {
            $msg=Msg::new("$en is already being used for the first low-priority heartbeat link", 40, 3497, "$en");
            $msg->print;
        } elsif ($en eq $ren->[$max_hipri_links+2]) {
            $msg=Msg::new("$en is already being used for the second low-priority heartbeat link", 40, 3498, "$en");
            $msg->print;
        } elsif ($en eq $ren->[$max_hipri_links+3]) {
            $msg=Msg::new("$en is already being used for the third low-priority heartbeat link", 40, 3499, "$en");
            $msg->print;
        } elsif ($en eq $ren->[$max_hipri_links+4]) {
            $msg=Msg::new("$en is already being used for the fourth low-priority heartbeat link", 40, 3500, "$en");
            $msg->print;
        } elsif ($en eq $ren->[$max_hipri_links+5]) {
            $msg=Msg::new("$en is already being used for the fifth low-priority heartbeat link", 40, 3501, "$en");
            $msg->print;
        } elsif ($en eq $ren->[$max_hipri_links+6]) {
            $msg=Msg::new("$en is already being used for the sixth low-priority heartbeat link", 40, 3502, "$en");
            $msg->print;
        } elsif ($en eq $ren->[$max_hipri_links+7]) {
            $msg=Msg::new("$en is already being used for the seventh low-priority heartbeat link", 40, 3503, "$en");
            $msg->print;
        } elsif ($en eq $ren->[$max_hipri_links+8]) {
            $msg=Msg::new("$en is already being used for the eighth low-priority heartbeat link", 40, 3504, "$en");
            $msg->print;
        } elsif ($en eq $ren->[1]) {
            $msg=Msg::new("$en is already being used for the first private heartbeat link", 40, 2410, "$en");
            $msg->print;
        } elsif ($en eq $ren->[2]) {
            $msg=Msg::new("$en is already being used for the second private heartbeat link", 40, 2411, "$en");
            $msg->print;
        } elsif ($en eq $ren->[3]) {
            $msg=Msg::new("$en is already being used for the third private heartbeat link", 40, 2412, "$en");
            $msg->print;
        } elsif ($en eq $ren->[4]) {
            $msg=Msg::new("$en is already being used for the fourth private heartbeat link", 40, 2413, "$en");
            $msg->print;
        } elsif ($en eq $ren->[5]) {
            $msg=Msg::new("$en is already being used for the fifth private heartbeat link", 40, 3505, "$en");
            $msg->print;
        } elsif ($en eq $ren->[6]) {
            $msg=Msg::new("$en is already being used for the sixth private heartbeat link", 40, 3506, "$en");
            $msg->print;
        } elsif ($en eq $ren->[7]) {
            $msg=Msg::new("$en is already being used for the seventh private heartbeat link", 40, 3507, "$en");
            $msg->print;
        } elsif ($en eq $ren->[8]) {
            $msg=Msg::new("$en is already being used for the eighth private heartbeat link", 40, 3508, "$en");
            $msg->print;
        } elsif ((($pn>=0) && ($fst !~ /lowpri/)) && !($cfg->{lltoverudp})|| 
            (($#$rsn>=0) && ($sn<0))) {
            if (($pn>=0) && ($fst !~ /lowpri/)) {
                $msg=Msg::new("$en has an IP address configured on it. It could be a public NIC on $sys.", 40, 2414, "$en", "$sys");
                $msg->print;
            } else {
                $msg=Msg::new("$en is not a NIC discovered on $sys", 40, 2415, "$en", "$sys");
                $msg->print;
                next unless (Cfg::opt("demo"));
            }
            $help=Msg::new("The NIC selected as a private heartbeat link is typically not plumbed or configured with an IP address. NIC devices used for each heartbeat link must have a private connection using network switches, hubs, or crossover cables.", 40, 2416);
            if ($fst eq 1) {
                $msg=Msg::new("Are you sure you want to use $en for the first private heartbeat link?", 40, 2417, "$en");
                $ayn=$msg->aynn($help,$backopt);
            } elsif ($fst eq 2) {
                $msg=Msg::new("Are you sure you want to use $en for the second private heartbeat link?", 40, 2418, "$en");
                $ayn=$msg->aynn($help,$backopt);
            } elsif ($fst eq 3) {
                $msg=Msg::new("Are you sure you want to use $en for the third private heartbeat link?", 40, 2419, "$en");
                $ayn=$msg->aynn($help,$backopt);
            } elsif ($fst eq 4) {
                $msg=Msg::new("Are you sure you want to use $en for the fourth private heartbeat link?", 40, 2420, "$en");
                $ayn=$msg->aynn($help,$backopt);
            } elsif ($fst eq 5) {
                $msg=Msg::new("Are you sure you want to use $en for the fifth private heartbeat link?", 40, 3509, "$en");
                $ayn=$msg->aynn($help,$backopt);
            } elsif ($fst eq 6) {
                $msg=Msg::new("Are you sure you want to use $en for the sixth private heartbeat link?", 40, 3510, "$en");
                $ayn=$msg->aynn($help,$backopt);
            } elsif ($fst eq 7) {
                $msg=Msg::new("Are you sure you want to use $en for the seventh private heartbeat link?", 40, 3511, "$en");
                $ayn=$msg->aynn($help,$backopt);
            } elsif ($fst eq 8) {
                $msg=Msg::new("Are you sure you want to use $en for the eighth private heartbeat link?", 40, 3512, "$en");
                $ayn=$msg->aynn($help,$backopt);
            } else {
                if ($fst eq "lowpri" || $fst eq "lowpri1") {
                    $msg=Msg::new("Are you sure you want to use $en for the low-priority heartbeat link?", 40, 3513, "$en");
                    $ayn=$msg->aynn($help,$backopt);
                } elsif ($fst eq "lowpri2") {
                    $msg=Msg::new("Are you sure you want to use $en for the second low-priority heartbeat link?", 40, 3514, "$en");
                    $ayn=$msg->aynn($help,$backopt);
                } elsif ($fst eq "lowpri3") {
                    $msg=Msg::new("Are you sure you want to use $en for the third low-priority heartbeat link?", 40, 3515, "$en");
                    $ayn=$msg->aynn($help,$backopt);
                } elsif ($fst eq "lowpri4") {
                    $msg=Msg::new("Are you sure you want to use $en for the fourth low-priority heartbeat link?", 40, 3516, "$en");
                    $ayn=$msg->aynn($help,$backopt);
                } elsif ($fst eq "lowpri5") {
                    $msg=Msg::new("Are you sure you want to use $en for the fifth low-priority heartbeat link?", 40, 3517, "$en");
                    $ayn=$msg->aynn($help,$backopt);
                } elsif ($fst eq "lowpri6") {
                    $msg=Msg::new("Are you sure you want to use $en for the sixth low-priority heartbeat link?", 40, 3518, "$en");
                    $ayn=$msg->aynn($help,$backopt);
                } elsif ($fst eq "lowpri7") {
                    $msg=Msg::new("Are you sure you want to use $en for the seventh low-priority heartbeat link?", 40, 3519, "$en");
                    $ayn=$msg->aynn($help,$backopt);
                } elsif ($fst eq "lowpri8") {
                    $msg=Msg::new("Are you sure you want to use $en for the eighth low-priority heartbeat link?", 40, 3520, "$en");
                    $ayn=$msg->aynn($help,$backopt);
                }
            }
            return $ayn if ($ayn eq $edr->{msg}{back});
            $nic=$en if ($ayn eq "Y");
        } else {
            $nic=$en;
        }
    }
    return "$nic";
}

# Ask about a second heartbeat link
sub ask_second_hb {
    my ($prod,$cfg,$ayn,$msg,$help,$backopt);
    $prod=shift;
    $cfg = Obj::cfg();
    $backopt=1;
    $help = Msg::new("A $prod->{abbr} cluster is typically configured to use two private heartbeat links, but some $prod->{abbr} clusters may be configured with just one private heartbeat link and a second low-priority heartbeat link. The NIC selected as a low-priority heartbeat link is typically the network card on which the system's public IP address is active.", 40, 3521, "$prod->{abbr}", "$prod->{abbr}");
    $msg = Msg::new("Would you like to configure a second private heartbeat link?", 40, 2423);
    if ($cfg->{fencingenabled}) {
        $ayn = $msg->aynn($help,$backopt);
    } else {
        $ayn = $msg->ayny($help,$backopt);
    }
    return $ayn;
}

# Ask about a third heartbeat link
sub ask_third_hb {
    my ($prod,$ayn,$msg,$help,$backopt);
    $prod=shift;
    $backopt=1;
    $help = Msg::new("A $prod->{abbr} cluster requires two private heartbeat links, but additional heartbeat links may be configured for increased performance and protection.", 40, 2424, "$prod->{abbr}");
    $msg = Msg::new("Would you like to configure a third private heartbeat link?", 40, 2425);
    $ayn = $msg->aynn($help,$backopt);
    return $ayn;
}

# Ask about a fourth heartbeat link
sub ask_fourth_hb {
    my ($prod,$ayn,$msg,$help,$backopt);
    $prod=shift;
    $backopt=1;
    $help = Msg::new("A $prod->{abbr} cluster requires two private heartbeat links, but additional heartbeat links may be configured for increased performance and protection.", 40, 2424, "$prod->{abbr}");
    $msg = Msg::new("Would you like to configure a fourth private heartbeat link?", 40, 2426);
    $ayn = $msg->aynn($help,$backopt);
    return $ayn;
}

# Ask about more (1st-8th) heartbeat links
sub ask_nth_hb {
    my ($prod,$abbr,$ayn,$msg,$help,$backopt,$n);
    ($prod,$n)=@_;
    $backopt=1;
    $abbr = $prod->{abbr};
    if ($n==1) {
        # the first heartbeat link is required.
        $help = Msg::new("A $abbr cluster requires two private heartbeat links, but additional heartbeat links may be configured for increased performance and protection.", 40, 2424, "$abbr");
        $msg = Msg::new("Would you like to configure the first private heartbeat link?", 40, 3522);
    } elsif ($n==2) {
        $help = Msg::new("A $abbr cluster is typically configured to use two private heartbeat links, but some $abbr clusters may be configured with just one private heartbeat link and a second low-priority heartbeat link. The NIC selected as a low-priority heartbeat link is typically the network card on which the system's public IP address is active.", 40, 3521, "$abbr", "$abbr");
        $msg = Msg::new("Would you like to configure a second private heartbeat link?", 40, 2423);
    } elsif ($n==3) {
        $help = Msg::new("A $abbr cluster requires two private heartbeat links, but additional heartbeat links may be configured for increased performance and protection.", 40, 2424, "$abbr");
        $msg = Msg::new("Would you like to configure a third private heartbeat link?", 40, 2425);
    } elsif ($n==4) {
        $help = Msg::new("A $abbr cluster requires two private heartbeat links, but additional heartbeat links may be configured for increased performance and protection.", 40, 2424, "$abbr");
        $msg = Msg::new("Would you like to configure a fourth private heartbeat link?", 40, 2426);
    } elsif ($n==5) {
        $help = Msg::new("A $abbr cluster requires two private heartbeat links, but additional heartbeat links may be configured for increased performance and protection.", 40, 2424, "$abbr");
        $msg = Msg::new("Would you like to configure a fifth private heartbeat link?", 40, 3523);
    } elsif ($n==6) {
        $help = Msg::new("A $abbr cluster requires two private heartbeat links, but additional heartbeat links may be configured for increased performance and protection.", 40, 2424, "$abbr");
        $msg = Msg::new("Would you like to configure a sixth private heartbeat link?", 40, 3524);
    } elsif ($n==7) {
        $help = Msg::new("A $abbr cluster requires two private heartbeat links, but additional heartbeat links may be configured for increased performance and protection.", 40, 2424, "$abbr");
        $msg = Msg::new("Would you like to configure a seventh private heartbeat link?", 40, 3525);
    } elsif ($n==8) {
        $help = Msg::new("A $abbr cluster requires two private heartbeat links, but additional heartbeat links may be configured for increased performance and protection.", 40, 2424, "$abbr");
        $msg = Msg::new("Would you like to configure an eighth private heartbeat link?", 40, 3526);
    }
    $ayn = $msg->aynn($help,$backopt);
    return $ayn;
}

# Ask about a lowpri heartbeat link
sub ask_lowpri_hb {
    my ($prod,$ayn,$msg,$help,$backopt,$n);
    ($prod,$n)=@_;
    $backopt=1;
    if (!defined($n)||$n==0||$n==1) {
        $msg = Msg::new("Do you want to configure an additional low-priority heartbeat link?", 40, 3527);
    } elsif ($n==2) {
        $msg = Msg::new("Do you want to configure a second low-priority heartbeat link?", 40, 3528);
    } elsif ($n==3) {
        $msg = Msg::new("Do you want to configure a third low-priority heartbeat link?", 40, 3529);
    } elsif ($n==4) {
        $msg = Msg::new("Do you want to configure a fourth low-priority heartbeat link?", 40, 3530);
    } elsif ($n==5) {
        $msg = Msg::new("Do you want to configure a fifth low-priority heartbeat link?", 40, 3531);
    } elsif ($n==6) {
        $msg = Msg::new("Do you want to configure a sixth low-priority heartbeat link?", 40, 3532);
    } elsif ($n==7) {
        $msg = Msg::new("Do you want to configure a seventh low-priority heartbeat link?", 40, 3533);
    } elsif ($n==8) {
        $msg = Msg::new("Do you want to configure an eighth low-priority heartbeat link?", 40, 3534);
    }
    $help=Msg::new("Configuring a low-priority heartbeat link on the public network is an option that provides an additional method of confirming system failure.", 40, 3535);
    $ayn = $msg->aynn($help,$backopt);
    return $ayn;
}

# Ask about using common NIC numbers
sub ask_common_nics {
    my ($prod,$cpic,$ayn,$msg,$help,$backopt,$warn);
    $prod=shift;
    $cpic = Obj::cpic();
    $backopt=1;
    $help = Msg::new("Enter 'y' if the NIC devices used for this heartbeat link are the same on all systems in the cluster. Enter 'n' if NIC devices used for this heartbeat link are not the same on all systems in the cluster.", 40, 2429);
    $msg = Msg::new("Are you using the same NICs for private heartbeat links on all systems?", 40, 2430);
    $ayn = $msg->ayny($help,$backopt);
    if ($ayn eq "N" && $cpic->{prod} eq "SFRAC51" ){
        $warn=Msg::new("Oracle Clusterware needs that you must use the same NICs for private heartbeat links on\nall systems of your cluster, otherwise it will not get configured on your cluster.", 40, 2431); 
        $warn->warning(); 
        $ayn = $msg->ayny($help,$backopt);
    }
    return $ayn;
}

# verify a clustername starts with a letter and has valid characters
sub verify_clustername {
    my($prod,$ask,$cname,$fc,$msg,$name);
    ($prod,$name)=(@_);
    $fc=substr($name,0,1);
    if ($fc =~ /[^A-Za-z]/) {
         $msg=Msg::new("Cluster Name must start with a letter", 40, 3536);
         $msg->print();
         return 0;
    }
    $cname=$name;
    $cname =~ s/[A-Za-z0-9_-]//g;
    if ($cname) {
        $msg = Msg::new("Cluster Name cannot use the characters: $cname", 40, 3537, "$cname");
        $msg->print();
    } elsif ($prod->vcs_reservedwords($name)) {
        $msg = Msg::new("$name is a VCS reserved word and cannot be used as a cluster name", 40, 3538, "$name");
        $msg->print();
    } else {
        return 1;
    }
    return 0;
}


# purpose:  transform the system name from user input to the proper format
#           in order to make sure correct hostname or IP format is used in main.cf as SystemList by VCS engine
#           besides fixing the incident 1010374,our purpose is to process IPv4,IPv6 and FQDN name simultaneous.
#           It can be applied in main.cf file.
# params:   $sysname original system name from user input
# returns:  $sysname system name VCS which engine can identify
#
sub transform_system_name {
    my $sysname = shift;
    return unless ($sysname);

    # previously, CPI add double quotas for IPv4 IPv6 and FQDN address used in main.cf
    # to make it accepted by VCS enginee, this is an un-documented VCS behavior
    # confirmed with VCS team, change design into:
    # CPI replace IPv4 IPv6 FQDN address with nodename as the system name
    
    if (ip_is_ipv4($sysname) || ip_is_ipv6($sysname) || ($sysname =~ /\.\w+/)) {
        # return "\"$sysname\""; # add double quotation marks
        my $sys = Obj::sys($sysname);
        return $sys->{hostname};
    } else {
        # return it when parameter is regular hostname
        return $sysname;
    }
}

# create an /etc/llttab entry for a NIC
sub convert_nic2lltlink_sys {
    my($prod,$plat,$dev,$l,$lowpri,$lp,$n,$nic,$sys,$lltlink,$vionics);
    ($prod,$sys,$nic,$lp)=(@_);
    return "" if (!$nic);
    if ($lp) {
        $lowpri="-lowpri";
    } else {
        $lowpri="";
    }

    $plat=EDR::plat($prod->{padv});
    $dev = ($plat eq "AIX") ? "/dev/dlpi" : "/dev";
    $l=$n=$nic;
    $l=~s/(.*[A-Za-z])\d*/$1/g;
    $n=~s/.*[A-Za-z](\d*)$/$1/g;
    if( $plat eq "AIX" ) {
        $vionics= $sys->padv->virtualnics_sys($sys);
        # if any configured NIC is found a virtual I/O Micro-partition NIC => set default MTU to be 1500 b/s
        if( EDR::inarr($nic, @$vionics) ) {
            return "link$lowpri $nic $dev/$l:$n - ether - 1500\n";
        }
    }
    return "link$lowpri $nic $dev/$l:$n - ether - -\n";
}

# get the original NIC name from an Ethernet link/link-lowpri devname in /etc/llttab (For LLT over Ethernet)
sub convert_linkdev2nic_sys {
    my($prod,$plat,$dev,$nic,$linkdev,$sys);
    ($prod,$sys,$linkdev)=(@_);
    return "" if (!$linkdev);
    $plat=EDR::plat($prod->{padv});
    $dev = ($plat eq "AIX") ? "/dev/dlpi/" : "/dev/";
    $nic = $linkdev;
    $nic=~s/$dev//;
    $nic=~s/://;
    return $nic;
}

# get the original NIC name from a link/link-lowpri IP address in /etc/llttab (For LLT over UDP)
sub convert_linkip2nic_sys {
    my($prod,$dev,$nic,$nici,$linkip,$sys,$found,$rnics,$output);
    ($prod,$sys,$linkip)=(@_);
    return "" if (!$linkip);
    $rnics = $sys->padv()->systemnics_sys($sys,1);
    $found = 0;
    for $nici (@$rnics) {
        next unless $nici;
        $output = EDR::cmd_sys($sys, "_cmd_ifconfig $nici");
        if ( $output =~ /[^A-Fa-f0-9]$linkip[^A-Fa-f0-9]/) {
            $found = 1;
            $nic = $nici;
            last;
        }
    }
    if ( !$found) {
        $nic = "";
    }
    return $nic;
}

sub generate_udplinks_sys {
    my($address,$content,$dev,$devn,$linktype,$n,$nic,$plat,$port,$sysi,$system,$nid);
    my($prod,$sys) = @_;
    my $cfg = Obj::cfg();
    my $edr = Obj::edr();
    
    $plat=EDR::plat($prod->{padv});
    $sysi=$sys->{sys};
    if($plat =~ /AIX/) {
        $dev = "/dev/xti/udp"
    } elsif ($plat =~ /SunOS|HPUX/) {
        $dev = "/dev/udp"
    } else {
        $dev = "udp";
    }
    $content = "";
    for $n (1..$prod->num_hbnics_cfg($cfg)) {
        $nic = $cfg->{"vcs_lltlink$n"}{$sysi};
        $port = $cfg->{"vcs_udplink$n"."_port"}{$sysi};
        $address = $cfg->{"vcs_udplink$n"."_address"}{$sysi};
        $devn = $dev;
        $linktype = "udp";
        if(ip_is_ipv6($address)) {
            $devn = "$dev"."6";
            $linktype = "udp6";
        }
        $content .= "link $nic $devn - $linktype $port - $address -\n";
    }

    for $n (1..$prod->num_lopri_hbnics_cfg($cfg)) {
        $nic = $cfg->{"vcs_lltlinklowpri$n"}{$sysi};
        $port = $cfg->{"vcs_udplinklowpri$n"."_port"}{$sysi};
        $address = $cfg->{"vcs_udplinklowpri$n"."_address"}{$sysi};
        $devn = $dev;
        $linktype = "udp";
        if(ip_is_ipv6($address)) {
            $devn = "$dev"."6";
            $linktype = "udp6";
        }
        $content .= "link-lowpri $nic $devn - $linktype $port - $address -\n";
    }
    $nid = 0;
    for $system (@{$edr->{systems}}) {
        my $sysname = $system->{sys};
        if ($sysname eq $sysi) {
            $nid++;
            next;
        }
        for $n (1..$prod->num_hbnics_cfg($cfg)) {
            $nic = $cfg->{"vcs_lltlink$n"}{$sysi};
            $address = $cfg->{"vcs_udplink$n"."_address"}{$sysname};
            $content.= "set-addr $nid $nic $address\n";
        }
        for $n (1..$prod->num_lopri_hbnics_cfg($cfg)) {
            $nic = $cfg->{"vcs_lltlinklowpri$n"}{$sysi};
            $address = $cfg->{"vcs_udplinklowpri$n"."_address"}{$sysname};
            $content.= "set-addr $nid $nic $address\n";
        }
        $nid++;
    }
    $content.="set-bcasthb 0\n";
    $content.="set-arp 0\n";
    
    return $content;
}

sub update_llttab {
    my ($prod,$sysi,$sys,$cfg,$edr,$llttab,$hostname,$n);
    $prod = shift;
    $cfg = Obj::cfg();
    $edr = Obj::edr();

    foreach $sys(@{$edr->{systems}}) {
        $sysi = $sys->{sys};
        $hostname = transform_system_name($sysi);
        $llttab="set-node $hostname\nset-cluster $cfg->{vcs_clusterid}\n";
        if ($cfg->{lltoverudp}) {
            $llttab.=$prod->generate_udplinks_sys($sys);
        } else {
            for $n (1..$prod->num_hbnics_cfg($cfg)) {
                $llttab.=$prod->convert_nic2lltlink_sys($sys, $cfg->{"vcs_lltlink$n"}{$sysi});
            }
            for $n (1..$prod->num_lopri_hbnics_cfg($cfg)) {
                $llttab.=$prod->convert_nic2lltlink_sys($sys, $cfg->{"vcs_lltlinklowpri$n"}{$sysi},1);
            }
        }
        EDR::writefile($llttab,"$edr->{tmpdir}/llttab.$sysi",1);
    }
}

sub update_llthosts {
    my ($prod,$cfg,$edr,$llthosts,$n,$sysi,$lltsys);
    $prod = shift;
    $cfg = Obj::cfg();
    $edr = Obj::edr();

    $llthosts = "";
    $n=0;
    foreach $sysi(@{$edr->{systems}}) {
        $lltsys = transform_system_name($sysi->{sys});
        $llthosts.="$n $lltsys\n";
        $n++;
    }
    EDR::writefile($llthosts,"$edr->{tmpdir}/llthosts",1 );
}

sub update_gabtab {
    my ($prod,$cfg,$edr,$gabtab,$n);
    $prod = shift;
    $edr = Obj::edr();
    $cfg = Obj::cfg();

    $n = scalar @{$edr->{systems}};
    $gabtab = "/sbin/gabconfig -c -n$n\n";
    EDR::writefile($gabtab,"$edr->{tmpdir}/gabtab", 1);
}

sub update_maincf {
    my ($extype,$extypes,$prod,$sys,$cfg,$edr,$maincf,$n,$users,$admins,$opers,$sysi,$sys2,$cdev,$cdev_attr,$ddev,$ddev_attr,$ddev_nwhosts,$csl,$drid,$drnd,$cons,$recp,$prefix,$cdev_nwhosts,$plat);
    ($prod,$sys) = @_;
    $cfg = Obj::cfg();
    $edr = Obj::edr();

    $maincf="include \"types.cf\"\n";
    # include extra vcs types
    $extypes = $prod->get_extra_types_sys($sys);
    if($extypes) {
        for $extype (@$extypes) {
            $maincf.="include \"$extype\"\n";
        }
    }
    $maincf.="\ncluster $cfg->{vcs_clustername} (";
    $maincf.="\n\tSecureClus = 1" if ($cfg->{at_rootdomain} || $prod->{vxssmenuopt}); 
    foreach $n(0..$#{$cfg->{vcs_username}}) {
        $users.=", " if ($users);
        $users.="\"${$cfg->{vcs_username}}[$n]\" = ${$cfg->{vcs_userenpw}}[$n]";
        if (${$cfg->{vcs_userpriv}}[$n] eq "Administrators") {
            $admins.=", " if ($admins);
            $admins.="\"${$cfg->{vcs_username}}[$n]\"";
        } elsif (${$cfg->{vcs_userpriv}}[$n] eq "Operators") {
            $opers.=", " if ($opers);
            $opers.="\"${$cfg->{vcs_username}}[$n]\"";
        }
    }
    $maincf .= "\n\tUserNames = { $users }" if ($users);
    $maincf .= "\n\tAdministrators = { $admins }" if ($admins);
    $maincf .= "\n\tOperators = { $opers }" if ($opers);
    if ($cfg->{vcs_gcovip}) {
        $maincf .= "\n\tClusterAddress = \"$cfg->{vcs_gcovip}\"";
    } elsif ($cfg->{vcs_csgvip}) {
        $maincf .= "\n\tClusterAddress = \"$cfg->{vcs_csgvip}\"";
    }
    $maincf .= "\n\tCounterInterval = 5\n\t)\n\n";

    $n=0;
    foreach $sysi(@{$edr->{systems}}) {
        $sys2 =transform_system_name($sysi->{sys});
        $maincf.="system $sys2 (\n\t)\n\n";
        $csl.= ", " if ($csl);
        $csl.= $sys2;
        $n++;
    }    
    
    #get the vcs_csgnic string
    $cdev = "\t\tDevice = $cfg->{vcs_csgnic}{all}\n" if ($cfg->{vcs_csgnic}{all});
    if ($cfg->{vcs_csgnic}{${$cfg->{systems}}[0]}) {
        foreach $sys(@{$cfg->{systems}}) { 
            $sys2 = transform_system_name($sys);
            $cdev.="\t\tDevice\@$sys2 = $cfg->{vcs_csgnic}{$sys}\n";
        }
    }
    $cdev_attr=$prod->tsub("nic_protocol_attr") if ($cdev); 
    
    # For AIX
    $plat=EDR::plat($prod->{padv});
    $cdev_nwhosts=$prod->tsub("nic_nwhosts_csg") if ($cdev && ($plat eq "AIX"));
    # For HP
    $cdev_nwhosts=$prod->tsub("store_nhip") if ($cdev && ($plat eq "HPUX"));
    
    #get the vcs_gconic string
    $ddev="\t\tDevice = $cfg->{vcs_gconic}{all}\n" if ($cfg->{vcs_gconic}{all});
    if ($cfg->{vcs_gconic}{${$cfg->{systems}}[0]}) {
        foreach $sys(@{$cfg->{systems}}) { 
            $sys2 = transform_system_name($sys);
            $ddev.="\t\tDevice\@$sys2 = $cfg->{vcs_gconic}{$sys}\n";
        }
    }
    $ddev_attr=$prod->tsub("nic_protocol_attr",1) if($ddev);
    # for AIX
    $ddev_nwhosts=$prod->tsub("nic_nwhosts_gco") if ($ddev && ($plat eq "AIX"));
    # for HP
    $ddev_nwhosts=$prod->tsub("store_nhip") if ($ddev && ($plat eq "HPUX"));
    
    #GCO info
    $prefix=ip_is_ipv6($cfg->{vcs_csgvip})? "PrefixLen" : "NetMask";
    $maincf .= "group ClusterService (\n\tSystemList = { $csl }\n\tAutoStartList = { $csl }\n\tOnlineRetryLimit = 3\n\tOnlineRetryInterval = 120\n\t)\n\n" if (($cdev) || ($ddev));
    $maincf .= "\tApplication wac (\n\t\tStartProgram = \"/opt/VRTSvcs/bin/wacstart\"\n\t\tStopProgram = \"/opt/VRTSvcs/bin/wacstop\"\n\t\tMonitorProcesses = { \"/opt/VRTSvcs/bin/wac\" }\n\t\tRestartLimit = 3\n\t\t)\n\n" if ($cfg->{vcs_gcovip});
    $maincf .= "\tIP webip (\n$cdev\t\tAddress = \"$cfg->{vcs_csgvip}\"\n\t\t$prefix = \"$cfg->{vcs_csgnetmask}\"\n\t\t)\n\n" if ($cfg->{vcs_csgvip});
    if (($cfg->{vcs_gcovip}) && ($cfg->{vcs_gcovip} ne $cfg->{vcs_csgvip})) {
        $prefix = ip_is_ipv6($cfg->{vcs_gcovip})? "PrefixLen" : "NetMask";
        $maincf .= "\tIP gcoip (\n$ddev\t\tAddress = \"$cfg->{vcs_gcovip}\"\n\t\t$prefix = \"$cfg->{vcs_gconetmask}\"\n\t\t)\n\n";
        $drid = "gcoip";
    } else {
        $drid = "webip";
    }
    #$maincf .= "\tNIC csgnic (\n$cdev\t\t)\n\n";
    if ($cdev) {
        if ($cdev_attr ) {
            if ($cdev_nwhosts) {
                $maincf .= "\tNIC csgnic (\n$cdev\t\t$cdev_attr\t\t$cdev_nwhosts\t\t)\n\n";
            } else {
                $maincf .= "\tNIC csgnic (\n$cdev\t\t$cdev_attr\t\t)\n\n";    
            }
        } else {
           if ($cdev_nwhosts) {
                $maincf .= "\tNIC csgnic (\n$cdev\t\t$cdev_nwhosts\t\t)\n\n";
           } else {
                $maincf .= "\tNIC csgnic (\n$cdev\t\t)\n\n";    
           }    
       } 
    }
    
    if ($ddev){
        if ($ddev ne $cdev){
            if($ddev_attr) {
                if ($ddev_nwhosts) {
                    $maincf .= "\tNIC gconic (\n$ddev\t\t$ddev_attr\t\t$ddev_nwhosts\t\t)\n\n";
                } else {
                    $maincf .= "\tNIC gconic (\n$ddev\t\t$ddev_attr\t\t)\n\n";    
                }
            } else {
                if ($ddev_nwhosts) {
                    $maincf .= "\tNIC gconic (\n$ddev\t\t$ddev_nwhosts\t\t)\n\n";
                } else {
                    $maincf .= "\tNIC gconic (\n$ddev\t\t)\n\n";    
                }
            }
            $drnd="gconic";
        } else {
            $drnd="csgnic";
        }
    }

    #log SNMP and SMTP
    $maincf .= "\tNotifierMngr ntfr (\n" if (($cfg->{vcs_smtpserver}) || ($cfg->{vcs_snmpport}));
    #log SNMP
    if ($cfg->{vcs_snmpport}) {
        foreach $n(0..$#{$cfg->{vcs_snmpcons}}) {
            $cons.=", " if ($cons);
            $cons.="\"${$cfg->{vcs_snmpcons}}[$n]\" = ${$cfg->{vcs_snmpcsev}}[$n]";
        }
        $maincf.="\t\tSnmpConsoles = { $cons }\n";
        $maincf.="\t\tSnmpdTrapPort = $cfg->{vcs_snmpport}\n" if ($cfg->{vcs_snmpport}!=162);
    }
    #log SMTP
    if ($cfg->{vcs_smtpserver}) {
        foreach $n(0..$#{$cfg->{vcs_smtprecp}}) {
            $recp.=", " if ($recp);
            $recp.="\"${$cfg->{vcs_smtprecp}}[$n]\" = ${$cfg->{vcs_smtprsev}}[$n]";
        }
        $maincf .= "\t\tSmtpServer = \"$cfg->{vcs_smtpserver}\"\n\t\tSmtpRecipients ={ $recp }\n";
    }
    $maincf .= "\t\t)\n\n" if (($cfg->{vcs_smtpserver}) || ($cfg->{vcs_snmpport}));

   #$maincf .= "\tVRTSWebApp VCSweb (\n\t\tCritical = 0\n\t\tAppName = \"cmc\"\n\t\tInstallDir = \"/opt/VRTSweb/VERITAS\"\n\t\tTimeForOnline = 5\n\t\tRestartLimit = 3\n\t\t)\n\n\tVCSweb requires webip\n" if ($cfg->{vcs_csgvip});

    $maincf .= "\twebip requires csgnic\n" if ($cfg->{vcs_csgvip});
    $maincf .= "\tntfr requires csgnic\n" if (($cdev) && (($cfg->{vcs_snmpport}) || ($cfg->{vcs_smtpserver})));
    
    $maincf .= "\twac requires $drid\n" if ($cfg->{vcs_gcovip});
    $maincf .= "\t$drid requires $drnd\n" if ($drnd);
    
    if ($cfg->{at_rootdomain} || $prod->{vxssmenuopt}) {
        $maincf .= "group VxSS (\n\tSystemList = { $csl }\n\tParallel = 1\n\tAutoStartList = { $csl }\n\tOnlineRetryLimit = 3\n\tOnlineRetryInterval = 120\n\t)\n\n\tProcessOnOnly vxatd (\n\t\tPathName = \"/opt/VRTSat/bin/vxatd\"\n\t\tIgnoreArgs = 1\n\t\t)\n\n\tPhantom phantom_vxss (\n\t\t)\n\n";
    }
    EDR::writefile($maincf,"$edr->{tmpdir}/main.cf", 1);
}

sub store_nhip {
    my ($prod)=@_;
    my ($cfg,$ip,$nwhosts,$nwhost);
    
    $cfg=Obj::cfg();
    if ($cfg->{vcs_networkhosts}) {
        for $ip (split(" ",$cfg->{vcs_networkhosts})) {
            $nwhost.="," if($nwhost);
            $nwhost.="\"$ip\"" if ($ip);
        }
        $nwhosts="NetworkHosts = {".$nwhost."}\n" if ($nwhost);
    }    
    return $nwhosts;
    
}

sub nic_nwhosts_gco {
    my ($prod)=@_;
    my ($cfg,$ip,$ddev_nwhosts,$nwhost);
    
    $cfg=Obj::cfg();
    if ($cfg->{vcs_gconwhosts}) {
        for $ip (split(" ",$cfg->{vcs_gconwhosts})) {
            $nwhost.="," if($nwhost);
            $nwhost.="\"$ip\"" if ($ip);
        }
        $ddev_nwhosts="NetworkHosts = {".$nwhost."}\n" if ($nwhost);
    }    
    return $ddev_nwhosts;
}

sub nic_nwhosts_csg {
    my ($prod)=@_;
    my ($cfg,$cpic,$ip,$cdev_nwhosts,$nwhost,$sys);
    
    $cfg=Obj::cfg();
    $cpic=Obj::cpic();
    
    for $sys (@{$cpic->{systems}}) {
        $nwhost=""; 
        if ($cfg->{vcs_csgnwhosts}{$sys->{sys}}) {
            for $ip (split(" ", $cfg->{vcs_csgnwhosts}{$sys->{sys}})) {
                $nwhost.="," if($nwhost);
                $nwhost.="\"$ip\"" if ($ip);
            }
            $cdev_nwhosts.="\t\t" if ($cdev_nwhosts);
            $cdev_nwhosts.="NetworkHosts\@$sys->{sys} = {".$nwhost."}\n" if ($nwhost); 
        }   
    }      
    return $cdev_nwhosts;
}
###
#
# performing vcs upgrade
#
###

sub check_upgradeable_sys {
    my ($iver,$conf);
    my ($prod,$sys) = @_;
    
    $iver = $prod->version_sys($sys);
    return 0 unless ($iver);
    return 0 unless (EDR::compvers($iver,$prod->{vers},4) == 2);
    $conf = $prod->get_config_sys($sys);
    return 0 unless ($conf);
    $sys->{vcs_upgradeable} = 1;
    Thr::setq($sys->{pool}, "vcs_upgradeable", 1);
    return 1;
}

sub prestop_sys {
    my($prod,$sys) = @_;
    return 1;
}

# Start vxatd on each node in AB mode
# Delete the HA_SERVICES from ab pdr,add it to to local pdr
# Create the  webserver_VCS_principal for the new pdr.
sub at_upgrade_configure_sys{
    my ($prod,$sys)=@_;
    my ($chroot,$domain,$rootpath);
    
    return unless($sys->{secureclus});
    $rootpath = Cfg::opt("rootpath") || "";
    $chroot = ($rootpath) ? "_cmd_chroot $rootpath" : "";
                                                            
    #delete the HA_SERVICES from ab pdr if any
    if ($rootpath && $sys->{osupgraded}) {
        # for LU with OS upgraded
        $prod->delete_domain_by_file_sys($sys,$rootpath);
    } else {
        # for normal upgrade or LU without OS upgraded
        $domain=EDR::cmd_sys($sys, "$chroot _cmd_vssat listpd --pdrtype ab");
        if ($domain=~/Domain\s+?Name\s+?HA_SERVICES\@/) {
            EDR::cmd_sys($sys, "$chroot _cmd_vssat deletepd --pdrtype ab --domain HA_SERVICES --silent");           
        }
    }
}

# delete the HA_SERVICES from ab type domain
# by editing /var/VRTSat/ABAuthSource file directly
sub delete_domain_by_file_sys {
    my ($prod,$sys,$rootpath)=@_;
    my ($content,$edr,$line,$output,$save,$abfile);
    
    return "" unless($rootpath);
    $edr=Obj::edr();
    $abfile="$rootpath/var/VRTSat/ABAuthSource";
    if (EDR::file_sys($sys,$abfile)) {
        $content=EDR::cmd_sys($sys,"_cmd_cat $abfile");
        for $line (split("\n",$content)) {
            # Modify the file to remove the HA_SERVICE attributes.
            if ($line=~/^\[HA_SERVICE/) {
                $save=0;
            } elsif ($line=~/^\[/) {
                $save=1;
            }
            $output.="$line\n" if ($save);
        }
        EDR::writefile_sys($sys,$output,$abfile);
    }
}

sub upgrade_configure_sys {
    my ($cfg,$cpic,$msg,$prod,$pkgi,$pkg,$rtn,$sys,$sysi,$sysm);
    my ($obj_sysi,$uuid);
    ($prod,$sys) = @_;
    return 1 if(Cfg::opt("upgrade_kernelpkgs")); 
    unless ($sys->{vcs_upgradeable}){
        if($sys->system1){
            EDR::create_flag("maincf_upgrade_done");
        }
        return 0; 
    }

    $cpic = Obj::cpic();
    $cfg = Obj::cfg();
    
    # by default, set vcs_allowcomms to 1 if it isn't defined
    # this happens if other products (eg. sfcfs sfrac) calls vcs upgrade_configure_sys 
    # but foget to set this attribute to 1 
    $cfg->{vcs_allowcomms} = 1 unless (defined($cfg->{vcs_allowcomms}));

    #security configuration
    $prod->tsub("at_upgrade_configure_sys",$sys);

    # perform following tasks only on first node of cluster
    if ($sys->system1) {
        # config uuid
        if ($prod->{upgrade_ignore_conf}) {
            #copy uuid from hosts upgraded in phased 1 to hosts upgraded in phased 2.
            for $sysi (@{$sys->{cluster_systems}}) {
                if (defined $Obj::pool{"Sys\::$sysi"}) { 
                    $obj_sysi = $Obj::pool{"Sys\::$sysi"}; 
                    next if (EDR::inarr($obj_sysi,@{$cpic->{systems}})); 
                    next unless ($obj_sysi->{rsh} || $obj_sysi->{islocal});
                } 
                $uuid=$prod->get_uuid_sys($obj_sysi);
                next unless ($uuid && $uuid !~ /NO_UUID/);
                for $sysm (@{$cpic->{systems}}) {
                    $prod->set_uuid_2sys($sysm,$uuid);
                }
                EDR::create_flag("maincf_upgrade_done");
                return 1;
            }
            $msg = Msg::new("Cannot copy UUID from the hosts upgraded in Phase 1 to the hosts upgraded in Phase 2. Manually create UUID on all the nodes in the cluster, before rebooting the hosts.", 40, 2433);
            EDR::push_warning_sys($sys,$msg);
            EDR::create_flag("maincf_upgrade_done");
            return 0;
        } else {
            $rtn = $prod->config_uuid();
            if($rtn == -1) {
                $msg = Msg::new("Cannot find uuidconfig.pl for UUID configuration. Manually create UUID before starting VCS", 40, 2434);
                EDR::push_warning_sys($sys,$msg);
            }
        }
        # perform dynamic upgrade
        unless ($prod->dynupgrade_upgrade_sys($sys)) {
            EDR::create_flag("maincf_upgrade_done");
            return 0;
        }
        
        # todo: some manually upgrade steps required
        
        EDR::create_flag("maincf_upgrade_done");
        
    } else {
        EDR::wait_for_flag("maincf_upgrade_done");
    }

    if ($cfg->{vcs_allowcomms}) {
        # Linux: remove ONENODE=yes in /etc/sysconfig/vcs
        # SunOS: remove manifest for system/vcs-onenode
        $prod->set_onenode_cluster_sys($sys,0) if ($prod->can("set_onenode_cluster_sys"));
    } else {
        # Linux: set ONENODE=yes in /etc/sysconfig/vcs
        # SunOS: import manifest vcs-onenode.xml
        $prod->set_onenode_cluster_sys($sys,1) if ($prod->can("set_onenode_cluster_sys"));
    }

    # link install: enable vcs when finish upgrade
    $prod->vcs_enable_sys($sys);
    
    return 1;
}

sub poststart_sys {
    my($prod,$sys) = @_;
    return 1;    
}

sub ru_prestop_sys {
    my ($prod,$sys)=@_;
    my $cpic=Obj::cpic();
    my (@procs,$proc,$procobj);
    if (Cfg::opt("upgrade_kernelpkgs")) {
        @procs=qw(gab51 vxfen51);
        foreach $proc(@procs) {
             $procobj=$sys->proc($proc);
             $procobj->tsub("prestop_sys",$sys);
        }
        if($cpic->{prod}=~/^(SFRAC)\d+/){
            $procobj=$sys->proc("vcsmm51");
            $procobj->tsub("prestop_sys",$sys) if($procobj);
        }
        #Stop had
        EDR::cmd_sys($sys,"_cmd_hastop -local -evacuate 2>/dev/null");
    }

}

sub upgrade_rolling_post_sys {
    my ($prod,$sys)=@_;
    my ($cv,$sys1,@syslist,$sysname,$sysobj,$cpic,$padv,$lltver,$vcsver,$notallsysru,$nsystems,$csystems,$msg,$vcs,$llt,$gab,$vxfen,$had,$hashadow,$CmdServer,$pids,$edr);
    $edr=Obj::edr();
    $cpic=Obj::cpic();
    #all systems are rolling_upgraded
    #check if post tasks are done.
    $msg=EDR::cmd_sys($sys,"_cmd_cat /etc/gabtab");
    return unless($msg=~/V/);
    for $sys1 (@{$cpic->{systems}}) {
        if(EDR::file_sys($sys1,"/etc/gabtab.rubak")){
            EDR::cmd_sys($sys1,"_cmd_rmr /etc/gabtab");
            EDR::cmd_sys($sys1,"_cmd_cp -rfp /etc/gabtab.rubak /etc/gabtab 2>/dev/null");
        }
        if(EDR::file_sys($sys1,"/etc/vcsmmtab.rubak") && $cpic->{prod}=~/^(SFCFSRAC|SFRAC)\d+/){
            EDR::cmd_sys($sys1,"_cmd_rmr /etc/vcsmmtab");
            EDR::cmd_sys($sys1,"_cmd_cp -rfp /etc/vcsmmtab.rubak /etc/vcsmmtab 2>/dev/null");
        }
        if(EDR::file_sys($sys1,"/etc/vxfenmode.rubak")){
            EDR::cmd_sys($sys1,"_cmd_rmr /etc/vxfenmode");
            EDR::cmd_sys($sys1,"_cmd_cp -rfp /etc/vxfenmode.rubak /etc/vxfenmode 2>/dev/null");
        }
        if($sys1->system1){
            EDR::cmd_sys($sys1,"_cmd_gabconfig -R ");
            EDR::cmd_sys($sys1,"_cmd_vxfenconfig -R");
            EDR::cmd_sys($sys1,"_cmd_vcsmmconfig -R") if ($cpic->{prod}=~/^(SFCFSRAC|SFRAC)\d+/);
        }
    }
}

sub stop_vcs_rollingupgrade {
    my ($prod,$sys)=@_;
    my ($cpic,$ck,$iter,$maxiters,$msg,$pids,$stop);
   $stop = EDR::cmd_sys($sys, "$prod->{bindir}/hastop -local -evacuate");
    Msg::log("Stopping VCS for Rolling ugprade before upgrade_precheck");
    sleep 3;
    if (EDR::isverror($stop)) {
            $pids=EDR::pids_sys($sys, "bin/had");
            EDR::kill_pids_sys($sys,@$pids);
            $pids=EDR::pids_sys($sys, "hashadow");
            EDR::kill_pids_sys($sys,@$pids);
            $pids=EDR::pids_sys($sys, "bin/CmdServer");
            EDR::kill_pids_sys($sys,@$pids);
    }
}


sub upgrade_prestop_sys {
    my($maincf,$prod,$sys);
    ($prod,$sys) = @_;
    $prod->tsub("upgrade_rolling_post_sys",$sys) if($sys->system1 &&  Cfg::opt("upgrade_nonkernelpkgs"));
    $prod->tsub("ru_prestop_sys",$sys) if(Cfg::opt("upgrade_kernelpkgs"));
    return 1  if(Cfg::opt("upgrade_kernelpkgs"));

    unless ($sys->{vcs_upgradeable} && $prod->upgrade_gabtab_sys($sys)){
        if($sys->system1){
            EDR::create_flag("maincf_pre_upgrade_done"); 
        }
        return 0;
    }
    
    # set secure cluster attribute to each system
    $maincf=EDR::cmd_sys($sys, "_cmd_cat $prod->{maincf}");
    EDR::set_value_sys($sys, "secureclus",1) if ($maincf=~/SecureClus\s*=\s*1/);

    # following tasks only performed on first node
    if ($sys->system1) {
        #return 0 unless ($prod->check_obsolete_types_sys($sys));

        # freeze service group
        if (!Cfg::opt("rootpath") && !$prod->{upgrade_ignore_conf} && $sys->{prod_running}) {
            $prod->haconf_makerw();
            $prod->freeze_groups();
            $prod->haconf_dumpmakero();
        }
    
        # todo: support for lower version upgrade
        # todo: support other exceptions those can not handle by dynmic upgrade
 
        EDR::create_flag("maincf_pre_upgrade_done");
    } else {
        EDR::wait_for_flag("maincf_pre_upgrade_done");
    }

    return 1;
}

sub dynupgrade_pre_upgrade_sys {
    my ($edr,$line,$maincf,$msg,$prod,$rootpath,$rtn,$sourcefile,$sourcefilebase,$sys,$upgradepath);
    ($prod,$sys) = @_;
    $edr=Obj::edr();
    $rootpath=Cfg::opt("rootpath");
    $upgradepath= EDR::tmpdir()."/upgrade";

    # below steps will ensure the old config files to be available

    # Pre Step 1- backup old cf files
    EDR::cmd_sys($sys, "_cmd_rmr $upgradepath");

    $rtn = $prod->backup_cf_files_sys($sys, "$rootpath$prod->{configdir}", "$upgradepath/old_config");
    return 0 unless ($rtn);

    $rtn = $prod->backup_cf_files_sys($sys, "$rootpath$prod->{confdir}", "$upgradepath/old_conf");
    return 0 unless ($rtn);

    $msg = Msg::new("\nFailed to upgrade because the system configuration files are not available\n", 40, 2435);

    # Pre Step 2- create cmd files for old user config
    if ($rootpath) {
        my $maincf=EDR::readfile_sys($sys,"$rootpath$prod->{configdir}/main.cf");
        EDR::cmd_sys($sys, "_cmd_mv $rootpath$prod->{configdir}/main.cf $rootpath$prod->{configdir}/main.cf.$edr->{uuid}");
        $maincf=~s/\s*include\s+\"(\/\S+)\"/\ninclude \"$rootpath$1\"/g;
        EDR::writefile_sys($sys,$maincf,"$rootpath$prod->{configdir}/main.cf");
    }

    # Verify if main.cf is valid before upgrade.
    $rtn= EDR::cmd_sys($sys, "$prod->{cmd_hacf} -verify $rootpath$prod->{configdir}");
    if (EDR::cmdexit() != 0) {
        $msg = Msg::new("main.cf is not valid:\n$rtn\nFix the errors, and verify the main.cf file before upgrade by running \'$prod->{cmd_hacf} -verify $rootpath$prod->{configdir}\'", 40, 3539, "$rtn", "$prod->{cmd_hacf}", "$rootpath$prod->{configdir}");
        EDR::push_error_sys($sys,$msg);
        return 0;
    }

    $rtn = $prod->translate_file_cf2cmd_sys( $sys, "$rootpath$prod->{configdir}", 1);
    EDR::cmd_sys($sys, "_cmd_mv $rootpath$prod->{configdir}/main.cf.$edr->{uuid} $rootpath$prod->{configdir}/main.cf") if ($rootpath);
    unless($rtn) {
        EDR::push_error_sys($sys,$msg);
        return 0;
    }

    # Fix 2026573.
    $prod->padv_mend_old_cf_sys($sys, "$upgradepath/old_config");
    $prod->padv_mend_old_cf_sys($sys, "$upgradepath/old_conf");

    # Pre Step 3- create cmd files for old sys conf
    $rtn = $prod->translate_file_cf2cmd_sys( $sys, "$upgradepath/old_conf");
    unless($rtn) {
        EDR::push_error_sys($sys,$msg);
        return 0;
    }

    return 1;
}

sub dynupgrade_upgrade_sys {
    my ($lines_ref,$msg,$prod,$rtn,$sourcefilebase,$sys);
    ($prod,$sys)= @_;
    my $cpic = Obj::cpic();
    my $rootpath=Cfg::opt("rootpath");
    my $upgradepath = EDR::tmpdir()."/upgrade";
    my ($treeref_oldsys, $treeref_olduser, $treeref_newsys, $treeref_result, $treeref_custom, $deleted_attrs_ref, $typename);

    # below process steps will ensure to create the result cmd list file after upgrade

    # Post Step 1- backup new system cf files 
    $rtn = $prod->backup_cf_files_sys( $sys, "$rootpath$prod->{confdir}", "$upgradepath/new_conf");
    return 0 unless ($rtn);
    
    # Post Step 2- create cmd files
    $msg = Msg::new("Failed to upgrade because the system configuration files are not available", 40, 3722);
    # for new sys conf
    $rtn = $prod->translate_file_cf2cmd_sys( $sys, "$upgradepath/new_conf");
    unless($rtn) {
        EDR::push_error_sys($sys,$msg);
        return 0;
    }
    # for old sys conf
    $rtn = $prod->translate_file_cf2cmd_sys( $sys, "$upgradepath/old_conf");
    unless($rtn) {
        EDR::push_error_sys($sys,$msg);
        return 0;
    }
    # old user config
    $rtn = $prod->translate_file_cf2cmd_sys( $sys, "$upgradepath/old_config", 1);
    unless($rtn) {
        EDR::push_error_sys($sys,$msg);
        return 0;
    }

    # Post Step 3- create the result directory
    EDR::cmd_sys($sys, "_cmd_mkdir -p $upgradepath/result_config");

    # calculate cmd list files existed
    unless ( EDR::file_sys( $sys, "$upgradepath/old_config/main.cmd" ) && 
        EDR::file_sys( $sys, "$upgradepath/old_conf/main.cmd" ) && 
        EDR::file_sys( $sys, "$upgradepath/new_conf/main.cmd" ) ) {
        
        $msg = Msg::new("Dynamic upgrade failed because required configuration files are not available", 40, 3540);
        EDR::push_warning_sys($sys,$msg);
        return 0;
    }

    # Post Step 4- load cmd list files
    $treeref_oldsys = dynupgrade_import_file2tree_sys( $sys, "$upgradepath/old_conf/main.cmd");
    $lines_ref = dynupgrade_import_file2lines_sys($sys, "$upgradepath/old_config/main.cmd");
    # update extra configuration
    $lines_ref = $prod->maincf_upgrade_sys($sys,$lines_ref);

    $treeref_olduser= dynupgrade_import_lines2tree($lines_ref);
    $treeref_newsys = dynupgrade_import_file2tree_sys( $sys, "$upgradepath/new_conf/main.cmd");
    
    # Post Step 5- find customized list
    $deleted_attrs_ref= dynupgrade_get_removed_attrs( $treeref_oldsys, $treeref_newsys );
    $treeref_custom= dynupgrade_minus_trees( $treeref_olduser, $treeref_oldsys);
    dynupgrade_export_tree2file_sys( $sys, "$upgradepath/result_config/custom.cmd", $treeref_custom);        
    
    # Post Step 6- merge the cumstomized change list into the new system tree
    $treeref_result= dynupgrade_merge_trees_A_over_B( $treeref_custom, $treeref_newsys);
    
    # Post Step 7- delete obsolete types and validate result tree 
    for $typename (@{$prod->{deleted_agents}}) {
        delete ($treeref_result->{TYPE}->{$typename}) if (defined($treeref_result->{TYPE}->{$typename}));
    }
    for $typename (keys %{$prod->{rename_attrs}}) {
        for my $attr (keys %{$prod->{rename_attrs}->{$typename}}) {
            dynupgrade_rename_attr_name($treeref_result, $typename, $attr, $prod->{rename_attrs}->{$typename}->{$attr});
        }
    }
    dynupgrade_validate_tree( $treeref_result, $deleted_attrs_ref);

    # Change sourcefile to original values.
    foreach my $group (sort keys %{$sys->{maincf_sourcefile_GRP}}) {
        my $sourcefile = $sys->{maincf_sourcefile_GRP}{$group};
        dynupgrade_set_sourcefile(
            $treeref_result,"GRP",$group,"\"$sourcefile\"");
    }
    foreach my $typename (sort keys %{$sys->{maincf_sourcefile_TYPE}}) {
        my $sourcefile = $sys->{maincf_sourcefile_TYPE}{$typename};
        dynupgrade_set_sourcefile(
            $treeref_result,"TYPE",$typename,"\"$sourcefile\"");
    }

    # Customized update the result tree. 
    # Only update the configurations that required the new configuration files.
    $treeref_result = $prod->maincf_result_upgrade_sys($sys, $treeref_result, $treeref_newsys);

    # Post Step 8- export tree into file
    dynupgrade_export_tree2file_sys( $sys, "$upgradepath/result_config/main.cmd", $treeref_result);

    if( EDR::file_sys( $sys, "$upgradepath/result_config/main.cmd" ) ) {
        # distribute updated *.cf to each node
        unless ($sys->{islocal}) {
            EDR::cmd_local("_cmd_mkdir -p $upgradepath/result_config");
            EDR::copy_sys($sys,"$upgradepath/result_config/main.cmd","$upgradepath/result_config/main.cmd",1);
        }
        # put the new main.cmd into the log
        EDR::cmd_sys($sys,"_cmd_cat $upgradepath/result_config/main.cmd");

        for $sys(@{$cpic->{systems}}) {
            EDR::copy_sys( $sys, "$upgradepath/result_config/main.cmd","$rootpath$prod->{configdir}/main.cmd");
            # create the .cf files based on the command list
            my $ret= $prod->translate_file_cmd2cf_sys( $sys, "$rootpath$prod->{configdir}");
            unless( $ret) {
                $msg = Msg::new("Dynamic upgrade failed because the configuration files are not valid", 40, 3541);
                EDR::push_warning_sys($sys,$msg);
                return 0;
            }

            # Remove $rootpath from main.cf
            if ($rootpath) {
                my $maincf=EDR::readfile_sys($sys,"$rootpath$prod->{configdir}/main.cf");
                $maincf=~s/\s*include\s+\"$rootpath(\/\S+)\"/\ninclude \"$1\"/g;
                EDR::writefile_sys($sys,$maincf,"$rootpath$prod->{configdir}/main.cf");
            }

            if ($sys->system1) {
                # put the new main.cf into the log
                EDR::cmd_sys($sys,"_cmd_cat $rootpath$prod->{configdir}/main.cf");
            }
            EDR::cmd_sys($sys,"_cmd_rmr $rootpath$prod->{configdir}/main.cmd");
        }
    } else {
        $msg = Msg::new("Dynamic upgrade failed because main.cmd is not available", 40, 3542);
        EDR::push_warning_sys($sys,$msg);
        return 0;
    }

    # Copy nfs_postoffline during upgrade.
    # But in 5.1SP1, the nfs related triggers will be moved back, so detete the copy
#    my @reslist= dynupgrade_get_resname_list_from_typename( $treeref_result, "NFSRestart" );
#    if (@reslist) {
#        Msg::log("There is NFSRestart resource in main.cf; Copy nfs_postoffline from /opt/VRTSvcs/bin/sample_triggers to /opt/VRTSvcs/bin/triggers.");
#        for $sys(@{$cpic->{systems}}) {
#            if (EDR::file_sys($sys, "$rootpath/opt/VRTSvcs/bin/sample_triggers/nfs_postoffline")) {
#                EDR::cmd_sys($sys, "_cmd_mkdir -p $rootpath/opt/VRTSvcs/bin/triggers");
#                EDR::cmd_sys($sys, "_cmd_cp $rootpath/opt/VRTSvcs/bin/sample_triggers/nfs_postoffline $rootpath/opt/VRTSvcs/bin/triggers/");
#            }
#        }
#    }

    return 1;
}

# restore extra informations for main.cf upgrade
sub maincf_upgrade_sys {
    my($prod,$sys,$line,$lines_ref,$category);
    ($prod,$sys,$lines_ref) = @_;

    # Update attributes StartVolumes and StopVolumes
    for $line (@{$lines_ref}) {
        if ($line=~/(hares\s+\-modify\s+\w+\s+(Start|Stop)Volumes)\s+(?!1\b)/) {
            Msg::log("Modify main.cmd from: $line");
            $line="$1 0";
            Msg::log("                  To: $line");
        }
        if ($line=~/(haattr\s+\-add\s+DiskGroup\s+(Start|Stop)Volumes)\s+\-string\s+(\S+)/) {
            Msg::log("Modify main.cmd from: $line");
            $line=($3 eq '1') ? "$1 -boolean 1" : "$1 -boolean 0";
            Msg::log("                  To: $line");
        }
    }

    foreach $category (sort keys %{$sys->{maincf_upgrade}}) {
        next unless ($sys->{maincf_upgrade}{$category});
        push(@{$lines_ref},split(/\n+/,$sys->{maincf_upgrade}{$category}));
        Msg::log("Upgrade main.cmd for user configuration: $sys->{maincf_upgrade}{$category}\n");
    }
    return $lines_ref;
}

# This sub is used for upgrade the main.cmd based on the new configuration files.
sub maincf_result_upgrade_sys {
    my ($prod,$sys,$treeref_result,$treeref_newsys) = @_;

    # Update NFSRestart ResRecord
    $treeref_result = maincf_result_upgrade_NFSRestart($sys,$treeref_result);

    # Update Oracle resource
    $treeref_result = $prod->maincf_result_upgrade_Oracle_sys($sys, $treeref_result);

    # update containrization settings.
    $treeref_result = $prod->padv_maincf_upgrade_sys($sys, $treeref_result, $treeref_newsys) if ($prod->can('padv_maincf_upgrade_sys'));;
    dynupgrade_modify_attr_value($treeref_result,"Sybase","Monscript","/opt/VRTSvcs/bin/Sybase/SqlTest.pl","/opt/VRTSagents/ha/bin/Sybase/SqlTest.pl");
    return $treeref_result;
}

# update the Oracle and Netlsnr resource when upgrade from 5.1 and lower to 5.1SP1
# for Oracle res, override LevelTwoMonitorFreq attr according to DetailMonitor attr,
#                 and set DetailMonitor to 1 if it is not 0;
# for Netlsnr, just override LevelTwoMonitorFreq attr and set it to 1.
sub maincf_result_upgrade_Oracle_sys {
    my ($prod,$sys,$treeref) = @_;
    my (@reslist,$res,$res_ref,$attr_ref,$attr_ref_in,$attr_value);
    my ($result_lines);
    my ($vcsvers);

    # check previous prod version
    for my $index (0..$#{$sys->{upgradeprod}}){
        if (${$sys->{upgradeprod}}[$index] =~ /^VCS/){
            $vcsvers = ${$sys->{prodvers}}[$index];
            last;
        }
    }
    return $treeref unless ($vcsvers && (EDR::compvers($vcsvers,"5.1.100") == 2));

    @reslist = dynupgrade_get_resname_list_from_typename($treeref,"Oracle");
    for $res (@reslist) {
        $res_ref = dynupgrade_get_node_from_tree($treeref,"RES",$res);
        $attr_ref = dynupgrade_get_attr_from_node($res_ref,"DetailMonitor");
        $attr_ref_in = dynupgrade_get_attr_from_node($res_ref,"LevelTwoMonitorFreq");
        $result_lines .= "hares -override $res LevelTwoMonitorFreq\n" unless $attr_ref_in;
        if ( $attr_ref ) {
            $attr_value = $attr_ref->{MODIFY}->{ATTRVAL}->[0];
            $result_lines .= "hares -modify $res LevelTwoMonitorFreq $attr_value\n";
            $result_lines .= "hares -modify $res DetailMonitor 1\n" unless $attr_value == 0;
        } else {
            $result_lines .= "hares -modify $res LevelTwoMonitorFreq 0\n";
        }
    }
    @reslist = dynupgrade_get_resname_list_from_typename($treeref,"Netlsnr");
    for $res (@reslist) {
        $attr_ref = dynupgrade_get_attr_from_node($res_ref,"LevelTwoMonitorFreq");
        $result_lines .= "hares -override $res LevelTwoMonitorFreq\n" unless $attr_ref;
        $result_lines .= "hares -modify $res LevelTwoMonitorFreq 1\n";
    }

    my @lines= split(/\n+/, $result_lines);
    $treeref= dynupgrade_import_lines2tree(\@lines,$treeref);
    return $treeref;
}

# update the NFSRestart related resources when dynupgrade:
# 1. Insert a new res NFSRestart_l into res_grps which already have NFSRestart res, new res is the same as old except Lower attr
# 2. If these groups have Share share_res, change require relation: share_res -> [child] to share_res -> NFSRestart_l -> [child].
sub maincf_result_upgrade_NFSRestart {
    my ($sys,$treeref) = @_;
    my (@reslist,@NFSRestart_groups,$res,$result_lines,$res_ref,$res_group_name,$attr,$attr_ref,$res_added);
    my (@share_reslist,$share_res,$share_res_ref);
#    my ($link_ref,$padv,$vcspkg,$vcsvers);
    my ($link_ref,$vcsvers);

    for my $index (0..$#{$sys->{upgradeprod}}){
        if (${$sys->{upgradeprod}}[$index] =~ /^VCS/){
            $vcsvers = ${$sys->{prodvers}}[$index];
            last;
        }
    }

#    $padv = $sys->{padv};
#    $vcspkg=$Obj::pool{"Pkg\::VRTSvcs51\::$padv"};
#    if (defined $vcspkg) {
#        $vcsvers=$vcspkg->version_sys($sys);
#    }
    return $treeref unless ($vcsvers && (EDR::compvers($vcsvers,"5.1.100") == 2));

    @reslist = dynupgrade_get_resname_list_from_typename($treeref,"NFSRestart");
    for $res (@reslist) {
        $res_ref = dynupgrade_get_node_from_tree($treeref,"RES",$res);
        $res_group_name = $res_ref->{MAIN}->{SERVGRP};
        next if EDR::inarr($res_group_name,@NFSRestart_groups);
        push(@NFSRestart_groups, $res_group_name);
        $res_added = 0;

        #update the "require relations"
        @share_reslist = dynupgrade_get_resname_list_from_typename($treeref,"Share");
        for $share_res (@share_reslist){
            $share_res_ref = dynupgrade_get_node_from_tree($treeref,"RES",$share_res);
            next unless $share_res_ref->{MAIN}->{SERVGRP} eq $res_group_name;

            unless ($res_added){
                #add another NFSRestart resource
                $result_lines .= "hares -add NFSRestart_L_$res_group_name NFSRestart $res_group_name\n";

                #copy resouce attributes
                for $attr (keys %{$res_ref->{ATTR}}){
                    $attr_ref = dynupgrade_get_node_from_tree($res_ref,"ATTR",$attr);
                    $result_lines .= "hares -modify NFSRestart_L_$res_group_name $attr $attr_ref->{MODIFY}->{ATTRVAL}->[0]\n";
                }
                $result_lines .= "hares -modify NFSRestart_L_$res_group_name Lower 1\n";
                $res_added = 1;            
            }

            for $link_ref (@{$treeref->{RESLINK}->{RESLINK}->{RESLINK}}){
                next unless $link_ref->{RESPARENT} eq $share_res;
                $result_lines .= "hares -unlink $link_ref->{RESPARENT} $link_ref->{RESCHILD}\n";
                $result_lines .= "hares -link NFSRestart_L_$res_group_name $link_ref->{RESCHILD}\n"
         unless $result_lines =~ "hares -link NFSRestart_L_$res_group_name $link_ref->{RESCHILD}\n";
            }
            $result_lines .= "hares -link $share_res NFSRestart_L_$res_group_name\n";
        }
    }
    my @lines= split(/\n+/, $result_lines);
    $treeref= dynupgrade_import_lines2tree(\@lines,$treeref);
    return $treeref;
}

sub upgrade_poststart_sys {
    my($prod,$sys) = @_;
    return 0 unless ($sys->{vcs_upgradeable});
    $prod->haconf_makerw();
    $prod->unfreeze_groups();
    $prod->haconf_dumpmakero();
}

sub upgrade_gabtab_sys {
    my(@l,$line,$gabtab,$ngabtab,$prod,$sys,$sysi,$tmpdir);
    ($prod,$sys) = @_;
    my $rootpath = Cfg::opt("rootpath");
    $sysi = $sys->{sys};
    $tmpdir = EDR::tmpdir();
    Msg::log("Updating gabtab on $sysi");
    $gabtab=EDR::cmd_sys($sys, "_cmd_cat $rootpath/etc/gabtab");
    @l=split(/\n/,$gabtab);
    foreach $line(@l) {
        next if ($line=~/gabdiskhb/);
        $ngabtab.="$line\n";
    }
    EDR::writefile($ngabtab, "$tmpdir/gabtab.$sysi");
    EDR::cmd_local("_cmd_chmod 644 $tmpdir/gabtab.$sysi");
    EDR::copy_sys($sys, "$tmpdir/gabtab.$sysi","$rootpath/etc/gabtab");
    return 1;
}

sub backup_cf_files_sys {
    my ($msg,$prod,$sys,$src,$dest);
    ($prod,$sys,$src,$dest)= @_;

    if(EDR::dir_sys($sys,$src)) {
        unless(EDR::dir_sys($sys,$dest)) {
            EDR::cmd_sys($sys,"_cmd_mkdir -p $dest");
        }
        EDR::cmd_sys($sys,"_cmd_cp -pf $src/*.cf $dest\/");
    } else {
        $msg = Msg::new("Backup of cf files on $sys->{sys} failed because $src directory was not found", 40, 2441, "$sys->{sys}", "$src");
        EDR::push_error_sys($sys,$msg);
        return 0;
    }
    return 1;
}

sub display_unsupported_agents_msg {
    my ($prod,$deleted_types_resources_ref) = @_;
    my ($logmsg,$msg,$typename,$resource);
    my $nfslockexist = 0;

    $msg=Msg::new("The installer has detected that your existing configuration contains the following types and resources that are no longer supported in $prod->{abbr} $prod->{vers}:\n", 40, 2442, "$prod->{abbr}", "$prod->{vers}");
    $logmsg=$msg->{msg};
    foreach $typename ( keys %{$deleted_types_resources_ref} ) {
        $msg=Msg::new("\tType : $typename\n", 40, 2443, "$typename");
        $logmsg.=$msg->{msg};
        # display type related resources
        $msg=Msg::new("\t$typename\'s Resource(s) : ${$deleted_types_resources_ref}{$typename} \n\n", 40, 2444, "$typename", "${$deleted_types_resources_ref}{$typename}");
        $logmsg.=$msg->{msg};
        # check NFSLock specifically
        if ("$typename" eq "NFSLock") {
            $nfslockexist = 1;
        }
    }

    # check with users if they hope to erase the obsolete types and resources manually 
    $msg = Msg::new("Obsolete types and resources can be erased from the system either automatically by the installer, or manually by the user\n\n", 40, 2445);
    $logmsg.=$msg->{msg};
    $msg = Msg::new("If you want to remove the obsolete types and resources manually, then follow the steps below:\n", 40, 2446);
    $logmsg.=$msg->{msg};
    $msg = Msg::new("    Step 1 - Quit the installer\n", 40, 2447);
    $logmsg.=$msg->{msg};
    $msg = Msg::new("    Step 2 - Stop $prod->{abbr} if it is running\n", 40, 2448, "$prod->{abbr}");
    $logmsg.=$msg->{msg};
    $msg = Msg::new("    Step 3 - Edit main.cf file to remove the obsolete resources\n", 40, 2449);
    $logmsg.=$msg->{msg};
    $msg = Msg::new("    Step 4 - Edit types.cf file to remove the obsolete types\n", 40, 2450);
    $logmsg.=$msg->{msg};
    $msg = Msg::new("    Step 5 - Restart $prod->{abbr}, and rerun the installer to proceed with the upgrade\n\n", 40, 2451, "$prod->{abbr}");
    $logmsg.=$msg->{msg};
    $msg = Msg::new("If you want to remove the obsolete types and resources automatically using the installer, then proceed with the upgrade\n\n", 40, 2452);
    $logmsg.=$msg->{msg};
    if ($nfslockexist == 1) {
        $logmsg.="\n";
        $msg = Msg::new("Note: The NFSLock agent is no longer supported in $prod->{abbr} $prod->{vers} and is now replaced by the new NFSRestart agent. Refer to the $prod->{prod} $prod->{vers} Bundled Agents Reference Guide for details on how to configure a NFSRestart resource.\n", 40, 2453, "$prod->{abbr}", "$prod->{vers}", "$prod->{prod}", "$prod->{vers}");
        $logmsg.=$msg->{msg};
    }
    return $logmsg;
}

sub check_obsolete_types_sys {
    my ($msg,$prod,$rtn,$sys,$typename,$treeref_olduser,$typeref);
    ($prod,$sys,$treeref_olduser) = @_;
    my @deleted_types= ();
    my %deleted_types_resources= ();

    # below steps are used for checking the obsolete types with customer

    # precheck step 3 - import old user cmd list for checking
    # $treeref_olduser= dynupgrade_import_lines2tree($lines_ref);

    # precheck step 5 - find obsolete types
    
    foreach $typename ( @{$prod->{deleted_agents}} ) {
        $typeref=  $treeref_olduser->{TYPE}->{$typename};
        if( defined( $typeref) ) {
            push( @deleted_types, $typename);
        }
    }

    # precheck step 6 - find obsolete resources
    foreach $typename ( @deleted_types ) {
        my @reslist= dynupgrade_get_resname_list_from_typename( $treeref_olduser, $typename );
        if( scalar(@reslist) > 0) {
            $deleted_types_resources{$typename}= join( "  ", @reslist);
        }
    }

    # precheck step 7 - check with user on the obsolete types/resources
    if( scalar (keys %deleted_types_resources )> 0 ) {
        $rtn=$prod->display_unsupported_agents_msg( \%deleted_types_resources );
        $msg=Msg::new($rtn);
        EDR::push_warning_sys($sys,$msg);
    }
    # precheck step 8 - go on installation
    return 1;
}

sub freeze_groups {
    my ($cpic,$grp,@grps,$line,$prod,$sys,$rtn);
    $prod = shift;
    $cpic = Obj::cpic();
    $sys = ${$cpic->{systems}}[0];

    $rtn = EDR::cmd_sys($sys,"$prod->{bindir}/hagrp -list Frozen=0 2>/dev/null"); 
    for $line (split(/\n+/,$rtn)) {
        ($grp)=split(/\s+/,$line, 2);
        if($grp && !EDR::inarr($grp,@grps)) {
            push(@grps,$grp);
        }        
    }
    for $grp (@grps) {
        next if ($grp eq "ClusterService"); # Do not freeze Service Group : ClusterService 
        Msg::log("Freezing group: $grp");
        EDR::cmd_sys($sys,"$prod->{bindir}/hagrp -freeze $grp -persistent");
    }
}

sub unfreeze_groups {
    my ($cpic,$grp,@grps,$line,$prod,$rtn,$sys);
    $prod = shift;
    $cpic = Obj::cpic();
    $sys = ${$cpic->{systems}}[0];
    $rtn = EDR::cmd_sys($sys, "$prod->{bindir}/hagrp -list Frozen=1 2>/dev/null");
    for $line (split(/\n+/,$rtn)) {
        ($grp)=split(/\s+/,$line, 2);
        if($grp && !EDR::inarr($grp,@grps)) {
            push(@grps,$grp);
        }
    }
    for $grp (@grps) {
        Msg::log("Unfreeze group: $grp");
        EDR::cmd_sys($sys,"$prod->{bindir}/hagrp -unfreeze $grp -persistent");
    }    
}

sub haconf_dumpmakero {
    my ($cpic,$prod,$sys,$rtn);
    $prod = shift;
    $cpic = Obj::cpic();
    $sys = ${$cpic->{systems}}[0];
    $rtn = EDR::cmd_sys($sys,"$prod->{bindir}/haclus -value ReadOnly");
    if ($rtn eq "0") {
        EDR::cmd_sys($sys,"$prod->{bindir}/haconf -dump -makero");
        sleep 5;
    }
}

sub haconf_makerw {
    my ($cpic,$prod,$sys,$rtn);
    $prod = shift;
    $cpic = Obj::cpic();
    $sys = ${$cpic->{systems}}[0];
    $rtn = EDR::cmd_sys($sys,"$prod->{bindir}/haclus -value ReadOnly");
    if ($rtn eq "1") {
        EDR::cmd_sys($sys,"$prod->{bindir}/haconf -makerw");
        sleep 3;
    }    
}

# return an array of deleted agents found in existing main.cf
sub get_unsupported_agents_sys {
    my (@agents,$agent,$prod,$sys,$out,$maincf);
    ($prod,$sys) = @_;

    $maincf=EDR::cmd_sys($sys, "_cmd_cat $prod->{maincf}");
    $maincf=~s/\/\/.*\n//g;
    foreach $agent(@{$prod->{deleted_agents}}) {
        if ($maincf=~/\b$agent\b/) {
            push(@agents, $agent);
        }
    }
    return \@agents;
}

sub translate_file_cmd2cf_sys {
    my ($prod,$sys,$path)= @_;
    my $cmd_hacf= $prod->{cmd_hacf};

    # create main.cmd for .cf files
    EDR::cmd_sys( $sys, "$cmd_hacf -cmdtocf $path" );
    return 0 if(EDR::cmdexit());

    # verify effectiveness of .cf files
    my $ret= EDR::cmd_sys( $sys, "$cmd_hacf -verify $path" );
    return 0 if($ret);
    return 1;
}

sub translate_file_cf2cmd_sys {
    my ($cmd_hacf,$is_userconfig,$maincf,$prod,$path,$sys,$rtn);
    ($prod,$sys,$path,$is_userconfig)= @_;
    $cmd_hacf= $prod->{cmd_hacf};

    unless(EDR::dir_sys($sys,$path)) {
        return 0;
    }
    # deal with system configuraiton directory (conf)=> create a main.cf file only including types.cf
    if(!$is_userconfig) {
        # create a null main.cf for sys
        $maincf= "include \"types.cf\"\n";
        for my $category (sort keys %{$sys->{maincf_include}}) {
            my $extra_type = $sys->{maincf_include}{$category};
            next unless ($extra_type);
            if (EDR::file_sys($sys, "$path/$extra_type")) {
                $maincf.="include \"$extra_type\"\n";
            }
        }
        # output to updated types.cf
        EDR::writefile_sys($sys, $maincf, "$path/main.cf" );
    }
    # verify effectiveness of .cf files
    $rtn = EDR::cmd_sys($sys, "$cmd_hacf -verify $path");
    return 0 if($rtn);
    # create main.cmd for .cf files
    EDR::cmd_sys($sys, "$cmd_hacf -cftocmd $path");
    return 1 if (EDR::file_sys($sys,"$path/main.cmd"));
    return 0;
}

sub padv_mend_old_cf_sys {
    my ($prod, $sys, $path) = (@_);
    return 1;
}

sub dynupgrade_get_branch_from_tree {
    my ($treeref, $branch_name)= @_;
    return $treeref->{$branch_name};
}

sub dynupgrade_get_node_from_tree {
    my ($treeref, $branch_name, $node_name)= @_;
    my $branchref= dynupgrade_get_branch_from_tree( $treeref, $branch_name);
    return $branchref->{$node_name};
}

sub dynupgrade_get_attr_from_tree {
    my ($treeref, $branch_name, $node_name, $attr_name)= @_;
    my $noderef= dynupgrade_get_node_from_tree( $treeref, $branch_name, $node_name);
    return $noderef->{ATTR}->{$attr_name};
}

sub dynupgrade_get_node_from_branch {
    my ($branchref, $node_name)= @_;
    return $branchref->{$node_name};
}

sub dynupgrade_get_attr_from_branch {
    my ($branchref, $node_name, $attr_name)= @_;
    my $noderef= dynupgrade_get_node_from_branch( $branchref, $node_name);
    return $noderef->{ATTR}->{$attr_name};
}

sub dynupgrade_get_attr_from_node {
    my ($noderef, $attr_name)= @_;
    return $noderef->{ATTR}->{$attr_name};
}

sub dynupgrade_get_resname_list_from_typename {
    my ($treeref, $type_name)= @_;
    my @resname_list= ();
    my $branchref= dynupgrade_get_branch_from_tree( $treeref, "RES");
    my ( $resname, $noderef, $cmdref);
    
    foreach $resname ( keys %{$branchref} ) {
        # get each resource node and main command
        $noderef= dynupgrade_get_node_from_branch( $branchref, $resname);
        $cmdref= $noderef->{MAIN};
        # compare the type name with required one
        if( $cmdref->{RESTYPE} eq $type_name) {
            push ( @resname_list, $resname);
        }
    }
    
    return @resname_list;
}

# Responsibility: To find the obsolete types-attributes matrix from the old tree
# Input:          Tree references
# Output:         Obsolete attributes
sub dynupgrade_get_removed_attrs {
    my $brachref_old= $_[0]->{TYPE};
    my $brachref_new= $_[1]->{TYPE};
    my ($typeref_old, $typeref_new, $typename, $attrname);
    my %deleted_attrs= ();

    # to browse each type node from TYPE branch
    foreach $typename (keys %{$brachref_old} ) {
        $typeref_old= dynupgrade_get_node_from_branch( $brachref_old, $typename);
        $typeref_new= dynupgrade_get_node_from_branch( $brachref_new, $typename);
        if( !$typeref_old || !$typeref_new) {
            next;
        }
        # to browse each attribute from type node
        foreach $attrname (keys %{$typeref_old->{ATTR}}) {
            my $attrref_old= dynupgrade_get_attr_from_node( $typeref_old, $attrname);
            my $attrref_new= dynupgrade_get_attr_from_node( $typeref_new, $attrname);
            if( defined( $attrref_old) && !defined( $attrref_new) ) {
                # to add the first obsolete attribute
                if( !defined($deleted_attrs{$typename}) ) {
                    $deleted_attrs{$typename}= [$attrname];
                } else {
                    # to add more obsolete attribute
                    push ( @{$deleted_attrs{$typename}}, $attrname);
                }
            }
        }
    }
    return \%deleted_attrs;
}

# Responsibility: To merge tree B by tree A
# Input:          Tree references
# Output:         Merged tree
sub dynupgrade_merge_trees_A_over_B {
    return dynupgrade_merge_trees_B_over_A( $_[1], $_[0]);
}

# Responsibility: To merge tree A by tree B
# Input:          Tree references
# Output:         Merged tree
sub dynupgrade_merge_trees_B_over_A {
    my ($treeref1, $treeref2)= @_;
    my $treeref_result= undef;

    #merge types
    $treeref_result= dynupgrade_merge_branches_B_over_A( $treeref1, $treeref2, "TYPE");

    #clusters/systems/resources/groups/links/others that are from main.cf should only be copied
    $treeref_result->{CLUS}   = $treeref2->{CLUS};
    $treeref_result->{SYS}    = $treeref2->{SYS};
    $treeref_result->{GRP}    = $treeref2->{GRP};
    $treeref_result->{RES}    = $treeref2->{RES};
    $treeref_result->{GRPLINK}= $treeref2->{GRPLINK};
    $treeref_result->{RESLINK}= $treeref2->{RESLINK};
    $treeref_result->{OTHER}  = $treeref2->{OTHER};

    return $treeref_result;
}

# Responsibility: To merge tree A by tree B with a branch
# Input:          tree references; tree branch
# Output:         merged tree
sub dynupgrade_merge_branches_B_over_A {
    my ($treeref1, $treeref2, $branch_name)= @_;
    my ($nodename, $noderef1, $noderef2, $treeref_result );

    #copy tree 1 to result
    %{$treeref_result}= %{$treeref1};

    #merge nodes
    foreach $nodename (keys %{$treeref2->{$branch_name}} ) {
        $noderef1= dynupgrade_get_node_from_tree( $treeref_result, $branch_name, $nodename);
        $noderef2= dynupgrade_get_node_from_tree( $treeref2, $branch_name, $nodename);
        
        #new node => added to list
        if( !defined( $noderef1) ) {
            $treeref_result->{$branch_name}->{$nodename}= $noderef2;
            next;
        }
        #overwrite
        $treeref_result->{$branch_name}->{$nodename}= dynupgrade_merge_nodes_B_over_A( $noderef1, $noderef2);
    }

    return $treeref_result;
}

# Responsibility: To merge node A by Node B
# Input:          node references; etc.
# Output:         merged node
sub dynupgrade_merge_nodes_B_over_A {
    my ($noderef1, $noderef2)= @_;
    my ( $attrname, $attrref1, $attrref2);
    my $noderef_result= {};

    #set result as noderef1
    %{$noderef_result}= %{$noderef1};

    #main cmd
    $noderef_result->{MAIN}= $noderef2->{MAIN};
    #file cmd
    $noderef_result->{FILE}= $noderef2->{FILE};
    
    #attributes
    foreach $attrname ( keys %{$noderef2->{ATTR}} ) {
        # exceptional for ArgList => reserve the values to be the ones from new version
        if( $attrname eq "ArgList" ) {
            next;
        }
        # other attributes
        $attrref1= dynupgrade_get_attr_from_node( $noderef_result, $attrname );
        $attrref2= dynupgrade_get_attr_from_node( $noderef2, $attrname );

        %{$attrref1}= %{$attrref2};
        $noderef_result->{ATTR}->{$attrname}= $attrref1;
    }
    return $noderef_result;
}

# Responsibility: Tree A minus tree B
# Input:          tree references
# Output:         Result tree
sub dynupgrade_minus_trees {
    my ($treeref1, $treeref2)= @_;
    my ( $treeref_result, $treeref_temp );

    # minus types only
    $treeref_temp= dynupgrade_minus_treebranches( $treeref1, $treeref2, "TYPE");
    $treeref_result->{TYPE}= $treeref_temp->{TYPE};

    # cluster/system/resources/groups/links/other should only be reserved for main.cf
    $treeref_result->{CLUS}   = $treeref1->{CLUS};
    $treeref_result->{SYS}    = $treeref1->{SYS};
    $treeref_result->{RES}    = $treeref1->{RES};
    $treeref_result->{GRP}    = $treeref1->{GRP};
    $treeref_result->{GRPLINK}= $treeref1->{GRPLINK};
    $treeref_result->{RESLINK}= $treeref1->{RESLINK};
    $treeref_result->{OTHER}  = $treeref1->{OTHER};

    return $treeref_result;
}

# Responsibility: Tree A minus tree B with branch
# Input:          tree references; branch name
# Output:         Result tree
sub dynupgrade_minus_treebranches {
    my ($treeref1, $treeref2, $branch_name)= @_;
    my ( $nodename, $noderef1, $noderef2, $noderef_result ) ;
    my $treeref_result  = {};

    # minus objects
    foreach $nodename (keys %{$treeref1->{$branch_name}} ) {
        $noderef1= dynupgrade_get_node_from_tree( $treeref1, $branch_name, $nodename );
        # object that should be reserved
        if( !defined( dynupgrade_get_node_from_tree( $treeref2, $branch_name, $nodename) ) ) {
            $treeref_result->{$branch_name}->{$nodename}= $noderef1;
            next;
        }

        # find the corresponding object
        $noderef2= dynupgrade_get_node_from_tree( $treeref2, $branch_name, $nodename );
        $noderef_result= dynupgrade_minus_nodes( $noderef1, $noderef2);
        if( $noderef_result ) {
            $treeref_result->{$branch_name}->{$nodename}= $noderef_result;
        } else {
            delete( $treeref_result->{$branch_name}->{$nodename} );
        }
    }
    return $treeref_result;
}

# Responsibility: Node A minus node B
# Input:          node references
# Output:         Result node
sub dynupgrade_minus_nodes {
    my ($noderef1, $noderef2 )= @_;
    my ($cmp, $attrname, $attrref1, $attrref2);
    my $noderef_result  = {};

    # set result as noderef1
    %{$noderef_result}= %{$noderef1};

    # attrubutes
    foreach $attrname ( keys %{$noderef2->{ATTR}} ) {
        $attrref1= dynupgrade_get_attr_from_node( $noderef_result, $attrname);
        $attrref2= dynupgrade_get_attr_from_node( $noderef2, $attrname);

        # add cmd
        if( defined( $attrref1->{ADD} ) && defined( $attrref2->{ADD}) ) {
            $cmp= dynupgrade_compare_ha_cmds( $attrref1->{ADD}, $attrref2->{ADD});
            if( $cmp ne "EQUAL_ATTR_ADD_TYPENAME_ATTRNAME_ATTRVAL" ) {
                next;
            }
        }
        # defalut attribute cmd
        if( defined( $attrref1->{DEFAULT}) && defined( $attrref2->{DEFAULT}) ) {
            $cmp= dynupgrade_compare_ha_cmds( $attrref1->{DEFAULT}, $attrref2->{DEFAULT});
            if( $cmp ne "EQUAL_ATTR_DEFAULT_TYPENAME_ATTRNAME_ATTRVAL" ) {
                next;
            }
        }
        # modify cmd
        if( defined( $attrref1->{MODIFY}) && defined( $attrref2->{MODIFY}) ) {
            $cmp= dynupgrade_compare_ha_cmds( $attrref1->{MODIFY}, $attrref2->{MODIFY});
            if( $cmp ne "EQUAL_TYPE_MODIFY_TYPENAME_ATTRNAME_ATTRVAL" ) {
                next;
            }
        }
        # attribute totally match => remove attribute
        delete( $noderef_result->{ATTR}->{$attrname} );
    }
    # no attributes left => remove node
    if( (keys %{$noderef_result->{ATTR}}) == 0) {
        $noderef_result= undef;
    }

    return $noderef_result;
}

# Responsibility: To update a command to tree
# Input:          Tree reference; cmd reference
# Output:         Updated tree
sub dynupgrade_update_cmd2tree {
    my ($treeref, $cmdref)= @_;
    my $branch_name = $cmdref->{TREE_BRANCH};
    my $node_name   = $cmdref->{TREE_NODENAME};
    my $node_class  = $cmdref->{TREE_NODECLASS};
    my $attr_class  = $cmdref->{TREE_ATTRCLASS};
    
    #delete a node
    if( $node_class eq "DELETE") {
        delete( $treeref->{$branch_name}->{$node_name} );
        return;
    }
    # clusters/systems/others commands
    if( $branch_name eq "CLUS" ||
        $branch_name eq "SYS" ||
        $branch_name eq "OTHER" ) {
        if( !defined ( $treeref->{$branch_name}) ) {
            $treeref->{$branch_name}= [];
        }
        push( @{$treeref->{$branch_name}}, $cmdref);
        return;
    }
    
    #update type/res/group/resource_link/group_link command to related branch and node
    my $noderef= dynupgrade_get_node_from_tree( $treeref, $branch_name, $node_name );
    $treeref->{$branch_name}->{$node_name}= dynupgrade_update_cmd2node( $noderef, $cmdref, $node_class, $attr_class);
    return;
}

# Responsibility: To update a command to tree node
# Input:          tree node; cmd reference; etc.
# Output:         updated node
sub dynupgrade_update_cmd2node {
    my ($noderef, $cmdref, $node_class, $attr_class)= @_;
    my ( $attrref, $attrname);

    # init node if not defined
    if( !defined ($noderef) ) {
        $noderef= {"MAIN"=>undef, "FILE"=>undef, "ATTR"=>{}, "GRPLINK"=>[], "RESLINK"=>[]};
    }
    
    # to add the command to related branch of the node
    if( $node_class eq "MAIN" ||
        $node_class eq "FILE" ) {
        $noderef->{$node_class}= $cmdref;
        return $noderef;
    }
    if( $node_class eq "ATTR" ) {
        $attrname= $cmdref->{ATTRNAME};
        # delete attr
        if( $attr_class eq "DELETE") {
            delete( $noderef->{ATTR}->{$attrname} );
            return $noderef;
        }
        # add
        if( !defined( $noderef->{ATTR}->{$attrname}) ) {
            $noderef->{ATTR}->{$attrname}= {};
        }
        # other attr actions
        $attrref= $noderef->{ATTR}->{$attrname};
        if( $attr_class eq "MODIFY" && $cmdref->{CMD} =~ /^(hares|hagrp)$/) {
            if ( !defined($attrref->{$attr_class}->{ATTRVAL}) || scalar(@{$attrref->{$attr_class}->{ATTRVAL}}) <= 0
                 || ( $cmdref->{ATTRVAL} !~ /\s+\-sys\s+/ && $cmdref->{ATTRVAL} !~ /^(\s+)?\-add\s+/) ) {

                my $cmd_attrval= $cmdref->{ATTRVAL};
                $attrref->{$attr_class}= $cmdref;
                $attrref->{$attr_class}->{ATTRVAL}= [$cmd_attrval];
            } else {
                push ( @{$attrref->{$attr_class}->{ATTRVAL}}, $cmdref->{ATTRVAL} );
            }
        } else {
            # other attr actions
            $attrref->{$attr_class}= $cmdref;
        }
        return $noderef;
    }
    # add grp link 
    if( $node_class eq "GRPLINK" ) {
        if( !defined( $noderef->{$node_class} ) ) {
            $noderef->{$node_class}= [];
        }
        push( @{$noderef->{$node_class}}, $cmdref);
        return $noderef;
    }
    # grp unlink
    if( $node_class eq "GRPUNLINK") {
        for( my $idx= $#{$noderef->{GRPLINK}}; $idx>= 0; $idx-- ) {
            my $cmdref1= ${$noderef->{GRPLINK}}[$idx];
            if( $cmdref1->{GRPPARENT} eq $cmdref->{GRPPARENT} &&
                $cmdref1->{GRPCHILD} eq $cmdref->{GRPCHILD} ) {
                dynupgrade_del_from_array_by_index( $noderef->{GRPLINK}, $idx);
                return $noderef;
            }
        }
        return $noderef;
    }
    # add res link 
    if( $node_class eq "RESLINK" ) {
        if( !defined( $noderef->{$node_class} ) ) {
            $noderef->{$node_class}= [];
        }
        push( @{$noderef->{$node_class}}, $cmdref);
        return $noderef;
    }
    # res unlink
    if( $node_class eq "RESUNLINK") {
        for( my $idx= $#{$noderef->{RESLINK}}; $idx>= 0; $idx-- ) {
            my $cmdref1= ${$noderef->{RESLINK}}[$idx];
            if( $cmdref1->{RESPARENT} eq $cmdref->{RESPARENT} &&
                $cmdref1->{RESCHILD} eq $cmdref->{RESCHILD} ) {
                dynupgrade_del_from_array_by_index( $noderef->{RESLINK}, $idx);
                return $noderef;
            }
        }
        return $noderef;
    }
    Msg::log("Error in processing command: [$cmdref->{CMD}]\n");
    return $noderef;
}

# Responsibility: To validate a tree to be effective
# Input:          Tree reference, the matrix of deleted attributes
# Output:         Updated tree
sub dynupgrade_validate_tree {
    my ($treeref, $deleted_attrs_ref) = @_;
    my ( $noderef, $nodename, $attrname, $cmdref, $num );

    # update type attributes
    foreach $nodename (keys %{$deleted_attrs_ref} ) {
        # get each type node
        $noderef= dynupgrade_get_node_from_tree( $treeref, "TYPE", $nodename);
        # browse the attributes in each type
        foreach $attrname ( @{$deleted_attrs_ref->{$nodename}} ) {
            # delete the obsolete ones
            if( defined ($noderef->{ATTR}->{$attrname} ) ) {
                delete( $noderef->{ATTR}->{$attrname} );
            }
        }
    }

    # update group link
    $noderef= $treeref->{GRPLINK}->{GRPLINK};
    if( $noderef) {
        $num= @{$noderef->{GRPLINK}};
        for( my $idx= $num-1; $idx>= 0; $idx-- ) {
            $cmdref= @{$noderef->{GRPLINK}}[$idx];
            my $name_parent= $cmdref->{GRPPARENT};
            my $name_child = $cmdref->{GRPCHILD};
            
            # not found related nodes => delete
            if( !defined( $treeref->{GRP}->{$name_parent} ) ||
                !defined( $treeref->{GRP}->{$name_child} ) ) {
                dynupgrade_del_from_array_by_index( $noderef->{GRPLINK}, $idx);
                next;
            }
        }
    }

    # update resource
    foreach $nodename (keys %{$treeref->{RES}} ) {
        # get each resource
        $noderef= dynupgrade_get_node_from_tree( $treeref, "RES", $nodename );

        $cmdref= $noderef->{MAIN};
        # get the name and group of a resource
        my $name_type= $cmdref->{RESTYPE};
        my $name_grp = $cmdref->{SERVGRP};
        # not found type|group in type|group list=> remove resource
        if( !defined( $treeref->{TYPE}->{$name_type} ) ||
            !defined( $treeref->{GRP}->{$name_grp} ) ) {
            delete( $treeref->{RES}->{$nodename} );
            next;
        }

        # get resource related type node
        my $noderef_type= dynupgrade_get_node_from_tree( $treeref, "TYPE", $name_type );

        # browse the attributes in the resource to delete the obsolete ones
        # the attribute in a resource can not be found in the deleted type attribute list => delete the attribute in the resource
        foreach $attrname ( @{$deleted_attrs_ref->{$name_type}} ) {
            # delete the obsolete attribute
            if( defined ($noderef->{ATTR}->{$attrname} ) ) {
                delete( $noderef->{ATTR}->{$attrname} );
            }
        }
    }

    # update resource link
    $noderef= dynupgrade_get_node_from_tree( $treeref, "RESLINK", "RESLINK");
        if( $noderef ) {
        $num= @{$noderef->{RESLINK}};
        for( my $idx= $num-1; $idx>= 0; $idx-- ) {
            $cmdref= @{$noderef->{RESLINK}}[$idx];
            my $name_parent= $cmdref->{RESPARENT};
            my $name_child = $cmdref->{RESCHILD};
            # not found related nodes => delete
            if( !defined( $treeref->{RES}->{$name_parent} ) ||
                !defined( $treeref->{RES}->{$name_child} ) ) {
                dynupgrade_del_from_array_by_index( $noderef->{RESLINK}, $idx);
                next;
            }
        }
    }

    $_[0]= $treeref;
}

# Responsibility: To import file into a tree
# Input:          System name; cmd file name
# Output:         Reference of result tree
sub dynupgrade_import_file2tree_sys {
    my ( $sys, $nameIn)= @_;
    my $lines_ref= dynupgrade_import_file2lines_sys( $sys, $nameIn);
    my $tree_ref= dynupgrade_import_lines2tree( $lines_ref);
    return $tree_ref;
}

# Responsibility: To import file into strings
# Input:          System name; cmd file name
# Output:         Reference of result strings 
sub dynupgrade_import_file2lines_sys {
    my ( $sys, $nameIn)= @_;

    my $content= EDR::readfile_sys($sys, $nameIn);

    # read file until the end
    my @lines= split(/\n+/, $content);
    return \@lines;
}

# Responsibility: To load a tree from file lines
# Input:          File lines; cmd reference
# Output:         Created tree
sub dynupgrade_import_lines2tree {
    my (@lines, $line, %tree);
    @lines= @{$_[0]};
    my $treeref = $_[1];
    if ($treeref) {
        %tree = %{$treeref};
    } else {
        %tree= ();
    }

    $line= undef;
    
    while( scalar(@lines) ) {
        $line= shift @lines;

        # The substitution group below is based on the regular expressions in order to get one line to be neated 
        # with items seperated by only one spaces

        # 1- to remove the redundant spaces in order to reserve only one space charracter between each couple of items
        $line=~s/\s+/ /g;
        # 2- to remove the spaces at the beginning
        $line=~s/^\s+//g;
        # 3- to remove the spaces in the end
        $line=~s/\s+$//g;
        if( !$line) {
            next;
        }

        # parse line into command
        my $cmdref= dynupgrade_parse_ha_cmd($line);
        # build command into tree
        dynupgrade_update_cmd2tree (\%tree, $cmdref);
        $line= undef;
    }
    return \%tree;
}

# Responsibility: To export a tree into a file
# Input:          System name; cmd file name; tree reference; etc.
# Output:         N/A
sub dynupgrade_export_tree2file_sys {
    my ( $sys, $nameOut, $tree_ref)= @_;
    my $result= dynupgrade_export_tree2string( $tree_ref);
    EDR::writefile_sys( $sys, $result, $nameOut );
}

# Responsibility: To serialize a tree into string
# Input:          Tree reference; sub tree branch name
# Output:         Serialized string
sub dynupgrade_export_tree2string {
    my ($treeref)= @_;
    my $result= "";
    
    # export types
    $result.= dynupgrade_export_branch2string( $treeref, "TYPE");

    # export clusters
    $result.= dynupgrade_export_branch2string( $treeref, "CLUS");

    # export systems
    $result.= dynupgrade_export_branch2string( $treeref, "SYS");

    # export groups
    $result.= dynupgrade_export_branch2string( $treeref, "GRP");

    # export resources
    $result.= dynupgrade_export_branch2string( $treeref, "RES");

    # export others
    $result.= dynupgrade_export_branch2string( $treeref, "OTHER");

    # export group/resource links
    $result.= dynupgrade_export_branch2string( $treeref, "GRPLINK");
    $result.= dynupgrade_export_branch2string( $treeref, "RESLINK");

    return $result;
}

# Responsibility: To serialize a subtree into string
# Input:          Tree reference; sub tree branch name
# Output:         Serialized string
sub dynupgrade_export_branch2string {
    my ($treeref, $branch_name)= @_;
    my $result= "";
    my $key= undef;
    my $noderef= undef;

    # export other list
    if( $branch_name eq "CLUS" ||
        $branch_name eq "SYS" ||
        $branch_name eq "OTHER" ) {
        my $cmdref= undef;
        foreach $cmdref ( @{$treeref->{$branch_name}} ) {
            $result.= dynupgrade_export_ha_cmd( $cmdref);
        }
        return $result;
    }
    # export one branch such as one of TYPE/RES/GRP/RESLINK/GRPLINK
    foreach $key (keys %{$treeref->{$branch_name}} ) {
        # export one node of a branch
        $noderef= $treeref->{$branch_name}->{$key};
        $result.= dynupgrade_export_node2string( $noderef);
    }
    return $result;
}

# Responsibility: To serialize a node into string
# Input:          Node reference
# Output:         Serialized string
sub dynupgrade_export_node2string {
    my ($noderef)= @_;
    my $result= "";
    my ( $cmdref, $attrref, $attrname );

    # main command
    $cmdref= $noderef->{MAIN};
    if( $cmdref) {
        $result.= dynupgrade_export_ha_cmd( $cmdref);
    }
    # file command
    $cmdref= $noderef->{FILE};
    if( $cmdref) {
        $result.= dynupgrade_export_ha_cmd( $cmdref);
    }
    # group link only
    foreach $cmdref ( @{$noderef->{GRPLINK}} ) {
        $result.= dynupgrade_export_ha_cmd( $cmdref);
    }
    # rsource link only
    foreach $cmdref ( @{$noderef->{RESLINK}} ) {
        $result.= dynupgrade_export_ha_cmd( $cmdref);
    }

    # export ATTR commands
    foreach $attrname (keys %{$noderef->{ATTR}} ) {
        $attrref= $noderef->{ATTR}->{$attrname};

        # add
        $cmdref= $attrref->{ADD};
        if( $cmdref) {
            $result.= dynupgrade_export_ha_cmd( $cmdref);
        }
        # local
        $cmdref= $attrref->{LOCAL};
        if( $cmdref) {
            $result.= dynupgrade_export_ha_cmd( $cmdref);
        }
        # default
        $cmdref= $attrref->{DEFAULT};
        if( $cmdref) {
            $result.= dynupgrade_export_ha_cmd( $cmdref);
        }
        # override
        $cmdref= $attrref->{OVERRIDE};
        if( $cmdref) {
            $result.= dynupgrade_export_ha_cmd( $cmdref);
        }
        # undooverride
        $cmdref= $attrref->{UNDOOVERRIDE};
        if( $cmdref) {
            $result.= dynupgrade_export_ha_cmd( $cmdref);
        }
        # modify
        $cmdref= $attrref->{MODIFY};
        if( $cmdref) {
            $result.= dynupgrade_export_ha_cmd( $cmdref);
        }
    }
    
    return $result;
}

# Responsibility: To parse the ha command into cmd object
# Input:          Ha command line name
# Output:         Cmd object reference
sub dynupgrade_parse_ha_cmd {
    my $line= $_[0];
    my %cmd = ();
    my @items= ();
    
    # split the line into items by spaces
    @items       = split( /\s+/, $line);
    $cmd{CMD}    = shift @items;
    $cmd{SUBCMD} = shift @items;
    
    if( scalar(@items) <= 0) {
        return \%cmd;
    }
    # commands for types
    if( $cmd{CMD} eq "hatype") {
        if( $cmd{SUBCMD} eq "-add" ) {
            $cmd{TYPENAME} = shift @items;
            $cmd{TREE_NODECLASS}= "MAIN";
        }
        elsif( $cmd{SUBCMD} eq "-modify" ) {
            $cmd{TYPENAME} = shift @items;
            $cmd{ATTRNAME} = shift @items;
            $cmd{ATTRVAL}  = join( ' ', @items);
            $cmd{TREE_NODECLASS}= "ATTR";
            if( $cmd{ATTRNAME} eq "SourceFile" ) {
                $cmd{TREE_NODECLASS}= "FILE";
            }
            $cmd{TREE_ATTRCLASS}= "MODIFY";
        } elsif( $cmd{SUBCMD} eq "-delete" ) {
            $cmd{TYPENAME} = shift @items;
            $cmd{TREE_NODECLASS}= "DELETE";
        } else {
            $cmd{CONTENT}= join( ' ', @items);
            $cmd{TREE_BRANCH}= "OTHER";
            return \%cmd;
        }
        $cmd{TREE_BRANCH}= "TYPE";
        $cmd{TREE_NODENAME}= $cmd{TYPENAME};
        return \%cmd;
    }
    # commands for attributes
    if( $cmd{CMD} eq "haattr") {
        $cmd{TREE_BRANCH}= "TYPE";
        $cmd{TREE_NODECLASS}= "ATTR";
        if( $cmd{SUBCMD} eq "-add") {
            if( $items[0] eq "-static" || $items[0] eq "-temp") {
                $cmd{ATTRPRETYPE} = shift @items;
            }
            $cmd{TYPENAME} = shift @items;
            $cmd{ATTRNAME} = shift @items;
            if( $items[0] eq "-string" || $items[0] eq "-integer" || $items[0] eq "-boolean" ) {
                $cmd{ATTRVALTYPE} = shift @items;
            }
            if( $items[0] eq "-scalar" || $items[0] eq "-keylist" || $items[0] eq "assoc" || $items[0] eq "vector") {
                $cmd{ATTRVALDIM} = shift @items;
            }
            if( scalar(@items) > 0) {
                $cmd{ATTRVAL} = join( ' ', @items);
            }
            $cmd{TREE_ATTRCLASS}= "ADD";
        } elsif( $cmd{SUBCMD} eq "-default") {
            $cmd{TYPENAME} = shift @items;
            $cmd{ATTRNAME} = shift @items;
            if( scalar(@items) > 0) {
                $cmd{ATTRVAL} = join( ' ', @items);
            }
            $cmd{TREE_ATTRCLASS}= "DEFAULT";
        } elsif( $cmd{SUBCMD} eq "-delete") {
            if( $items[0] eq "-static" || $items[0] eq "-temp") {
                $cmd{ATTRPRETYPE} = shift @items;
            }
            ( $cmd{TYPENAME}, $cmd{ATTRNAME} )= @items;
            $cmd{TREE_ATTRCLASS}= "DELETE";
        } else {
            $cmd{CONTENT}= join( ' ', @items);
            $cmd{TREE_BRANCH}= "OTHER";
            return \%cmd;
        }
        $cmd{TREE_NODENAME}= $cmd{TYPENAME};
        return \%cmd;
    }
    # commands for group
    if( $cmd{CMD} eq "hagrp") {
        $cmd{TREE_BRANCH}= "GRPLINK";
        $cmd{TREE_NODENAME}= "GRPLINK";
        if( $cmd{SUBCMD} eq "-link") {
            ( $cmd{GRPPARENT},$cmd{GRPCHILD},$cmd{GDCATEGORY}, $cmd{GDLOCATION}, $cmd{GDTYPE} )= @items;
            $cmd{TREE_NODECLASS}= "GRPLINK";
            return \%cmd;
        } elsif( $cmd{SUBCMD} eq "-unlink") {
            ( $cmd{GRPPARENT}, $cmd{GRPCHILD} )= @items;
            $cmd{TREE_NODECLASS}= "GRPUNLINK";
            return \%cmd;
        }
        
        $cmd{TREE_BRANCH}= "GRP";
        if( $cmd{SUBCMD} eq "-add") {
            $cmd{TYPENAME} = shift @items;
            $cmd{TREE_NODECLASS}= "MAIN";
        } elsif( $cmd{SUBCMD} eq "-modify") {
            $cmd{TYPENAME} = shift @items;
            $cmd{ATTRNAME} = shift @items;
            if( scalar(@items) > 0) {
                $cmd{ATTRVAL} = join( ' ', @items);
            }
            $cmd{TREE_NODECLASS}= "ATTR";
            if( $cmd{ATTRNAME} eq "SourceFile" ) {
                $cmd{TREE_NODECLASS}= "FILE";
            }
            $cmd{TREE_ATTRCLASS}= "MODIFY";
        } elsif( $cmd{SUBCMD} eq "-delete") {
            $cmd{TYPENAME} = shift @items;
            $cmd{TREE_NODECLASS}= "DELETE";
        } else {
            $cmd{CONTENT}= join( ' ', @items);
            $cmd{TREE_BRANCH}= "OTHER";
            return \%cmd;
        }
        $cmd{TREE_NODENAME}= $cmd{TYPENAME};
        return \%cmd;
    }
    # commands for resources
    if( $cmd{CMD} eq "hares") {
        $cmd{TREE_BRANCH}= "RESLINK";
        $cmd{TREE_NODENAME}= "RESLINK";
        if( $cmd{SUBCMD} eq "-link") {
            ( $cmd{RESPARENT}, $cmd{RESCHILD} )= @items;
            $cmd{TREE_NODECLASS}= "RESLINK";
            return \%cmd;
        }
        elsif( $cmd{SUBCMD} eq "-unlink") {
            ( $cmd{RESPARENT}, $cmd{RESCHILD} )= @items;
            $cmd{TREE_NODECLASS}= "RESUNLINK";
            return \%cmd;
        }

        $cmd{TREE_BRANCH}= "RES";
        if( $cmd{SUBCMD} eq "-add") {
            ( $cmd{RESNAME}, $cmd{RESTYPE}, $cmd{SERVGRP} )= @items;
            $cmd{TREE_NODECLASS}= "MAIN";
        } elsif( $cmd{SUBCMD} eq "-delete") {
            $cmd{RESNAME} = shift @items;
            $cmd{TREE_NODECLASS}= "DELETE";
        } elsif( $cmd{SUBCMD} eq "-modify") {
            $cmd{RESNAME} = shift @items;
            $cmd{ATTRNAME} = shift @items;
            if( scalar(@items) > 0) {
                $cmd{ATTRVAL} = join( ' ', @items);
            }
            $cmd{TREE_NODECLASS}= "ATTR";
            if( $cmd{ATTRNAME} eq "SourceFile" ) {
                $cmd{TREE_NODECLASS}= "FILE";
            }
            $cmd{TREE_ATTRCLASS}= "MODIFY";
        } elsif( $cmd{SUBCMD} eq "-override") {
            ( $cmd{RESNAME}, $cmd{ATTRNAME} )= @items;
            $cmd{TREE_NODECLASS}= "ATTR";
            $cmd{TREE_ATTRCLASS}= "OVERRIDE";
        } elsif( $cmd{SUBCMD} eq "-undooverride") {
            ( $cmd{RESNAME}, $cmd{ATTRNAME} )= @items;
            $cmd{TREE_NODECLASS}= "ATTR";
            $cmd{TREE_ATTRCLASS}= "UNDOOVERRIDE";
        } elsif( $cmd{SUBCMD} eq "-local") {
            ( $cmd{RESNAME}, $cmd{ATTRNAME} )= @items;
            $cmd{TREE_NODECLASS}= "ATTR";
            $cmd{TREE_ATTRCLASS}= "LOCAL";
        } else {
            $cmd{CONTENT}= join( ' ', @items);
            $cmd{TREE_BRANCH}= "OTHER";
            return \%cmd;
        }
        $cmd{TREE_NODENAME}= $cmd{RESNAME};
        return \%cmd;
    }
    # commands for clusters
    if( $cmd{CMD} eq "haclus") {
        $cmd{CONTENT}= join( ' ', @items);
        $cmd{TREE_BRANCH}= "CLUS";
        return \%cmd;
    }
    # commands for systems
    if( $cmd{CMD} eq "hasys") {
        $cmd{CONTENT}= join( ' ', @items);
        $cmd{TREE_BRANCH}= "SYS";
        return \%cmd;
    }

    # others
    $cmd{CONTENT}= join( ' ', @items);
    $cmd{TREE_BRANCH}= "OTHER";
    return \%cmd;
}

# Responsibility: To serialize the cmd object to a string line
# Input:          Ha command line name
# Output:         A string line of serialized ha cmd
sub dynupgrade_export_ha_cmd {
    my %cmd= %{$_[0]};
    my $line= "";

    if( !%cmd ) {
        return $line;
    }
    
    $line.= "$cmd{CMD}" . " $cmd{SUBCMD}";
    # cluster/system/other types firstly
    if( $cmd{TREE_BRANCH} eq "CLUS" ||
        $cmd{TREE_BRANCH} eq "SYS" ||
        $cmd{TREE_BRANCH} eq "OTHER" ) {
        $line.= " $cmd{CONTENT}";
        return $line."\n";
    }
    if( $cmd{CMD} eq "hatype") {
        if( $cmd{SUBCMD} eq "-add" ) {
            $line.= " $cmd{TYPENAME}";
            return $line."\n";
        }
        if( $cmd{SUBCMD} eq "-modify" ) {
            $line.= " $cmd{TYPENAME}"." $cmd{ATTRNAME}"." $cmd{ATTRVAL}";
            return $line."\n";
        }
    }
    if( $cmd{CMD} eq "haattr") {
        if( $cmd{SUBCMD} eq "-add") {
            if( defined $cmd{ATTRPRETYPE}) {
                $line.= " $cmd{ATTRPRETYPE}";
            }
            $line.= " $cmd{TYPENAME}"." $cmd{ATTRNAME}";
            if( defined $cmd{ATTRVALTYPE} ) {
                $line.= " $cmd{ATTRVALTYPE}";
            }
            if( defined $cmd{ATTRVALDIM}) {
                $line.= " $cmd{ATTRVALDIM}";
            }
            if( defined $cmd{ATTRVAL}) {
                $line.= " $cmd{ATTRVAL}";
            }
            return $line."\n";
        }
        if( $cmd{SUBCMD} eq "-default") {
            $line.= " $cmd{TYPENAME}"." $cmd{ATTRNAME}";
            if( defined $cmd{ATTRVAL}) {
                $line.= " $cmd{ATTRVAL}";
            }
            return $line."\n";
        }
    }
    if( $cmd{CMD} eq "hagrp") {
        if( $cmd{SUBCMD} eq "-add") {
            $line.= " $cmd{TYPENAME}";
            return $line."\n";
        }
        if( $cmd{SUBCMD} eq "-modify") {
            $line.= " $cmd{TYPENAME}"." $cmd{ATTRNAME}";
            if ($cmd{TREE_NODECLASS} eq "FILE") {
                return $line." $cmd{ATTRVAL}\n";
            }
            if( defined $cmd{ATTRVAL} && scalar(@{$cmd{ATTRVAL}})> 0 ) {
                my $idx;
                for( $idx=0; $idx< scalar(@{$cmd{ATTRVAL}})-1; $idx++ ) {
                    $line.= " $cmd{ATTRVAL}->[$idx]". "\n" . "$cmd{CMD}". " $cmd{SUBCMD}". " $cmd{TYPENAME}"." $cmd{ATTRNAME}";
                }
                $line.= " $cmd{ATTRVAL}->[$idx]";
            }
            return $line."\n";
        }
        # GRP_LINK
        if( $cmd{SUBCMD} eq "-link") {
            $line.= " $cmd{GRPPARENT}"." $cmd{GRPCHILD}"." $cmd{GDCATEGORY}"." $cmd{GDLOCATION}"." $cmd{GDTYPE}";
            return $line."\n";
        }
    }
    if( $cmd{CMD} eq "hares") {
        if( $cmd{SUBCMD} eq "-add") {
            $line.= " $cmd{RESNAME}"." $cmd{RESTYPE}"." $cmd{SERVGRP}";
            return $line."\n";
        }
        if( $cmd{SUBCMD} eq "-modify") {
            $line.= " $cmd{RESNAME}"." $cmd{ATTRNAME}";
            if ($cmd{TREE_NODECLASS} eq "FILE") {
                return $line." $cmd{ATTRVAL}\n";
            }
            if( defined $cmd{ATTRVAL} && scalar(@{$cmd{ATTRVAL}})> 0 ) {
                my $idx;
                for( $idx=0; $idx< scalar(@{$cmd{ATTRVAL}})-1; $idx++ ) {
                    $line.= " $cmd{ATTRVAL}->[$idx]". "\n" . "$cmd{CMD}". " $cmd{SUBCMD}". " $cmd{RESNAME}"." $cmd{ATTRNAME}";
                }
                $line.= " $cmd{ATTRVAL}->[$idx]";
            }
            return $line."\n";
        }
        if( $cmd{SUBCMD} eq "-override") {
            $line.= " $cmd{RESNAME}"." $cmd{ATTRNAME}";
            return $line."\n";
        }
        if( $cmd{SUBCMD} eq "-undooverride") {
            $line.= " $cmd{RESNAME}"." $cmd{ATTRNAME}";
            return $line."\n";
        }
        if( $cmd{SUBCMD} eq "-local") {
            $line.= " $cmd{RESNAME}"." $cmd{ATTRNAME}";
            return $line."\n";
        }
        if( $cmd{SUBCMD} eq "-link") {
            $line.= " $cmd{RESPARENT}"." $cmd{RESCHILD}";
            return $line."\n";
        }
    }
    #error 
    Msg::log("Cannot get effective command [$cmd{CMD}] and sub command [$cmd{SUBCMD}]\n");
    return "";
}

# Responsibility: To delete an item from array according to the index
# Input:          Array reference; index
# Output:         N/A
sub dynupgrade_del_from_array_by_index {
    my ($arrayref, $idx)= @_;
    # over bridge
    if( $idx< 0 || $idx > $#{$arrayref} ) {
        return;
    }
    @{$arrayref}=(@{$arrayref}[0..$idx-1], @{$arrayref}[$idx+1..$#{$arrayref}]);
    $_[0]= $arrayref;
}

# Responsibility: To compare ha command objects
# Input:          command object
# Output:         compared string result
sub dynupgrade_compare_ha_cmds {
    my %cmd1= %{$_[0]};
    my %cmd2= %{$_[1]};
    
    if( $cmd1{CMD} eq "hatype" && $cmd2{CMD} eq "hatype") {
        if( $cmd1{SUBCMD} eq "-add" && $cmd2{SUBCMD} eq "-add") {
            if( $cmd1{TYPENAME} eq $cmd2{TYPENAME}) {
                return "EQUAL_TYPE_ADD_TYPENAME";
            }
            return "EQUAL_TYPE_ADD";
        }
        if( $cmd1{SUBCMD} eq "-modify" && $cmd2{SUBCMD} eq "-modify") {
            if( $cmd1{TYPENAME} eq $cmd2{TYPENAME}) {
                if( $cmd1{ATTRNAME} eq $cmd2{ATTRNAME}) {
                    if( $cmd1{ATTRVAL} eq $cmd2{ATTRVAL}) {
                        return "EQUAL_TYPE_MODIFY_TYPENAME_ATTRNAME_ATTRVAL";
                    }
                    return "EQUAL_TYPE_MODIFY_TYPENAME_ATTRNAME";
                }
                return "EQUAL_TYPE_MODIFY_TYPENAME";
            }
            return "EQUAL_TYPE_MODIFY";
        }
        return "EQUAL_TYPE";
    }
    if( $cmd1{CMD} eq "haattr" && $cmd2{CMD} eq "haattr") {
        if( $cmd1{SUBCMD} eq "-add" && $cmd2{SUBCMD} eq "-add") {
            if( $cmd1{TYPENAME} eq $cmd2{TYPENAME}) {
                if( $cmd1{ATTRNAME} eq $cmd2{ATTRNAME}) {
                    if( $cmd1{ATTRVAL} eq $cmd2{ATTRVAL} &&
                        $cmd1{ATTRPRETYPE} eq $cmd2{ATTRPRETYPE} &&
                        $cmd1{ATTRVALTYPE} eq $cmd2{ATTRVALTYPE} &&
                        $cmd1{ATTRVALDIM} eq $cmd2{ATTRVALDIM} ) {
                        return "EQUAL_ATTR_ADD_TYPENAME_ATTRNAME_ATTRVAL";
                    }
                    return "EQUAL_ATTR_ADD_TYPENAME_ATTRNAME";
                }
                return "EQUAL_ATTR_ADD_TYPENAME";
            }
            return "EQUAL_ATTR_ADD";
        }
        if( $cmd1{SUBCMD} eq "-default" && $cmd2{SUBCMD} eq "-default") {
            if( $cmd1{TYPENAME} eq $cmd2{TYPENAME}) {
                if( $cmd1{ATTRNAME} eq $cmd2{ATTRNAME}) {
                    if( $cmd1{ATTRVAL} eq $cmd2{ATTRVAL} ) {
                        return "EQUAL_ATTR_DEFAULT_TYPENAME_ATTRNAME_ATTRVAL";
                    }
                    return "EQUAL_ATTR_DEFAULT_TYPENAME_ATTRNAME";
                }
                return "EQUAL_ATTR_DEFAULT_TYPENAME";
            }
            return "EQUAL_ATTR_DEFAULT";
        }
        return "EQUAL_ATTR";
    }
    if( $cmd1{CMD} eq "hagrp" && $cmd2{CMD} eq "hagrp") {
        if( $cmd1{SUBCMD} eq "-add" && $cmd2{SUBCMD} eq "-add") {
            if( $cmd1{TYPENAME} eq $cmd2{TYPENAME}) {
                return "EQUAL_GRP_ADD_TYPENAME";
            }
            return "EQUAL_GRP_ADD";
        }
        if( $cmd1{SUBCMD} eq "-modify" && $cmd2{SUBCMD} eq "-modify") {
            if( $cmd1{TYPENAME} eq $cmd2{TYPENAME}) {
                if( $cmd1{ATTRNAME} eq $cmd2{ATTRNAME}) {
                    if( $cmd1{ATTRVAL} eq $cmd2{ATTRVAL}) {
                        return "EQUAL_GRP_MODIFY_TYPENAME_ATTRNAME_ATTRVAL";
                    }
                    return "EQUAL_GRP_MODIFY_TYPENAME_ATTRNAME";
                }
                return "EQUAL_GRP_MODIFY_TYPENAME";
            }
            return "EQUAL_GRP_MODIFY";
        }
        if( $cmd1{SUBCMD} eq "-link" && $cmd2{SUBCMD} eq "-link") {
            if( $cmd1{GRPPARENT} eq $cmd2{GRPPARENT}) {
                if( $cmd1{GRPCHILD} eq $cmd2{GRPCHILD}) {
                    return "EQUAL_GRP_LINK_GRPPARENT_GRPCHILD";
                }
                return "EQUAL_GRP_LINK_GRPPARENT";
            }
            return "EQUAL_GRP_LINK";
        }
        return "EQUAL_GRP";
    }
    if( $cmd1{CMD} eq "hares" && $cmd2{CMD} eq "hares") {
        if( $cmd1{SUBCMD} eq "-add" && $cmd2{SUBCMD} eq "-add") {
            if( $cmd1{RESNAME} eq $cmd2{RESNAME}) {
                if( $cmd1{RESTYPE} eq $cmd2{RESTYPE}) {
                    if( $cmd1{SERVGRP} eq $cmd2{SERVGRP}) {
                        return "EQUAL_RES_ADD_RESNAME_RESTYPE_SERVGRP";
                    }
                    return "EQUAL_RES_ADD_RESNAME_RESTYPE";
                }
                return "EQUAL_RES_ADD_RESNAME";
            }
            return "EQUAL_RES_ADD";
        }
        if( $cmd1{SUBCMD} eq "-modify" && $cmd2{SUBCMD} eq "-modify") {
            if( $cmd1{RESNAME} eq $cmd2{RESNAME}) {
                if( $cmd1{ATTRNAME} eq $cmd2{ATTRNAME}) {
                    if( $cmd1{ATTRVAL} eq $cmd2{ATTRVAL}) {
                        return "EQUAL_RES_MODIFY_RESNAME_ATTRNAME_ATTRVAL";
                    }
                    return "EQUAL_RES_MODIFY_RESNAME_ATTRNAME";
                }
                return "EQUAL_RES_MODIFY_RESNAME";
            }
            return "EQUAL_RES_MODIFY";
        }
        if( $cmd1{SUBCMD} eq "-override" && $cmd2{SUBCMD} eq "-override") {
            if( $cmd1{RESNAME} eq $cmd2{RESNAME}) {
                if( $cmd1{ATTRNAME} eq $cmd2{ATTRNAME}) {
                    return "EQUAL_RES_OVERRIDE_RESNAME_ATTRNAME";
                }
                return "EQUAL_RES_OVERRIDE_RESNAME";
            }
            return "EQUAL_RES_OVERRIDE";
        }
        if( $cmd1{SUBCMD} eq "-undooverride" && $cmd2{SUBCMD} eq "-undooverride") {
            if( $cmd1{RESNAME} eq $cmd2{RESNAME}) {
                if( $cmd1{ATTRNAME} eq $cmd2{ATTRNAME}) {
                    return "EQUAL_RES_UNDOOVERRIDE_RESNAME_ATTRNAME";
                }
                return "EQUAL_RES_UNDOOVERRIDE_RESNAME";
            }
            return "EQUAL_RES_UNDOOVERRIDE";
        }
        if( $cmd1{SUBCMD} eq "-local" && $cmd2{SUBCMD} eq "-local") {
            if( $cmd1{RESNAME} eq $cmd2{RESNAME}) {
                if( $cmd1{ATTRNAME} eq $cmd2{ATTRNAME}) {
                    return "EQUAL_RES_LOCAL_RESNAME_ATTRNAME";
                }
                return "EQUAL_RES_LOCAL_RESNAME";
            }
            return "EQUAL_RES_LOCAL";
        }
        if( $cmd1{SUBCMD} eq "-link" && $cmd2{SUBCMD} eq "-link") {
            if( $cmd1{RESPARENT} eq $cmd2{RESPARENT}) {
                if( $cmd1{RESCHILD} eq $cmd2{RESCHILD}) {
                    return "EQUAL_RES_LINK_RESPARENT_RESCHILD";
                }
                return "EQUAL_RES_LINK_RESPARENT";
            }
            return "EQUAL_RES_LINK";
        }
        return "EQUAL_RES";
    }
    return "NONE";
}

#
# End: perform vcs upgrade
#

### START OF VXSS SUBROUTINES

# break out here with the -security option, not running normal CPI flow
sub cli_prod_option {
    my ($prod)=@_;
    $prod->tsub("security_menu") if (Cfg::opt("security"));
    $prod->tsub("vxfen_option") if (Cfg::opt("fencing"));
    $prod->tsub("opt_addnode") if (Cfg::opt("addnode"));
    return "";
}

sub web_prod_option {
    my ($prod) = @_;        
    $prod->tsub("web_opt_addnode") if (Cfg::opt("addnode"));
    $prod->tsub("vxfen_option") if (Cfg::opt("fencing"));
    return "";
}

sub config_vxss {
    my ($prod)=@_;
    my ($atpkg,$ayn,$done,$cpic,$edr,$msg,$rtn);

    # lighten/remove platform block as VCS+VxSS versions are released
    $cpic=Obj::cpic();
    $edr=Obj::edr();
    while (!$done) {
        Msg::title();
        $msg=Msg::new("$prod->{name} can be configured to utilize Symantec Security Services\n", 40, 2454, "$prod->{name}");
        $msg->print;
        $msg=Msg::new("Running $prod->{abbr} in Secure Mode guarantees that all inter-system communication is encrypted, and users are verified with security credentials.\n", 40, 2455, "$prod->{abbr}");
        $msg->print;
        $msg=Msg::new("When running $prod->{abbr} in Secure Mode, NIS and system usernames and passwords are used to verify identity. $prod->{abbr} usernames and passwords are no longer utilized when a cluster is running in Secure Mode.\n", 40, 2456, "$prod->{abbr}", "$prod->{abbr}");
        $msg->print;
        $msg=Msg::new("Before configuring a cluster to operate using Symantec Security Services, another system must already have Symantec Security Services installed and be operating as a Root Broker. Refer to the $prod->{name} Installation Guide for more information on configuring a Symantec Product Authentication Service Root Broker.\n", 40, 2457, "$prod->{name}");
        $msg->print;
        return "" if Cfg::opt("responsefile");
    
        $msg=Msg::new("Would you like to configure $prod->{abbr} to use Symantec Security Services?", 40, 2458, "$prod->{abbr}");
        $ayn=$msg->aynn;
        return if ($ayn eq "N");
        $rtn=$prod->security_configuration_options();
        $done=1 unless ($rtn eq $edr->{msg}{back});
    }    
}

sub web_config_vxss {
    my $prod = shift;
    my $web = Obj::web();
    my ($answer,$cfg,$description,$msg,$option,$rtn);
    
    $cfg = Obj::cfg();
    
    $msg=Msg::new("$prod->{name} can be configured to utilize Symantec Security Services\n", 40, 2454, "$prod->{name}");
    $description = $msg->{msg};
    $msg=Msg::new("Running $prod->{abbr} in Secure Mode guarantees that all inter-system communication is encrypted, and users are verified with security credentials.\n", 40, 2455, "$prod->{abbr}");
    $description .= $msg->{msg};
    $msg=Msg::new("When running $prod->{abbr} in Secure Mode, NIS and system usernames and passwords are used to verify identity. $prod->{abbr} usernames and passwords are no longer utilized when a cluster is running in Secure Mode.\n", 40, 2456, "$prod->{abbr}", "$prod->{abbr}");
    $description .= $msg->{msg};
    $msg=Msg::new("Would you like to configure $prod->{abbr} to use Symantec Security Services?", 40, 2458, "$prod->{abbr}");
    $answer = $msg->aynn_web("","",$description);
    return "" if ($answer eq "N");
    $web->{vxss_enabled} = 1;
    $rtn = $web->tsub("web_script_form", "vcs_security", "init");
    while ($rtn eq "chkRB") {
        if ($prod->web_check_rootbroker()) {
            $msg=Msg::new("Systems will use $cfg->{at_rootdomain} as its Symantec Product Authentication Service Domain", 40, 3543, "$cfg->{at_rootdomain}");
            $web->tsub("web_script_form","alert",$msg->{msg});
            last;
        } else {
            $rtn = $web->tsub("web_script_form", "vcs_security", "chkRB");
        }
    }
    if ($rtn eq "cfgRAB") {
        $prod->{rootbroker}=$cfg->{rootbroker};
        $option = 1;
    }
    while ($rtn eq "getRB") {
        if ($prod->web_get_rootbroker()) {
            $option = 1;
            last;
        } else {
            $rtn = $web->tsub("web_script_form", "vcs_security", "chkRB");
        }
    }
    
    return "back" if($rtn eq "back");
    
    # todo: set up the right option: 1 2 3
    $cfg->{vcs_securitymenuopt} = $prod->{vxssmenuopt} = $option;
}

sub get_vxss_cluster {
    my ($prod)=@_;
    my ($ccc,$cpic,$edr,$msg,$rcs,$sys);
    
    $cpic=Obj::cpic();
    $edr=Obj::edr();
    $sys=${$cpic->{systems}}[0]||$edr->{localsys} ;
    $rcs=$prod->tsub("check_vxssconfig_sys",$sys);
    while (!$rcs) {
        if ($prod->{disablevxss}) {
            $msg=Msg::new("Enter the name of one system in the $prod->{abbr} Cluster that you would like to disable Symantec Security Services:", 40, 2459, "$prod->{abbr}");
            $sys=$msg->ask;
        } else {
            $msg=Msg::new("Enter the name of one system in the $prod->{abbr} Cluster that you would like to enable Symantec Security Services:", 40, 2460, "$prod->{abbr}");
            $sys=$msg->ask;
        }
        # chop extra systems, in case they enter more
        $sys=EDR::despace($sys);
        $sys=~s/\s.*$//;
        
        #initialize the input systems now
        $sys=($Obj::pool{"Sys::$sys"}) ? Obj::sys($sys)
                                       : EDR::tsub("Sys","new",$sys); 
        Msg::n();
        if (!$edr->tsub("transport_sys",$sys)) {
            $msg=Msg::new("Cannot communicate with system $sys->{sys}\n", 40, 2461, "$sys->{sys}");
            $msg->print;
        } else {
            $rcs=$prod->tsub("check_vxssconfig_sys",$sys);
        }
    }
    return $rcs;
}

# check to see if valid configuration files exist on the local system
# if so, return a reference to an array holding ("version", "systems")
sub check_vcs_vxssconfig_sys {
    my ($prod,$sys)=@_;;
    my ($cpic,$cf,$clnm,$dsys,@ga,$iv,$lltn,$llts,$mcfg,$mcfs,$msg,@sa,$vcsmainpkg);

    #check for core config files existence on sys.
    $cpic=Obj::cpic();
    if ($#{$cpic->{systems}} == -1) {
        for $cf(@{$prod->{configfiles}}) { 
            return unless (EDR::file_sys($sys, $cf));
        }
    }
    
    # all files are there, get cluster info 
    # rsh grep chokes on the ^ but cat doesn't?
    $clnm=EDR::cmd_sys($sys, "_cmd_grep '^cluster' $prod->{maincf} | _cmd_awk '{print \$2}'");
    if ($#{$cpic->{systems}} == -1) {
        $lltn=EDR::cmd_sys($sys, "_cmd_grep set-cluster $prod->{llttab} | _cmd_awk '{print \$2}'") if (EDR::file_sys($sys, $prod->{llttab}));
        $llts=EDR::cmd_sys($sys, "_cmd_cat $prod->{llthosts} | _cmd_awk '{print \$2}' | _cmd_sort") if (EDR::file_sys($sys, $prod->{llthosts}));
    }
    $mcfs=EDR::cmd_sys($sys, "_cmd_grep '^system' $prod->{maincf} | _cmd_awk '{print \$2}' | _cmd_sort");
    return unless ($clnm && $mcfs);
    
    $vcsmainpkg=Obj::pkg($prod->{mainpkg},$prod->{padv});
    $iv=$vcsmainpkg->version_sys($sys);    
    
    
    if ($#{$cpic->{systems}} == -1 && $lltn && $llts) {
        return if (($lltn < 0) || ($lltn > 65535));
    }
    return if ((Cfg::opt(qw(install installonly))) && ($iv eq $prod->{vers}));

    @sa=split(/\s+/,$mcfs);
    $dsys=join(" ",@sa);
    # $mcfs=~ s/\s+/ /g;
    $mcfg=EDR::cmd_sys($sys, "_cmd_grep '^group' $prod->{maincf} | _cmd_awk '{print \$2}'");
    # @ga=split(/\s+/,$mcfg);
    # $gl=join(" ",@ga);
    $mcfg =~ s/\s+/ /g;
    $msg=Msg::new("$prod->{abbr} configuration files exist on this system with the following information:\n", 40, 2462, "$prod->{abbr}");
    $msg=Msg::new("\tCluster Name: $clnm", 40, 2388, "$clnm");
    $msg->print;
    if ($lltn) {
        $msg=Msg::new("\tCluster ID Number: $lltn", 40, 2385, "$lltn");
        $msg->print;
    }
    $msg=Msg::new("\tSystems: $dsys", 40, 2463, "$dsys");
    $msg->print;
    $msg=Msg::new("\tService Groups: $mcfg\n", 40, 2464, "$mcfg");
    $msg->print;
    if (Cfg::opt(qw(install installonly))) {
        $msg=Msg::new("No checks have been made to ensure the validity of the configuration\n", 40, 2465);
        $msg->print;
    }
    unshift(@sa,$iv);
    return \@sa;
}

sub is_secure_cluster{
    my ($prod,$sys)=@_;
    my $mode=EDR::cmd_sys($sys, "$prod->{bindir}/haclus -value SecureClus 2>/dev/null");
    return $mode;
}

sub check_vxssconfig_sys {
    my ($prod,$sys)=@_;
    my($ayn,$ccc,$edr,$msg,$nr,$rcs,$sc,$sysi,$sysname,@sysc,$cfg);
    
    Msg::n();
    $edr=Obj::edr();
    $cfg=Obj::cfg();
    $rcs= $prod->tsub("check_vcs_vxssconfig_sys",$sys);
    if ($#$rcs<0) {
        $msg=Msg::new("$prod->{abbr} is not configured on $sys->{sys}\n", 40, 2466, "$prod->{abbr}", "$sys->{sys}");
        $msg->print;
    } elsif (EDR::compvers($$rcs[0],$prod->{minvcssecvers})==2) {
        $msg=Msg::new("$prod->{abbr} version $$rcs[0] is installed on this system.\nVersion $prod->{minvcssecvers} or higher is required to enable Symantec Security Services with $prod->{abbr}.\n", 40, 2467, "$prod->{abbr}", "$$rcs[0]", "$prod->{minvcssecvers}", "$prod->{abbr}");
        $msg->print;
    } else {
        $ayn="";
        $sc=EDR::cmd_sys($sys, "$prod->{bindir}/haclus -value SecureClus");
        $nr=EDR::cmd_sys($sys, "$prod->{bindir}/hasys -state | _cmd_grep RUNNING | _cmd_wc -l");
        $nr=~s/\s+//;
        if (EDR::isverror($sc)) {
            $msg=Msg::new("$prod->{abbr} is not running on system $sys->{sys}\n", 40, 2468, "$prod->{abbr}", "$sys->{sys}");
            $msg->print;
        } elsif ($nr<$#$rcs) { 
            $msg=Msg::new("$prod->{abbr} is not running on all systems in this cluster. All $prod->{abbr} systems must be in RUNNING state to enable or disable Symantec Security Services.", 40, 2469, "$prod->{abbr}", "$prod->{abbr}");
            $msg->print;
        } elsif (($prod->{disablevxss}) && (!$sc)) {
            $msg=Msg::new("The $prod->{abbr} cluster including $sys->{sys} is not running in Secure Mode\n", 40, 2470, "$prod->{abbr}", "$sys->{sys}");
            $msg->print;
        } elsif ((!$prod->{disablevxss}) && ($sc)) {
            $msg=Msg::new("The $prod->{abbr} cluster including $sys->{sys} is already running in Secure Mode\n", 40, 2471, "$prod->{abbr}", "$sys->{sys}");
            $msg->print;
        } elsif ($prod->{disablevxss}) {
            $msg=Msg::new("$prod->{abbr} must be restarted briefly to disable Symantec Security Services.", 40, 2472, "$prod->{abbr}");
            $msg->print;
            $msg=Msg::new("$prod->{abbr} service groups will not be affected\n", 40, 2473, "$prod->{abbr}");
            $msg->print;
            $msg=Msg::new("Would you like to disable Symantec Security Services on this cluster?", 40, 2474);
            $ayn=$msg->ayny;
        } else {
            $msg=Msg::new("$prod->{abbr} must be restarted briefly to enable Symantec Security Services.", 40, 2475, "$prod->{abbr}");
            $msg->print;
            $msg=Msg::new("$prod->{abbr} service groups will not be affected.\n", 40, 2476, "$prod->{abbr}");
            $msg->print;
            $msg=Msg::new("Would you like to enable Symantec Security Services on this cluster?", 40, 2477);
            if($cfg->{fencing_cpc_security}){
                $ayn="Y";
            } else {
                $ayn=$msg->ayny;
            }
        }
        
        if ($ayn eq "Y") {
            # check communication with the other systems
            # $rcs is a reference to an array holding ("version", "systems")
            # shift the version out
            shift(@$rcs);
            for $sysname (@$rcs) {
                $sysi = $sysname;
                $sysi = ($Obj::pool{"Sys::$sysi"}) ? Obj::sys($sysi)
                                                   : EDR::tsub("Sys","new",$sysi); 
                next if ($sysi eq $sys);
                if (!$edr->tsub("transport_sys",$sysi)) {
                    $msg=Msg::new("Cannot communicate with system $sysi->{sys}\n", 40, 2461, "$sysi->{sys}");
                    $msg->print;
                    return "";
                }
            }
            return $rcs;
        }
    }
    return "";
}

sub randpwd {
    my ($prod,$len)=@_;
    my ($n,$pwd,$r);
    
    for $n(1..$len) {
        $r=int(rand()*62);
        $pwd .= ($r<10) ? $r : ($r<36) ? chr($r+55) : chr($r+61);
    }
    return $pwd;
}

sub stop_vcs {
    my ($prod)=@_;
    my ($cpic,$ck,$iter,$maxiters,$msg,$pids,$stop,$sys);

    
    $prod->tsub("haconf_makerw");
    $prod->tsub("freeze_groups");

    $cpic=Obj::cpic();
    $sys=${$cpic->{systems}}[0];
    $msg=Msg::new("Stopping $prod->{abbr}", 40, 2478, "$prod->{abbr}");
    $msg->left;
    $prod->tsub("haconf_dumpmakero");
    $stop = EDR::cmd_sys($sys, "$prod->{bindir}/hastop -all -force");
    # if there was any error in performing 'hastop -all -force',
    # the error number will be caught in $stop (something like: 'VCS ERROR V-16-1-10026')
    if (EDR::isverror($stop)) {
        for $sys(@{$cpic->{systems}}) {
            $pids=EDR::pids_sys($sys, "bin/had");
            EDR::kill_pids_sys($sys,@$pids);
            $pids=EDR::pids_sys($sys, "hashadow");
            EDR::kill_pids_sys($sys,@$pids);
        }
    }
    # CSG system takes a while to shut down 
    # must verify before restart
    if ( $#{$cpic->{systems}} ) {
        for $sys(@{$cpic->{systems}}) {
            $ck=EDR::cmd_sys($sys, "_cmd_gabconfig -a | _cmd_grep Port.h");
            while ($ck) {
                sleep(1);
                $ck=EDR::cmd_sys($sys, "_cmd_gabconfig -a | _cmd_grep Port.h");
            }
        }
    } else {
        $sys = ${$cpic->{systems}}[0];
        $maxiters = 15;
        for ($iter = 0; $iter < $maxiters; $iter++) {
            $pids=EDR::pids_sys($sys, "hashadow");
            $pids||=EDR::pids_sys($sys, "bin/had");
            last unless ($#$pids>=0);
            sleep(1);
        }
        if ($iter >= $maxiters) {
            # VCS did not stop after $maxiters seconds, even after issuing 'hastop -all -force'
            # kill vcs processes
            $pids=EDR::pids_sys($sys, "bin/had");
            EDR::kill_pids_sys($sys,@$pids) if (@$pids);
            $pids=EDR::pids_sys($sys, "hashadow");
            EDR::kill_pids_sys($sys,@$pids) if (@$pids);
        }
    }
    $msg=Msg::new("Done", 40, 1116);
    $msg->right;
}

sub secure_startup {
    my ($prod)=@_;
    my ($cfg,$cpic,$edr,$msg,$nr,$pids,$proc,$rc,$sys,$vcspkg);

    $cpic=Obj::cpic();
    $edr=Obj::edr();
    for $sys(@{$cpic->{systems}}) {
        EDR::cmd_sys($sys,"/opt/VRTSvcs/bin/CmdServer -stop");
    }
    
    for $sys(@{$cpic->{systems}}) {
        if ($prod->{disablevxss}) {
            $msg=Msg::new("Starting $prod->{abbr} on $sys->{sys}", 40, 1259, "$prod->{abbr}", "$sys->{sys}");
        } else {
            $msg=Msg::new("Starting $prod->{abbr} on $sys->{sys} in Secure Mode", 40, 2479, "$prod->{abbr}", "$sys->{sys}");
        }
        $msg->left;
        EDR::copy_sys($sys, "$edr->{tmpdir}/main.cf", $prod->{maincf});
        sleep(1);
        $proc=Obj::proc("had51",$prod->{padv});
        $proc->start_sys($sys);
        sleep(1);
        $msg=Msg::new("Done", 40, 1116);
        $msg->right;
    }
    
    #Start CmdServer after had/hashadow is started
    for $sys(@{$cpic->{systems}}) {
        EDR::cmd_sys($sys,"/opt/VRTSvcs/bin/CmdServer");
    }
    
    $sys=${$cpic->{systems}}[0];
    $msg=Msg::new("Confirming $prod->{abbr} startup", 40, 2480, "$prod->{abbr}");
    $msg->left;
    while (($rc<=10) && ($nr<=$#{$cpic->{systems}})) {
        sleep 6;
        $nr=EDR::cmd_sys($sys, "$prod->{bindir}/hasys -state | _cmd_grep RUNNING | _cmd_wc -l");
        $rc++;
    }
    # so all systems have same main.cf
    $prod->tsub("haconf_makerw");
    sleep 1;
    $prod->tsub("haconf_dumpmakero");
    if ($nr==1) {
        $msg=Msg::new("$nr system RUNNING", 40, 2481, "$nr");
    } else {
        $msg=Msg::new("$nr systems RUNNING", 40, 2482, "$nr");
    }
    $msg->right;
    $prod->tsub("haconf_makerw");
    $prod->tsub("unfreeze_groups");
    $prod->tsub("haconf_dumpmakero");
    Msg::n();
    return $nr;
}

# double check that all things VCS requires are in place:
# VRTSat is installed and in default /opt/VRTSat path
# mode is 1 or 3, return root credential
sub check_vxss_sys {
    my ($prod,$sys)=@_;
    my ($at,$atpkg,$cred,$grep,$j,$mode,$msg,$path,$ver);

    #ensure AT is installed on sys.
    $atpkg=Obj::pkg("VRTSat50",$prod->{padv});
    $ver=$atpkg->version_sys($sys,1); # add force_flag 1 to check AT version forcely
    return "" if (!$ver);
    
    $at=Obj::prod("AT50",$prod->{padv});
    $at->check_vxssinstallpath_sys($sys);
    $mode=$at->check_vxssmode_sys($sys);
    #mode 1=AB, 2=Root, 3=R+AB
    if ($mode==2) {
        $msg=Msg::new("Symantec Product Authentication Service is running in Root mode on system $sys->{sys}\n$prod->{abbr} systems require that Symantec Product Authentication Service be configured in R+AB or AB mode", 40, 2483, "$sys->{sys}", "$prod->{abbr}");
        $msg->die;
    }
    
    $cred=$at->check_root_credential_sys($sys,$mode);
    return $cred;
}

# 3 generic tasks
# set $prod->{rootbroker} from $cfg->{at_rootdomain}
# set $prod->{rootfqdn}
# copy roothash file to $edr->{tmpdir}
sub get_root_hash {
    my ($prod)=@_;
    my ($at,$cfg,$edr,@d,$f,$j,$mode,$msg,$pids,$sys,@v,$vers,$installdir);
    
    $edr=Obj::edr();
    $cfg=Obj::cfg();
    return "" if ($cfg->{vcs_roothashpath});
    
    #Get root broker here from response file configure                                                                
    $sys=Obj::sys($prod->{rootbroker});
    $at=Obj::prod("AT50",$prod->{padv});
    
    if (Cfg::opt("responsefile")) { 
        $pids=EDR::pids_sys($sys, "vxatd");
        if ($#$pids == -1) {
            $msg=Msg::new("\nvxatd process is not running on $sys->{sys}\n", 40, 1146, "$sys->{sys}");
            $msg->die;
        } else {
            Msg::log("vxatd process is running on $sys->{sys}");
        }
               
        $at->tsub("check_vxssinstallpath_sys",$sys);
        
        #check for RB version
        $installdir=$at->tsub("get_atinstalldir_sys",$sys, 1);
        $vers=EDR::cmd_sys($sys, "$installdir/bin/vssat showversion");
        # doing it this way in case their funky header ever changes
        # version better stay last
        @v=split(/\s+/,$vers);
        Msg::log("Checking vxatd version.....$v[$#v]");
        $mode=$at->tsub("check_vxssmode_sys",$sys);
        if ($mode==1) {
            $msg=Msg::new("\nThe vxatd process running on $sys->{sys} is not running in Root or R+AB mode", 40, 2486, "$sys->{sys}");
            $msg->die;
        } else {
            Msg::log("Response file checks completed for AT_ROOTDOMAIN=$cfg->{at_rootdomain}. Root broker node is $sys->{sys}");
        }
    } 
    $prod->{rootfqdn}||=$prod->vxss_get_fqdn($sys);
    EDR::copy_sys($sys, $at->{roothash}{all},$edr->{tmpdir}, 1) if (EDR::file_sys($sys,$at->{roothash}{all}));
}

#get the broker full qualified domain name (FQDN) "cdc.veritas.com "
#from "Domain Name broker@redhat92154.cdc.veritas.com"
#input : $prod, $sys
#output : none
sub vxss_get_fqdn {
    my ($prod,$sys)=@_;
    my ($at,$installdir,$output,@lines,$line,@d,$rootfqdn);
    
    $at=Obj::prod("AT50",$prod->{padv});
    $installdir=$at->tsub("get_atinstalldir_sys",$sys, 1);
    
    $output=EDR::cmd_sys($sys, "$installdir/bin/vssat listpd --pdrtype ab | _cmd_grep Domain.Name | _cmd_grep broker");
    @lines=split(/\n/,$output);
    for $line (@lines) {
        @d=split(/\./,$line,2);
        if ($d[1]) {
            $rootfqdn=".$d[1]";
            last;
        }
    }
    return $rootfqdn;
}

#initialize root broker obj and check the communication with it.
#input parameter: $rbname -- The root broker's name
#return value: root broker reference
#Notice: this routine should not be called in multi-thread fashion.
sub initial_rootbroker{
    my ($prod,$rbname)=@_;
    my ($edr,$msg,$rb);
    $edr=Obj::edr();
    $rb=$Obj::pool{"Sys::$rbname"} || EDR::tsub("Sys","new",$rbname);
    EDR::set_value_sys($rb, "stop_checks", 0);
    if (!$edr->transport_sys($rb)) {
        $msg=Msg::new("\nCannot communicate with system $rb->{sys}\n", 40, 2488, "$rb->{sys}");
        $msg->print;
        return "";
    }
    return $rb;
}

#check if a sys is inside a cluster
# input : $sys name or ip
# return $sys name if inside, or 0 not.
sub is_rab_inner_cluster{
    my ($prod,$rab)= @_;
    my ($cpic,$node,$msg,$syslist);
    $cpic=Obj::cpic();
    #check if rab is inner cluster node
    for $node (@{$cpic->{systems}}) {
        $syslist.="$node->{hostname} ";
        if ($rab eq $node->{sys}) {
            Msg::log("$rab is an inner cluster node");
            return $node->{sys};
         } elsif ($rab eq $node->{ip}) {
            Msg::log("$rab is IP address of an inner cluster node");
            return $node->{sys};
         }
    }
    $msg=Msg::new("$rab is not an inner cluster node. Enter a node name from the list($syslist).", 40, 2490, "$rab", "$syslist");
    $msg->print;
    return 0;
}

sub get_inner_rab {
    my ($prod)=@_;
    my ($backopt,$edr,$msg,$rab,@sysi,$rabname);
    $edr=Obj::edr();
    $backopt=1;
    
    while (1) {
        $msg=Msg::new("Enter the node name which you want to configure as RAB:", 40, 2491);
        $rab = $msg->ask("","",$backopt);
        return $rab if ($rab eq $edr->{msg}{back});
        @sysi = split(/\s+/, $rab);     
        if ($#sysi == 0) {
            if ( $edr->validate_systemnames($rab) eq "" ) {
                $rabname=$prod->is_rab_inner_cluster($rab);
                return $rabname if ($rabname);
            }
        } else {
           $msg = Msg::new("Do not enter multiple systems", 40, 1143); 
           $msg->print;
        } 
    }
}

#input : $prod
#output : cluster rb name
sub get_cluster_brokername {
    my ($prod)=@_;
    my ($at,$cfg,$cpic,$j,$msg,$name,$shortname,$sys);
    
    $cpic=Obj::cpic();
    $cfg=Obj::cfg();
    $at=Obj::prod("AT50",$prod->{padv});
    for $sys (@{$cpic->{systems}}) {
        next if (EDR::inarr($sys->{sys},@{$cfg->{newnodes}}) ||EDR::inarr($sys->{hostname},@{$cfg->{newnodes}}) ); 
        $name=$at->get_brokername_sys($sys);
        if ($name) {
            if (EDR::isip($name)) {
                $shortname=$name;
            } else {
                ($shortname,$j)=split(/\./,$name,2);
            }
            if (($cfg->{rootbroker}) && ($shortname ne $cfg->{rootbroker})) {
                $msg=Msg::new("$prod->{abbr} systems are operating Symantec Product Authentication Service with different root brokers\nEach system within a $prod->{abbr} cluster must utilize Symantec Product Authentication Service using the same root broker in order to operate in Secure Mode. Check the nodes accordingly.", 40, 2492, "$prod->{abbr}", "$prod->{abbr}");
                $msg->print;
            }
            $cfg->{rootbroker}=$shortname;
        }
    }
    return $cfg->{rootbroker};
}

sub check_rootbroker {
    my ($prod)=@_;
    my($asked,$at,$atsuffix,$ayn,$backopt,$cfg,$cpic,$cred,$done,$edr,$hash,$j,$msg,$pdfr,$rab,$rb,$rootdomain,$sys,$sys0,$rtn);

    $cpic=Obj::cpic();
    $cfg=Obj::cfg();
    $edr=Obj::edr();
    $at=Obj::prod("AT50",$prod->{padv});
    $backopt=1;
    
    for $sys (@{$cpic->{systems}}) {
        $cred=$prod->check_vxss_sys($sys);
        if ($cred) {
            if (($cfg->{at_rootdomain}) && ($cred ne $cfg->{at_rootdomain})) {
                $msg=Msg::new("$prod->{abbr} systems are operating Symantec Product Authentication Service with different root brokers\nEach system within a $prod->{abbr} cluster must utilize Symantec Product Authentication Service using the same root broker in order to operate in Secure Mode.", 40, 2493, "$prod->{abbr}", "$prod->{abbr}");
                $msg->print;
            }
            $cfg->{at_rootdomain}=$cred;
            $prod->{vxsscred}{$sys->{sys}}=1;
        } else {
            $prod->{novxsscred}++;
        }
    }
    
    if (!$cfg->{at_rootdomain}) {
        $rootdomain=$at->get_rootbroker;
        return $rootdomain if ($rootdomain eq $edr->{msg}{back});
        if ($rootdomain=~/root@/) {
            $cfg->{at_rootdomain}=$rootdomain;
            $asked=1;
        } else {
            $sys0=${$cpic->{systems}}[0];
            $atsuffix = "Server" if ($sys0->{plat} eq "Linux");
            $msg=Msg::new("\nThere is no security configuration information anywhere on cluster nodes. Proceed to configure <$sys0->{sys}> as RAB (Root Broker+Authentication Broker) and other nodes as AB (Authentication Broker). If it is ok, press Y. If you want to configure another cluster node as RAB, enter N.\n", 40, 2494, "$sys0->{sys}");
            $msg->print;
            $msg=Msg::new("Do you want to configure <$sys0->{sys}> as RAB, and other nodes as AB?", 40, 2495, "$sys0->{sys}");
            $ayn=$msg->ayny("",$backopt);
            return $ayn if ($ayn eq $edr->{msg}{back});
            if ( $ayn eq "Y" ) {
                $cfg->{rootbroker}=$prod->{rootbroker}=$sys0->{sys};            
            } else {
                $rab=$prod->get_inner_rab;
                return $rab if ($rab eq $edr->{msg}{back});
                $cfg->{rootbroker}=$prod->{rootbroker}=$rab;                               
            }
        }
    } else {
        $cfg->{rootbroker}=$prod->{rootbroker}=$prod->get_cluster_brokername();
        #initialize root broker here
        $rb=$prod->initial_rootbroker($prod->{rootbroker});
        if (!$rb) {
            $msg=Msg::new("Failed to initialize original RB $prod->{rootbroker}.", 40, 3544, "$prod->{rootbroker}");
            $msg->print;
            $rtn=$prod->reconfig_secure_cluster();
            if ($rtn) {
                undef ($cfg->{at_rootdomain});
                undef ($cfg->{rootbroker});
                undef ($prod->{novxsscred});
                undef ($prod->{rootbroker});
                undef ($prod->{vxsscred});
                return $edr->{msg}{back}; 
            } else {
                $msg=Msg::new("You have chosen not to reconfigure the cluster with another RB. Check ssh/rsh communication with original RB $prod->{rootbroker}, then run installer to configure again.", 40, 3545, "$prod->{rootbroker}");
                $msg->die;
            }
        }
    }
    
    # For RAB node inside cluster, get_root_hash should be called after RAB node is up
    # It should be called in the sub create_vxsscredentials  
    #$prod->tsub("get_root_hash");
        
    if ($cfg->{at_rootdomain}) {
        if ($asked) {
            $msg=Msg::new("\nSystems will use $cfg->{at_rootdomain} as its Symantec Product Authentication Service Domain\n", 40, 2497, "$cfg->{at_rootdomain}");
            $msg->print;
        } elsif ($prod->{novxsscred}) {
            $msg=Msg::new("\nSystems have credentials from $cfg->{at_rootdomain}\nUsing $cfg->{at_rootdomain} as root domain for other cluster systems", 40, 2498, "$cfg->{at_rootdomain}", "$cfg->{at_rootdomain}");
            $msg->print;
            $rtn=$prod->reconfig_secure_cluster();
            if ($rtn) {
                undef ($cfg->{at_rootdomain});
                undef ($cfg->{rootbroker});
                undef ($prod->{novxsscred});
                undef ($prod->{rootbroker});
                undef ($prod->{vxsscred});
                return $edr->{msg}{back}; 
            }
        } else {
            $msg=Msg::new("\nAll systems already have established trust within the Symantec Product Authentication Service domain $cfg->{at_rootdomain}", 40, 2499, "$cfg->{at_rootdomain}");
            $msg->print;
            $rtn=$prod->reconfig_secure_cluster();
            if ($rtn) {
                undef ($cfg->{at_rootdomain});
                undef ($cfg->{rootbroker});
                undef ($prod->{novxsscred});
                undef ($prod->{rootbroker});
                undef ($prod->{vxsscred});
                return $edr->{msg}{back}; 
            }
        }
    } else {
        $msg=Msg::new("Systems will use internal cluster node $prod->{rootbroker} as RAB, all the other nodes as AB.", 40, 2500, "$prod->{rootbroker}");
        $msg->print;
    }
    
    sleep (2);
}

sub web_check_rootbroker {
    my ($prod)=@_;
    my ($at,$cfg,$cpic,$cred,$edr,$msg,$rb,$rtn,$sys,$web);

    $cpic = Obj::cpic();
    $cfg = Obj::cfg();
    $edr = Obj::edr();
    $web = Obj::web();
    $at=Obj::prod("AT50",$prod->{padv});
    
    for $sys (@{$cpic->{systems}}) {
        # todo: check_vxss_sys may die in some cases.
        $cred=$prod->check_vxss_sys($sys);
        if ($cred) {
            $cfg->{at_rootdomain}=$cred;
            $prod->{vxsscred}{$sys->{sys}}=1;
        } else {
            $prod->{novxsscred}++;
        }
    }
    
    if ($cfg->{at_rootdomain}) {
        $cfg->{rootbroker}=$prod->{rootbroker}=$prod->get_cluster_brokername();
        #initialize root broker here
        $rb=$prod->initial_rootbroker($prod->{rootbroker});
        if (!$rb) {
            $msg=Msg::new("Failed to initialize original RB $prod->{rootbroker}.", 40, 3544, "$prod->{rootbroker}");
            $web->tsub("web_script_form","alert",$msg->{msg});
        }
        $rtn=$prod->reconfig_secure_cluster();
        if ($rtn) {
            undef ($cfg->{at_rootdomain});
            undef ($cfg->{rootbroker});
            undef ($prod->{novxsscred});
            undef ($prod->{rootbroker});
            undef ($prod->{vxsscred});
            return ""
        }        
    } else {
        # no rootbroker found, ask for one
        return "";
    }
    return 1;
}

sub web_get_rootbroker {
    my ($prod) = @_;
    my ($at,$atsuffix,$ayn,$description,$msg,$rab,$rootdomain,$rtn,$sys0);
    my $cpic = Obj::cpic();
    my $cfg = Obj::cfg();

    $at=Obj::prod("AT50",$prod->{padv});
    $rootdomain = $at->web_get_rootbroker($cfg->{rootbroker});
    if ($rootdomain=~/root@/) {
        $cfg->{at_rootdomain}=$rootdomain;
    } else {
        return 0;
    }
    return 1;   
}

sub check_vxss_service_group {
    my ($prod)=@_;
    my ($cpic,$sys0,$msg,$cmdoutput);
    
    $cpic=Obj::cpic();
    $sys0=${$cpic->{systems}}[0];
    $cmdoutput=EDR::cmd_sys($sys0,"$prod->{bindir}/hagrp -list 2>/dev/null |_cmd_grep vxss_service");
    if ($cmdoutput eq "") {
        return 0;
    } else {
        $msg=Msg::new("AT is in HA mode in VCS. It is prohibited to reconfigure secure cluster with another root broker under such circumstance. Proceed with the older RB information.", 40, 3546);
        $msg->print;
        return 1;
    }
}

sub disable_vcs_resource {
    my ($prod,$res)=@_;
    my ($sys0,$cpic,$cmdoutput,$rtn);
    
    $cpic=Obj::cpic();
    $sys0=${$cpic->{systems}}[0];
    $cmdoutput=EDR::cmd_sys($sys0,"$prod->{bindir}/hares -list 2>/dev/null |_cmd_grep $res");
    if ($cmdoutput ne ""){ 
        $prod->haconf_makerw();
        EDR::cmd_sys($sys0,"$prod->{bindir}/hares -modify $res Enabled 0 2>/dev/null");
        $rtn=( EDR::cmdexit()!=0 ) ? 0 : 1 ;
        $prod->haconf_dumpmakero();
        return $rtn;
    } else {
        return 1;
    }
}

sub reconfig_secure_cluster {
    my ($prod)=@_;
    my ($at,$ayn,$cpic,$msg,$vxatd,$rtn,$sys,$sys0);
    
    # Reconfigure is blocked for addnode option.
    return 0 if (Cfg::opt("addnode"));
    # Reconfigure is blocked for AT HA mode.
    return 0 if ($prod->check_vxss_service_group());
    
    $msg=Msg::new("Would you like to clear older AB configuration and reconfigure the secure cluster with another RB?", 40, 3547);
    $ayn=$msg->aynn_web();
    return 0 if ($ayn eq "N");
    
    $cpic=Obj::cpic();
    $vxatd=Obj::proc("vxatd50",$prod->{padv});
    $at=Obj::prod("AT50",$prod->{padv});
    # step 1. Disable vxatd resource.
    $msg=Msg::new("Disable vxatd resource", 40, 3548);
    $msg->left;
    $rtn=$prod->disable_vcs_resource("vxatd");
    if ($rtn) {
        Msg::right_done();
    } else {
        Msg::right_failed();
    }
    
    # step 2. Stop vxatd process on each cluster node.
    for $sys (@{$cpic->{systems}}) {
        $msg=Msg::new("Stopping vxatd on $sys->{sys}", 40, 3549, "$sys->{sys}");
        $msg->left;
        $rtn=$vxatd->stop_sys($sys);
        if ($rtn) {
            Msg::right_done();
        } else {
            Msg::right_failed();
        }
    } 
    # step 3. Clear the AB configs on each cluster node.
    for $sys (@{$cpic->{systems}}) {
        $msg=Msg::new("Clearing AT configurations on $sys->{sys}", 40, 3550, "$sys->{sys}");
        $msg->left;
        $rtn=$at->clear_at_config_sys($sys);
        if ($rtn) {
            Msg::right_done();
        } else {
            Msg::right_failed();
        }
    }
        
    return 1; 
}

sub create_vxsscredentials {
    my ($prod)=@_;
    my ($at,$cfg,$cpic,$cmdoutput,$domain,$edr,$j,$msg,$prpl,$prplname,$pwd,$result,$rb,$swaped_cmd,$sys,$sys0,$installdir);

    $cpic=Obj::cpic();
    $cfg=Obj::cfg();
    $edr=Obj::edr();
    $sys0=${$cpic->{systems}}[0];
    # get cluster name and domain to append on system credentials
    $cfg->{vcs_clustername}||=EDR::cmd_sys($sys0, "_cmd_grep '^cluster' $prod->{maincf} | _cmd_awk '{print \$2}'");
    
    #deal with the inner RAB node first
    for $sys (@{$cpic->{systems}}) {
        next if  ($prod->{vxsscred}{$sys->{sys}});
        if ($sys->{sys} eq $prod->{rootbroker}) {
            EDR::cmd_sys($sys, "_cmd_vxatd -o -a -r");
            if ($edr->{cmdexit}) {
                $msg=Msg::new("Failed to configure RAB on cluster node $sys->{sys}", 40, 2501, "$sys->{sys}");
                EDR::push_error_sys($sys,$msg);
                return 0;
            }
            EDR::cmd_sys($sys, "_cmd_vxatd");
            sleep 2; # add this sleep in case that addprpl right after RAB is started.
            $at=Obj::prod("AT50",$prod->{padv});
            $cfg->{at_rootdomain}=$at->tsub("get_rootdomain_sys",$sys);
            #For thread configure mode, set $cfg->{at_rootdomain}
            Thr::setq($cfg->{pool}, "at_rootdomain",$cfg->{at_rootdomain});
            last;
        }
    }
    
    #copy the root_hash file from RB to installer node's $edr->{tmpdir}/
    $prod->tsub("get_root_hash");
    #get prplname and passwd for each sys , then start AB on each sys and establish trust with RB.
    for $sys(@{$cpic->{systems}}) {
        next if (($prod->{vxsscred}{$sys->{sys}}) || ($sys->{sys} eq $prod->{rootbroker}));
        $prplname = $cfg->{vcs_ab_prplname}{$sys->{sys}} || $prod->{ab_prplname}{$sys->{sys}} || "$sys->{sys}$prod->{rootfqdn}";
        # must make transmission of pwd secure
        $pwd = $cfg->{vcs_ab_password}{$sys->{sys}} || $prod->{ab_passwd}{$sys->{sys}};

        #get passwd and addprpl for ABs on the RB.
        if (!$pwd) {
            $rb=$prod->{rootbroker};
            $rb=Obj::sys($rb);
            $at=Obj::prod("AT50",$prod->{padv});
            $installdir=$at->tsub("get_atinstalldir_sys",$rb, 1);
    
            $prod->{ab_passwd}{$sys->{sys}}=$pwd=$prod->randpwd(16);
            # create prpl on the root server
            # check are there any existing credentials on root
            # delete if prpl already exists, as it will have different pwd
            $prpl=EDR::cmd_sys($rb, "$installdir/bin/vssat showprpl --pdrtype root --domain $cfg->{at_rootdomain} --prplname $prplname");
            EDR::cmd_sys($rb, "$installdir/bin/vssat deleteprpl --pdrtype root --domain $cfg->{at_rootdomain} --prplname $prplname --silent") if ($prpl=~/Principal Name:/);
            $edr->{donotlog}=1;
            $cmdoutput = EDR::cmd_sys($rb, "$installdir/bin/vssat addprpl --pdrtype root --domain $cfg->{at_rootdomain} --prplname $prplname --password $pwd --prpltype service");
            $result=$edr->{cmdexit};
            $edr->{donotlog}="";
            $swaped_cmd = EDR::cmdswap_sys($rb, "$installdir/bin/vssat addprpl --pdrtype root --domain $cfg->{at_rootdomain} --prplname $prplname --password password_removed --prpltype service");
            Msg::log("$swaped_cmd\n$cmdoutput");
            if (!$result) {
                 Msg::log("Creating $prplname security principal on $rb->{sys}.....Done");
            } else {
                 $msg=Msg::new("Creating $prplname security principal on $rb->{sys}.....Failed", 40, 2503, "$prplname", "$rb->{sys}");
                 EDR::push_error_sys($sys,$msg);
                 return 0;
            }
        }

        # start in AB mode and establish trust
        $prod->{roothashpath} ||= "$edr->{tmpdir}/root_hash";
        $prod->{vssdefport}||=2821;
        EDR::copy_sys($sys, "$prod->{roothashpath}", "$edr->{tmpdir}/root_hash") 
            unless ($sys->{islocal} && "$prod->{roothashpath}" eq "$edr->{tmpdir}/root_hash");
        $edr->{donotlog}=1;
        $cmdoutput = EDR::cmd_sys($sys, "_cmd_vxatd -o -a -n $prplname -p $pwd -x vx -y $cfg->{at_rootdomain} -q $prod->{rootbroker} -z $prod->{vssdefport} -h $edr->{tmpdir}/root_hash");
        $result=$edr->{cmdexit};
        $edr->{donotlog}="";
        $swaped_cmd = EDR::cmdswap_sys($sys, "_cmd_vxatd -o -a -n $prplname -p password_removed -x vx -y $cfg->{at_rootdomain} -q $prod->{rootbroker} -z $prod->{vssdefport} -h $edr->{tmpdir}/root_hash");
        Msg::log("$swaped_cmd");
        Msg::log("$cmdoutput");
        if ($result == 91) {
            $msg=Msg::new("Error 91: Configuration Failed. Authentication with the root broker failed due to clock skew error. Make sure that the time on $sys->{sys} and the $prod->{rootbroker} is in sync.", 40, 2504, "$sys->{sys}", "$prod->{rootbroker}");
            $msg->log;
            EDR::push_error_sys($sys,$msg);
            return 0;
        } elsif ($result==6) {
            $msg=Msg::new("Error 6: Failed to start cluster node $sys->{sys} in AB mode. It maybe because $sys->{sys} cannot resolve RB's name ($prod->{rootbroker}). Add the RB name ($prod->{rootbroker}) in $sys->{sys} /etc/hosts file and then configure again.", 40, 2505, "$sys->{sys}", "$sys->{sys}", "$prod->{rootbroker}", "$prod->{rootbroker}", "$sys->{sys}");
            $msg->log;
            EDR::push_error_sys($sys,$msg);
            return 0;
        } elsif ($result) {
            $msg=Msg::new("Cannot start vxatd service. The output is $swaped_cmd, the return error number is $result", 40, 2506, "$swaped_cmd", "$result");
            $msg->log;
            EDR::push_error_sys($sys,$msg);
            return 0;
        } else {
            Msg::log("Starting Symantec Security Services on $sys->{sys}......Done");
        }  
        EDR::cmd_sys($sys, "_cmd_rmr $edr->{tmpdir}/root_hash") unless($sys->{islocal});
        $result=0;
    }

    #create pd/principal for VCS and Web server
    for $sys(@{$cpic->{systems}}) {
        next if (($prod->{vxsscred}{$sys->{sys}}) && Cfg::opt("addnode"));        
        EDR::cmd_sys($sys, "_cmd_vxatd 2>/dev/null");
    }

    #rm root_hash file on sys0
    ##TODO:: what if sys0 is the RB and the path is /opt/VRTSat/bin/root_hash??
    EDR::cmd_sys($sys0, "_cmd_rmr $prod->{roothashpath}") 
        if ($sys->{islocal} && "$prod->{roothashpath}" ne "$edr->{tmpdir}/root_hash" && "$prod->{roothashpath}" ne "/opt/VRTSat/bin/root_hash" && "$prod->{roothashpath}" ne "/opt/VRTS/bin/root_hash" );
    return 1;
}

sub list_rootbroker_options
{
    my ($prod)=@_;
    my ($ao,$backopt,$bo,$co,$defanswer,$edr,$help,$localsys,$menu,$menuopt,$msg);

    $edr=Obj::edr();
    $localsys=$edr->{localsys}->{sys};
    Msg::title();
    $msg=Msg::new("Security can be configured automatically or semi-automatically by the installer. In automatic mode, user intervention is not required. In semi automatic mode, Authentication Broker related setup on Root Broker is expected to be performed by the Root Broker Administrator, and installer will prompt for some information that will be used to configure Authentication Brokers.\n", 40, 2510);
    $msg->print;
    $msg=Msg::new("Security Menu", 40, 2511);
    $msg->bold;
    $defanswer=1;
    $backopt=1;
    $help=Msg::new("In option 1, passwordless rsh/ssh login to root broker is required\nIn option 2, Symantec Product Authentication Service version that supports BLOBs is required\nIn option 3, User is required to create AB accounts on RB and provide the credentials. User is also required to copy the root_hash file to the installer node ($localsys)\n\nOption 1 is automatic mode while options 2 and 3 are semi automatic modes.", 40, 2512, "$localsys");
    $msg=Msg::new("Configure security completely automatically", 40, 2513);
    $ao=$msg->{msg};
    $msg=Msg::new("Provide AB credentials using BLOBs", 40, 2514);
    $bo=$msg->{msg};
    $msg=Msg::new("Provide AB credentials without using BLOBs", 40, 2515);
    $co=$msg->{msg};
    $menuopt=[ $ao, $bo, $co ];
    $msg=Msg::new("Select the Security option you would like to perform:", 40, 2516);
    $menu=$msg->menu($menuopt,$defanswer,$help,$backopt);
    return $menu;
}

sub configure_vxss {
    my ($prod)=@_;
    my ($cfg,$cpic,$edr,$maincf,$mode,$msg,$nsys,$sys,$sysname,$syss,$systems,$result);
    
    $msg=Msg::new("\nRestarting $prod->{abbr} in secure mode:\n", 40, 2517, "$prod->{abbr}");
    $msg->bold;
    $msg=Msg::new("Creating Symantec Product Authentication Service Group", 40, 2518);
    $msg->left;

    $cpic=Obj::cpic();
    for $sys(@{$cpic->{systems}}) { 
        EDR::cmd_sys($sys, "_cmd_touch $prod->{secure}"); 
    }
    $sys=${$cpic->{systems}}[0];
    $prod->tsub("haconf_makerw");
    EDR::cmd_sys($sys, "$prod->{bindir}/hagrp -add VxSS");
    for $nsys(0..$#{$cpic->{systems}}) { 
        $sysname="$cpic->{systems}->[$nsys]->{sys}";
        $syss.="$sysname ";
        $systems.="$sysname $nsys "; 
    }
    EDR::cmd_sys($sys, "$prod->{bindir}/hagrp -modify VxSS SystemList $systems");
    EDR::cmd_sys($sys, "$prod->{bindir}/hagrp -modify VxSS Parallel 1");
    EDR::cmd_sys($sys, "$prod->{bindir}/hagrp -modify VxSS AutoStartList $syss");
    EDR::cmd_sys($sys, "$prod->{bindir}/hagrp -modify VxSS OnlineRetryLimit 3");
    EDR::cmd_sys($sys, "$prod->{bindir}/hagrp -modify VxSS OnlineRetryInterval 120");
    EDR::cmd_sys($sys, "$prod->{bindir}/hares -add vxatd ProcessOnOnly VxSS");
    #EDR::cmd_sys($sys, "$prod->{bindir}/hares -modify vxatd PathName \"/opt/VRTSat/bin/vxatd\"");
    EDR::cmd_sys($sys, "$prod->{bindir}/hares -modify vxatd PathName /opt/VRTSat/bin/vxatd");
    EDR::cmd_sys($sys, "$prod->{bindir}/hares -modify vxatd IgnoreArgs 1");
    EDR::cmd_sys($sys, "$prod->{bindir}/hares -modify vxatd Enabled 1");
    EDR::cmd_sys($sys, "$prod->{bindir}/hares -add phantom_vxss Phantom VxSS");
    EDR::cmd_sys($sys, "$prod->{bindir}/hares -modify phantom_vxss Enabled 1");
    $cfg=Obj::cfg();
    push(@{$cfg->{vcs_username}}, "root\@$sys->{sys}$prod->{rootfqdn}");
    $prod->tsub("haconf_dumpmakero");
    $msg=Msg::new("Done", 40, 1116);
    $msg->right;
    
    $prod->tsub("stop_vcs");
    # modify main.cf
    $msg=Msg::new("Updating $prod->{abbr} configuration", 40, 2519, "$prod->{abbr}");
    $msg->left;
    $maincf=EDR::cmd_sys($sys, "_cmd_cat $prod->{maincf}");
    if ($maincf=~/SecureClus\s*=\s*0/) {
        $maincf=~s/SecureClus\s*=\s*0/SecureClus = 1/;
    } else {
        $cfg->{vcs_clustername}||=EDR::cmd_sys($sys, "_cmd_grep '^cluster' $prod->{maincf} | _cmd_awk '{print \$2}'");
        $maincf=~s/cluster\s+$cfg->{vcs_clustername}\s*\(/cluster $cfg->{vcs_clustername} \(\n\tSecureClus = 1/;
    }
    $edr=Obj::edr();
    EDR::writefile($maincf, "$edr->{tmpdir}/main.cf", 1);
    $msg=Msg::new("Done", 40, 1116);
    $msg->right;
    
    $nsys=$prod->tsub("secure_startup");
    $prod->tsub("haconf_makerw");
    for $sys(@{$cpic->{systems}}) {
        EDR::cmd_sys($sys, "$prod->{bindir}/hauser -add root\@$sys->{sys}$prod->{rootfqdn} -priv Administrator");
    }
    
    $prod->tsub("haconf_dumpmakero");
    $sys=${$cpic->{systems}}[0];
    $mode=EDR::cmd_sys($sys, "$prod->{bindir}/haclus -value SecureClus");
    $result=$prod->tsub("check_at_onsystems");
    if (($mode==1) && ($nsys==$#{$cpic->{systems}}+1) && ($#$result>=0)) {
        Msg::log("Symantec Security Services enabled successfully\n");
    } else {
        $msg=Msg::new("Symantec Security Services not enabled successfully\n", 40, 2521);
        $msg->print;
    }
    $prod->{completiontitle}=$msg;
}

# Check AT is started on all systems or not.
sub check_at_onsystems {
    my ($prod)=@_;
    my ($at,$cpic,$pids,$sys);

    $cpic=Obj::cpic();
    for $sys (@{$cpic->{systems}}){
        $pids=EDR::pids_sys($sys, "vxatd");
        last if($#$pids == -1);
    }
    return $pids;
}

sub show_blob_sampleinput {
    my $help=<<"_HELP_";

[setuptrust]
broker=<root_broker>
hash=<root_hash>
securitylevel=high

[configab]
identity=<ab_identity>
password=<ab_password>
root_domain=<domain_type>:<root_FQDN>
root_broker=<root_broker>:<root_broker_port>
broker_admin_password=<root_broker_admin_password>
start_broker=false
enable_pbx=false

_HELP_
   Msg::print($help);    
}

sub get_atsuffix{
    my ($prod)=@_;
    my ($cpic,$suffix,$sys0);
    
    $cpic=Obj::cpic();
    $sys0=${$cpic->{systems}}[0];
    $suffix=($sys0->{plat} eq "Linux") ? "Server" : "";
    return $suffix;
}

sub get_blob {
    my ($prod)=@_;
    my ($atsuffix,$ayny,$backopt,$blobpath,$cfg,$cpic,$done,$edr,$msg,$sys);

    $edr=Obj::edr();
    $cpic=Obj::cpic();
    $cfg=Obj::cfg();
    $atsuffix = $prod->get_atsuffix();

    Msg::title();
    $msg=Msg::new("You need to create AB account for each cluster node on the Root Broker. Also user needs to create a BLOB per cluster node. Verify that the version of VRTSat$atsuffix installed on root broker supports BLOB creation. User needs to use --in option to create BLOBs. Make sure that the input file is in the following format:", 40, 2522, "$atsuffix");
    $msg->print;
    $prod->show_blob_sampleinput();
    $msg=Msg::new("Refer to the Symantec Product Authentication Service Documentation for steps to create BLOBs. Installer needs a locally accessible path for BLOBs.  You can either copy the BLOBs on $edr->{localsys}->{sys} or mount the BLOBs using some removable media.", 40, 2523, "$edr->{localsys}->{sys}"); 
    $msg->print;

    $backopt=1;
    $msg=Msg::new("Do you want to continue?", 40, 1337);
    $ayny=$msg->ayny("",$backopt);
    return $ayny if ($ayny eq $edr->{msg}{back});
    EDR::cleanup(0,1) if ($ayny eq "N");
    
    for $sys(@{$cpic->{systems}}) {
        $done=0;
        while(!$done) {
            $msg=Msg::new("Enter the path of BLOB for $sys->{sys}:", 40, 2524, "$sys->{sys}");
            $blobpath=$msg->ask("","",$backopt);
            $cfg->{vcs_blobpath}{$sys->{sys}}=$prod->{blobpath}{$sys->{sys}}=$blobpath;
            return $blobpath if ($blobpath eq $edr->{msg}{back});
            
            next if (!$prod->{blobpath}{$sys->{sys}});
            if (! -e $prod->{blobpath}{$sys->{sys}}) {
                $msg=Msg::new("File $prod->{blobpath}{$sys->{sys}} does not exist", 40, 2525, "$prod->{blobpath}{$sys->{sys}}");
                $msg->print;
                next;
            }
            if (-d $prod->{blobpath}{$sys->{sys}}) {
                $msg=Msg::new("$prod->{blobpath}{$sys->{sys}} is a directory. Input a file.", 40, 2526, "$prod->{blobpath}{$sys->{sys}}");
                $msg->print;
                next;
            }
            $done=1;
        }
    }
}

sub create_vxsscred_using_blob {
    my ($prod)=@_;
    my ($cfg,$cpic,@d,$domain,$edr,$f,$msg,$prpl,$pwd,$rtn,$sys);

    $cpic=Obj::cpic();
    $cfg=Obj::cfg();
    $edr=Obj::edr();
    
    for $sys(@{$cpic->{systems}}) {
        next if (($prod->{vxsscred}{$sys->{sys}}) || ($sys->{sys} eq $prod->{rootbroker}));#skip the RB in cluster.

        $prod->{blobpath}{$sys->{sys}} = $cfg->{vcs_blobpath}{$sys->{sys}} if ($cfg->{vcs_blobpath}{$sys->{sys}});
        EDR::copy_sys($sys, $prod->{blobpath}{$sys->{sys}}, "$edr->{tmpdir}/credentials.$sys->{sys}");
        EDR::cmd_sys($sys, "_cmd_vssat execpkg --in $edr->{tmpdir}/credentials.$sys->{sys} --ob --host_ctx");
        $rtn=$edr->{cmdexit};
        if ($rtn) {
            $msg=Msg::new("Executing BLOB on $sys->{sys}.....Failed", 40, 2527, "$sys->{sys}");
            EDR::push_error_sys($sys,$msg);
            return 0;
        } else {
            EDR::cmd_sys($sys, "_cmd_vxatd 2>/dev/null");
            Msg::log("Executing BLOB on $sys->{sys}.....Done");
        }
    }
    return 1;
}

sub get_vxsscred_using_prompts {
    my ($prod)=@_;
    my ($at,$backopt,$cfg,$cpic,$def,$edr,$msg,$prplname,$passwd,$sys);

    $edr=Obj::edr();
    $cpic=Obj::cpic();
    $cfg=Obj::cfg();
    Msg::title();
    $msg=Msg::new("User needs to create authentication broker principal for each cluster node on the Root Broker. Refer to the Symantec Product Authentication Service Documentation for the configuration steps. Also make sure that the root_hash file is either copied to the installer node, or is locally accessible (via mounted file system or any other means). Installer will ask for locally accessible path of root_hash file. User also needs the AB principal passwords.\n\nPress 'b' anytime (expect when prompted for passwords) to go to the previous menu.", 40, 2530);
    $msg->print;
    Msg::prtc();

    $backopt=1;
    #ask for RB name
    while(1) {
        $msg=Msg::new("Enter root broker name: ", 40, 2531);
        $prod->{rootbroker}=$msg->ask("","",$backopt);
        return $edr->{msg}{back} if ($prod->{rootbroker} eq $edr->{msg}{back});
        if (!$edr->hosttest_sysname($prod->{rootbroker})) {
            $msg=Msg::new("Error: Cannot resolve the root broker name $prod->{rootbroker}", 40, 3551, "$prod->{rootbroker}");
            $msg->print;
            next;
        }
        last;
    }
    #ask for RB FQDN.
    ($prod->{rootbroker},$prod->{rootfqdn}) = split(/\./, $prod->{rootbroker}, 2);
    $def = $prod->{rootfqdn};
    $msg=Msg::new("Enter root broker FQDN: ", 40, 2532);
    $prod->{rootfqdn}=$msg->ask($def,"",$backopt);
    return $edr->{msg}{back} if ( $prod->{rootfqdn} eq $edr->{msg}{back});
    $prod->{rootfqdn}=".$prod->{rootfqdn}" if $prod->{rootfqdn} !~ /^\./;
    
    #ask for RB domain
    $def="root\@$prod->{rootbroker}$prod->{rootfqdn}"; # if $prod->{rootbroker} is not an IP
    $msg=Msg::new("Enter the root broker domain name for the Authentication Broker's identity: ", 40, 2533);
    $prod->{at_rootdomain} = $msg->ask($def,"",$backopt);
    return $edr->{msg}{back} if ($prod->{at_rootdomain} eq $edr->{msg}{back});

    #ask for RB port
    $def=$prod->{vssdefport};
    while (1) {
        $msg=Msg::new("Enter root broker port: ", 40, 2534);
        $prod->{vssdefport}=$msg->ask($def,"",$backopt);
        return $edr->{msg}{back} if ($prod->{vssdefport} eq $edr->{msg}{back});
        $prod->{vssdefport}=~s/^0*//;
        if ($prod->{vssdefport}=~/\D/ || $prod->{vssdefport}<1 || $prod->{vssdefport}>65535) {
            $msg=Msg::new("Error: The root broker port should be a positive number between 1 and 65535", 40, 3552);
            $msg->print;
            next;
        }
        last;
    }
    
    #ask for root_hash file's local location.
    while(1) {
        $msg=Msg::new("Enter path to the locally accessible root hash", 40, 2535);
        $prod->{roothashpath} = $msg->ask("","",$backopt);
        return $edr->{msg}{back} if ($prod->{roothashpath} eq $edr->{msg}{back});
        
        next if (!$prod->{roothashpath});
        if (! -e $prod->{roothashpath}) {
            $msg=Msg::new("File $prod->{roothashpath} does not exist", 40, 2525, "$prod->{roothashpath}");
            $msg->print;
            next;
        }
        if (-d $prod->{roothashpath}) {
            $msg=Msg::new("$prod->{roothashpath} is a directory", 40, 2536, "$prod->{roothashpath}");
            $msg->print;
            next;
        }
        last;
    }

    #ask prplname/password for each system. 
    $at=Obj::prod("AT50",$prod->{padv});
    for $sys(@{$cpic->{systems}}) {
        $def="$sys->{sys}$prod->{rootfqdn}";
        $msg=Msg::new("Enter Authentication broker's identity on $sys->{sys}", 40, 2537, "$sys->{sys}");
        $prplname=$msg->ask($def,"",$backopt);
        $cfg->{vcs_ab_prplname}{$sys->{sys}}=$prod->{ab_prplname}{$sys->{sys}}=$prplname;
        return $edr->{msg}{back} if ($prplname eq $edr->{msg}{back});

        $msg=Msg::new("Enter the password for the Authentication broker's identity on $sys->{sys}: ", 40, 2538, "$sys->{sys}");
        $passwd=$at->at_read_password("$edr->{tput}{bs}$msg->{msg}$edr->{tput}{be}");
        $cfg->{vcs_ab_password}{$sys->{sys}} = $prod->{ab_passwd}{$sys->{sys}}=$passwd;
        return $edr->{msg}{back} if ($passwd eq $edr->{msg}{back});
    }

    #save cfg variables.
    $cfg->{rootbroker} = $prod->{rootbroker};
    $cfg->{at_rootdomain} = $prod->{at_rootdomain};
    $cfg->{vcs_vssdefport} = $prod->{vssdefport};
    $cfg->{vcs_roothashpath} = $prod->{roothashpath};
    
}

sub blob {
    my ($prod)=@_;
    my ($edr,$rtn);
    $edr=Obj::edr();
    
    $rtn=$prod->tsub("get_blob");
    return $rtn if ($rtn eq $edr->{msg}{back});
    $rtn=$prod->tsub("create_vxsscred_using_blob");
    return $rtn;
}

sub vxss_prompts {
    my ($prod)=@_;
    my ($edr,$rtn);
    $edr=Obj::edr();
    
    $rtn=$prod->tsub("get_vxsscred_using_prompts");
    return $rtn if ($rtn eq $edr->{msg}{back});
    $rtn=$prod->tsub("create_vxsscredentials");
    return $rtn;
}

sub create_vxsscred_automatically {
    my ($prod)=@_;
    my ($edr,$rtn);
    $edr=Obj::edr();
       
    $rtn=$prod->tsub("check_rootbroker");
    return $rtn if ($rtn eq $edr->{msg}{back});
    $rtn=$prod->tsub("create_vxsscredentials");
    return $rtn;
}

sub setup_vxsscredentials {
    my ($prod)=@_;
    my $rtn;
    if ($prod->{vxssmenuopt} == 2 ) {
        $rtn=$prod->tsub("create_vxsscred_using_blob");
    } else {
        $rtn=$prod->tsub("create_vxsscredentials");
    }
    return $rtn;
}

sub security_configuration_options {
    my ($prod)=@_;
    my ($cfg,$done,$edr,$menu,$rtn);

    $cfg=Obj::cfg();
    $edr=Obj::edr();
    while (!$done) {
        $rtn="";
        $menu=$prod->list_rootbroker_options;
        return $menu if ($menu eq $edr->{msg}{back});
        if ($menu==1) {
            $rtn=$prod->check_rootbroker;
        } elsif ($menu==2){
            $rtn=$prod->get_blob;
        } elsif ($menu==3){
            $rtn=$prod->get_vxsscred_using_prompts;
        }
        $done=1 unless ($rtn eq $edr->{msg}{back});
    }
    $cfg->{vcs_securitymenuopt}=$prod->{vxssmenuopt}=$menu;
}

sub enable_vxss {
    my ($prod)=@_;
    my ($cpic,$done,$edr,$sysname,$sys,$menu,$msg,$rcs,$rtn,$errmsg);

    $rcs=$prod->tsub("get_vxss_cluster");
    $cpic=Obj::cpic();
    $edr=Obj::edr();
    # generate sys obj according to sysname list
    if ($rcs) {
        for $sysname (@$rcs) {
            $sys = ($Obj::pool{"Sys::$sysname"}) ? Obj::sys($sysname)
                                                 : EDR::tsub("Sys","new",$sysname);
            if (!$edr->tsub("transport_sys",$sys)) {
                $msg=Msg::new("Cannot communicate with system $sys->{sys}\n", 40, 2461, "$sys->{sys}");
                $msg->die;
            } 
            push (@{$cpic->{systems}},$sys) unless (EDR::inarr($sys,@{$cpic->{systems}}));
            push (@{$edr->{systems}},$sys) unless (EDR::inarr($sys,@{$edr->{systems}}));
        }
    } else {
        $msg=Msg::new("Enable vxss error while getting cluster information", 40, 2539);
        $msg->die;
    }
    
    while (!$done) {
        $rtn="";
        $menu=$prod->tsub("list_rootbroker_options");
        if ($menu eq $edr->{msg}{back}) {
            undef($cpic->{systems});
            return $menu;
        }
        $rtn=$prod->tsub("create_vxsscred_automatically") if ($menu==1);
        $rtn=$prod->tsub("blob")                          if ($menu==2);
        $rtn=$prod->tsub("vxss_prompts")                  if ($menu==3);
        $done=1 unless ($rtn eq $edr->{msg}{back});
    }
    if ($rtn) {
        $prod->tsub("configure_vxss");
    } else {
        $errmsg=Msg::new("Failed to set up secure cluster due to incorrect configuration.", 40, 3553);
        $edr->tsub("display_errors_warnings","",$errmsg);
    }
    
}

sub disable_vxss {
    my ($prod)=@_;
    my($cpic,$edr,$grp,@grplist,$maincf,$mode,$msg,$nsys,$rcs,$rg_attr,$sys,$system,$sysname);

    $cpic=Obj::cpic();
    $prod->{disablevxss}=1;
    $edr=Obj::edr();
    $rcs=$prod->tsub("get_vxss_cluster");
    # generate sys obj according to sysname list
    if ($rcs) {
        for $sysname (@$rcs) {
            $sys = ($Obj::pool{"Sys::$sysname"}) ? Obj::sys($sysname)
                                                 : EDR::tsub("Sys","new",$sysname);
            if (!$edr->tsub("transport_sys",$sys)) {
                $msg=Msg::new("Cannot communicate with system $sys->{sys}\n", 40, 2461, "$sys->{sys}");
                $msg->die;
            }
            push (@{$cpic->{systems}},$sys) unless (EDR::inarr($sys,@{$cpic->{systems}}));
        }
    } else {
        $msg=Msg::new("Disable vxss error while getting cluster information", 40, 2540);
        $msg->die;
    }
    $sys=${$cpic->{systems}}[0];
    $prod->{rootfqdn}||=$prod->vxss_get_fqdn($sys);

    Msg::title();
    $msg=Msg::new("\nRestarting $prod->{abbr} in non-secure mode:\n", 40, 2541, "$prod->{abbr}");
    $msg->bold;
    $msg=Msg::new("Deleting Symantec Product Authentication Service Group", 40, 2542);
    $msg->left;
    $prod->tsub("haconf_makerw");
    EDR::cmd_sys($sys, "$prod->{bindir}/hares -delete vxatd");
    EDR::cmd_sys($sys, "$prod->{bindir}/hares -delete phantom_vxss");
    EDR::cmd_sys($sys, "$prod->{bindir}/hagrp -delete VxSS");
    $prod->tsub("haconf_dumpmakero");
    $msg->right_done;

    $msg=Msg::new("Deleting secure cluster users", 40, 2543);
    $msg->left;
    $prod->tsub("haconf_makerw");
    for $system (@{$cpic->{systems}}) {
        EDR::cmd_sys($sys, "$prod->{bindir}/hauser -delpriv root\@$system->{sys}$prod->{rootfqdn} Administrator");
        EDR::cmd_sys($sys, "$prod->{bindir}/hauser -delete root\@$system->{sys}$prod->{rootfqdn}");
    }
    $prod->tsub("haconf_dumpmakero");
    $msg->right_done;

    $msg=Msg::new("Clearing any role grouping attributes", 40, 2544);
    $msg->left;
    $prod->tsub("haconf_makerw");
    for $rg_attr ( @{$prod->{rolegroupingattrs}} ) {
        EDR::cmd_sys($sys, "$prod->{bindir}/haclus -modify $rg_attr -delete -keys");
    }
    $grp = EDR::cmd_sys($sys, "$prod->{bindir}/hagrp -list 2>/dev/null | _cmd_awk '{print \$1}' | _cmd_sort | _cmd_uniq");
    @grplist = split (/\n/, $grp);
    for $grp (@grplist) {
        for $rg_attr (@{$prod->{rolegroupingattrs}}) {
            EDR::cmd_sys($sys, "$prod->{bindir}/hagrp -modify $grp $rg_attr -delete -keys");
        }
    }

    $prod->tsub("haconf_dumpmakero");
    $msg->right_done;

    $prod->tsub("stop_vcs");
    $msg=Msg::new("Updating $prod->{abbr} configuration", 40, 2519, "$prod->{abbr}");
    $msg->left;
    $maincf=EDR::cmd_sys($sys, "_cmd_cat $prod->{maincf}");
    $maincf=~s/SecureClus\s*=\s*1/SecureClus = 0/;
    $edr=Obj::edr();
    EDR::writefile($maincf, "$edr->{tmpdir}/main.cf", 1);
    $msg->right_done;
    for $sys(@{$cpic->{systems}}) { 
        EDR::cmd_sys($sys, "_cmd_rmr $prod->{secure}"); 
    }
    $nsys=$prod->tsub("secure_startup");
    $sys=${$cpic->{systems}}[0];
    $mode=EDR::cmd_sys($sys, "$prod->{bindir}/haclus -value SecureClus");
    if (($nsys==$#{$cpic->{systems}}+1) && ($mode==0)) {
        $prod->{completiontitle}=Msg::new("Symantec Security Services disabled successfully\n", 40, 2545);
    } else {
        $prod->{completiontitle}=Msg::new("Symantec Security Services not disabled successfully\n", 40, 2546);
    }
}

sub security_menu {
    my ($prod)=@_;
    my ($cfg,$cpic,$do,$done,$edr,$eo,$io,$menu,$menuopt,$msg,$plat,$ro,$rtn);

    Cfg::unset_opt("install");
    Cfg::set_opt("security",1);
    $cpic=Obj::cpic();
    $edr=Obj::edr();
    $edr->{savelog} = 1;
    
    $msg=Msg::new("Enable Symantec Security Services on a $prod->{abbr} Cluster", 40, 2547, "$prod->{abbr}");
    $eo=$msg->{msg};
    $msg=Msg::new("Disable Symantec Security Services on a $prod->{abbr} Cluster", 40, 2548, "$prod->{abbr}");
    $do=$msg->{msg};
    while (!$done) {
        $rtn="";
        $menuopt=[$eo,$do];
        $msg=Msg::new("A Symantec Security Services Root Broker and Authentication Broker must be installed using the Veritas Storage Solutions CD\n", 40, 2549);
        $msg->bold;
        $msg=Msg::new("Select the Security option you would like to perform", 40, 2550);
        $menu=$msg->menu($menuopt);

        $rtn=$prod->tsub("enable_vxss") if ($menu==1);
        $prod->tsub("disable_vxss") if ($menu==2);
        $done=1 unless ($rtn eq $edr->{msg}{back});
    }
    Cfg::set_opt("responsefile");
    $cpic->tsub("completion");
}

sub vxfen_option {
    my ($cfg,$cpic,$edr,$pkg,$prod);
    $prod=shift;
    $cpic=Obj::cpic();
    $cfg=Obj::cfg();
    $pkg=Obj::pkg("VRTSvxfen51",$prod->{padv});
    $edr=Obj::edr();
    $edr->{savelog} = 1;
    $prod->tsub("get_cluster_system");
    $pkg->tsub("config_vxfen");
    $cpic->tsub("completion");
}

sub get_cluster_system {
    # get the system cluster configuration infomation
    my (@systems,$ayn,$clustername,$clusterid,$conf,$done,$failed,$msg,$prod,$sys,$sysname,$syslist);
    my ($cfg,$cpic,$edr,$localsys);
    $prod=shift;
    $cfg=Obj::cfg();
    $cpic=Obj::cpic();
    $edr=Obj::edr();
    $localsys=Obj::localsys();
    $failed=0;
    while(1){
        if ($failed) {
            if (Cfg::opt("responsefile")) {
                $msg=Msg::new("An error occured when using responsefile", 40, 2551);
                $msg->die;
            }
            $msg=Msg::new("Would you like to configure I/O fencing for another $prod->{abbr} cluster?", 40, 2552, "$prod->{abbr}");
            $ayn=$msg->aynn;
            EDR::cleanup(0,1) if ($ayn eq "N"); 
            do {
                $msg=Msg::new("Enter the name of one system in another $prod->{abbr} cluster that you would like to configure I/O fencing:", 40, 2553, "$prod->{abbr}");
                $sysname=$msg->ask;
                # chop extra systems, in case they enter more
                $sysname=EDR::despace($sysname);
                $sysname=~s/\s.*$//;
            } while ($edr->tsub("validate_systemnames",$sysname));
        } else {
            if(${$cfg->{systems}}[0]) {
                $sysname=${$cfg->{systems}}[0];
            } else {
                $sysname=$localsys->{sys};
            }
        }
        $failed=1;

        $sys=($Obj::pool{"Sys::$sysname"}) ? 
            Obj::sys($sysname) : EDR::tsub("Sys","new",$sysname);
        $msg=Msg::new("Checking communication on $sys->{sys}", 40, 2554, "$sys->{sys}");
        $msg->left;
        EDR::set_value_sys($sys, "stop_checks", 0);
        if ($edr->tsub("transport_sys", $sys)) {
            $msg->right_done;
        } else {
            $msg->right_failed;
            next;
        }
        $msg=Msg::new("Checking release compatibility on $sys->{sys}", 40, 2555, "$sys->{sys}");
        $msg->left;
        if ($edr->tsub("rel_padv_sys", $sys)) {
            Msg::right_done();
        } else {
            Msg::right_failed();
            next;
        }
        next if (!$prod->tsub("check_vcs_install_sys", $sys));

        $conf=$prod->get_config_sys($sys);
        if ($conf) {
                $clustername=$conf->{clustername};
                @systems=@{$conf->{systems}};
                $clusterid=$conf->{clusterid};
        } else {
            $msg=Msg::new("Cluster configuration information check failed on $sys->{sys}", 40, 2556, "$sys->{sys}");
            $msg->print;
            next;
        }

        Msg::title();
        $msg=Msg::new("Cluster information verification:\n", 40, 2383);
        $msg->bold;
        $msg=Msg::new("\tCluster Name: $clustername\n", 40, 2557, "$clustername");
        $msg->print;
        $msg=Msg::new("\tCluster ID Number: $clusterid\n", 40, 2558, "$clusterid");
        $msg->print;
        $syslist=join(" ", @systems);
        $msg=Msg::new("\tSystems: $syslist", 40, 2463, "$syslist");
        $msg->print;

        if (!Cfg::opt("responsefile")) {
            $msg=Msg::new("Would you like to configure I/O fencing on the cluster?", 40, 2559);
            $ayn=$msg->ayn;
        } else {
            $ayn="Y";
        }
        next unless ($ayn eq "Y");

        $done=1;
        for $sysname(@systems) {
            $sys=($Obj::pool{"Sys::$sysname"}) ? Obj::sys($sysname)
                                                 : EDR::tsub("Sys","new",$sysname);
            $msg=Msg::new("Checking communication on $sys->{sys}", 40, 2554, "$sys->{sys}");
            $msg->left;
            EDR::set_value_sys($sys, "stop_checks", 0);
            if ($edr->tsub("transport_sys", $sys)) {
                $msg->right_done;
            } else {
                $msg->right_failed;
                $done=0;
                next;
            }
            $msg=Msg::new("Checking release compatibility on $sys->{sys}", 40, 2555, "$sys->{sys}");
            $msg->left;
            if ($edr->tsub("rel_padv_sys", $sys)) {
                Msg::right_done();
            } else {
                Msg::right_failed();
                $done=0;
                next;
            }
            if (!$prod->tsub("check_vcs_install_sys", $sys)) {
                $done=0;
                next;
            }
        }
        last if ($done);
    }
    $cfg->{vcs_clusterid}=$clusterid;
    $cfg->{vcs_clustername}=$clustername;
    @{$cfg->{systems}}=@systems;
    return 1;
}

sub check_vcs_install_sys {
    my ($ivers,$msg,$prod,$result,$sys);
    ($prod,$sys)=(@_);
    $msg=Msg::new("Checking $prod->{abbr} installation on $sys->{sys}", 40, 2560, "$prod->{abbr}", "$sys->{sys}");
    $msg->left;
    $ivers=$prod->tsub("version_sys", $sys, 1);
    if ($ivers) {
        $msg=Msg::new("$ivers", 40, 1149, "$ivers");
        $msg->right;
        $result=1;
    } else {
        $msg=Msg::new("not installed", 40, 2561);
        $msg->right;
        $result=0;
    }
    return $result;
}

# 0 the link speed of each private NICs are equal
# 1 the link speed of some private NICs are different
# 2 failed to detect link speed on some private NICs
sub check_link_speed {
    my ($diff,$fail,$key,$key_high, $key_low, $link,$msg,$nic,$selected_speed,$common_speed,$speed_value);
    my ($out,$sys,$sysi,$sysii);
    my ($prod,$systems,$nics) = @_;
    my $web = Obj::web();
    
    $diff = 0;
    $fail = 0;
    for $sysii (@$systems) {
        if (ref($sysii) eq "Sys") {
            $sys = $sysii;
        } else {
            $sys = Obj::sys("$sysii");
        }
        $sysi = $sys->{sys};
        for $link (1..$prod->num_hbnics($nics)) {
            $key = "lltlink$link";
            $key_high = "lltlink$link"."_high";
            $key_low = "lltlink$link"."_low";
            next unless (defined($nics->{$key}{$sysi}));
            $nic = $nics->{$key}{$sysi};
            $msg = Msg::new("Checking media speed for $nic on $sysi", 40, 3554, "$nic", "$sysi");
            $msg->left();
            $selected_speed = $sys->padv()->nic_speed_sys($sys,$nic);

            if ($selected_speed =~ /\D*(\d+)\s*[Gg]/) {
                $speed_value = $1;
                $speed_value *= 1000;
            } elsif ($selected_speed =~ /\D*(\d+)\D*/) {
                $speed_value = $1;
            } else {
                $speed_value = 0;
            }
            if ( ! defined($nics->{$key_high})) {
                $nics->{$key_high} = $speed_value;
            } elsif ( $nics->{$key_high} < $speed_value ) {
                $nics->{$key_high} = $speed_value;
            }
            if ( !defined($nics->{$key_low})) {
                $nics->{$key_low} = $speed_value;
            } elsif ( $nics->{$key_low} > $speed_value ) {
                $nics->{$key_low} = $speed_value;
            }
            
            if ($selected_speed eq "") {
                $out = Msg::new("Unknown", 40, 3555)->{msg};
            } else {
                $out = $selected_speed;
            }
            Msg::right("$out");
            if (Obj::webui()) {
                $web->{nicspeedinfo} .= Msg::new("The media speed of $nic on $sysi is: $out", 40, 3556, "$nic", "$sysi", "$out")->{msg};
                $web->{nicspeedinfo} .= "\n";
            }
            if ($selected_speed !~ /\d+/) {
                $fail = 1;
                next;
            }                
            unless (defined($common_speed) || $fail) {
                $common_speed = $selected_speed;
                next;
            }
            unless($diff || $fail) {
                $diff = 1 if ($common_speed ne $selected_speed);
            }
        }
        $sys->{nics_comm_speed} = $common_speed unless($diff || $fail);
    }

    return 2 if ($fail);
    return 1 if ($diff);
    return 0;
}

sub web_check_link_speed {
    my ($prod, $diff) = @_;
    my ($desc,$msg);
    my $edr = Obj::edr();
    my $web = Obj::web();
    
    if ($diff == 1) {
        $desc = Msg::new("The private NICs do not have same media speed.\n", 40, 2958)->{msg};
        $desc .= Msg::new("It is recommended that the media speed be same for all the private NICs. Without this, LLT may not function properly. Consult your Operating System manual for information on how to set the Media Speed.\n", 40, 2381)->{msg};
        $desc .= $web->{nicspeedinfo};
    }                    
    if ($diff == 2) {
        $desc = Msg::new("$edr->{script} can not detect media speed for the selected private NICs properly. Consult your Operating System manual for information on how to set the Media Speed.\n", 40, 2959, "$edr->{script}")->{msg};
        $desc .= $web->{nicspeedinfo};
    }
    delete $web->{nicspeedinfo};
    if ($diff) {    
        $msg = Msg::new("Do you want to continue with current heartbeat configuration?", 40, 2382);
        my $answer = $msg->ayny_web('','',$desc);
        return 0 if ($answer ne "Y");
    }
    return 1;
}

sub postcheck_lltstat_sys {
    my ($prod,$sys,$rtn,$msg,$line,$llt_sys,@down_links,$link,$links);
    ($prod,$sys)=(@_);

    $rtn=EDR::cmd_sys($sys, "_cmd_lltstat -nvv 2>&1");
    $llt_sys="";
    foreach $line(split(/\n/,$rtn)) {
        if ($line=~/^\s*\*?\s*\d{1,2}\s+(\S+)\s+\S+/) {
            $llt_sys=$1;
        } elsif ($line=~/^\s*\*?\s*\d{1,2}\s+/) {
            $llt_sys="";
        } elsif ($llt_sys && $line=~/^\s*(\S+)\s+DOWN/) {
            push(@down_links, "$llt_sys\t$1");
        }
    }

    if (@down_links) {
        $links=join("\n\t",@down_links);
        $msg=Msg::new("Checking from $sys->{sys}, 'lltstat -nvv' command shows the following LLT links are in DOWN state:\n\t$links", 40, 3557, "$sys->{sys}", "$links");
        EDR::push_warning_sys($sys,$msg);
        return 0;
    }

    return 1;
}

sub postcheck_lltconfig_sys {
    my ($prod,$sys,$rtn,$msg);
    ($prod,$sys)=(@_);

    $rtn=EDR::cmd_sys($sys, "_cmd_lltconfig 2>&1");
    unless ($rtn=~/LLT is running/) {
        $msg=Msg::new("LLT is not in 'running' state on $sys->{sys}", 40, 3558, "$sys->{sys}");
        EDR::push_error_sys($sys,$msg);
        return 0;
    }
    return 1;
}

sub postcheck_vxfen_sys {
    my ($prod,$sys,$rtn,$line,$msg);
    ($prod,$sys)=(@_);

    $rtn=EDR::cmd_sys($sys, "_cmd_vxfenadm -d 2>&1");
    foreach $line(split(/\n/,$rtn)) {
        if ($line=~/VXFEN vxfenadm ERROR/) {
            $msg=Msg::new("VxFEN is not started on $sys->{sys}", 40, 3559, "$sys->{sys}");
            EDR::push_error_sys($sys,$msg);
            return 0;
        } elsif ($line=~/Fencing Mode: Disabled/) {
            $msg=Msg::new("VxFEN is in disable state on $sys->{sys}", 40, 3560, "$sys->{sys}");
            EDR::push_warning_sys($sys,$msg);
            return 0;
        }
    }
    return 1;
}

sub postcheck_gabconfig_sys {
    my ($prod,$sys,$rtn,$line,$msg,%gab_ports,%gab_jeopardy_ports,@ports,@jeopardy_ports,$ports,$port);
    ($prod,$sys)=(@_);

    $rtn=EDR::cmd_sys($sys,"_cmd_gabconfig -l 2> /dev/null| _cmd_grep 'Driver state' | _cmd_awk '{ print \$4;}'");
    if (($rtn ne "Configured") && ($rtn ne "ConfiguredPartition")) {
        $msg=Msg::new("GAB is not in configured state on $sys->{sys}", 40, 3243, "$sys->{sys}");
        EDR::push_error_sys($sys,$msg);
        return 0;
    }

    $rtn=EDR::cmd_sys($sys, "_cmd_gabconfig -a 2>&1 | _cmd_grep 'Port' | _cmd_grep 'gen'");
    foreach $line (split(/\n/,$rtn)) {
        if ($line=~/GAB gabconfig ERROR/) {
            $msg=Msg::new("GAB is not started on $sys->{sys}", 40, 3244, "$sys->{sys}");
            EDR::push_error_sys($sys,$msg);
            return 0;
        } elsif ($line=~/^Port (\S)\s+gen\s+\S+\s+membership/) {
            $gab_ports{$1}=1;
        } elsif ($line=~/^Port (\S)\s+gen\s+\S+\s+jeopardy/) {
            $gab_jeopardy_ports{$1}=1;
        }
    }

    foreach $port (qw(a b h)) {
        push (@ports, $port) unless($gab_ports{$port});
        push (@jeopardy_ports, $port) if($gab_jeopardy_ports{$port});
    }

    if (@ports) {
        $ports=join(',',@ports);
        $msg=Msg::new("The following gab ports are not started on $sys->{sys}:\n\t$ports", 40, 3245, "$sys->{sys}", "$ports");
        EDR::push_warning_sys($sys,$msg);
    }

    if (@jeopardy_ports) {
        $ports=join(',',@jeopardy_ports);
        $msg=Msg::new("The following gab ports are in jeopardy state on $sys->{sys}:\n\t$ports", 40, 3246, "$sys->{sys}", "$ports");
        EDR::push_warning_sys($sys,$msg);
        return 0;
    }

    return 1 unless (@ports);
    return 0;
}

sub postcheck_hastatus_sys {
    my ($prod,$sys,$rtn,$line,$msg,$warning);
    my (@systems,$systems,@offline_groups,@failover_groups,$group,$state,$parallel,$onlines,$groups);
    ($prod,$sys)=(@_);

    $rtn=EDR::cmd_sys($sys, "_cmd_hasys -state 2>&1");
    foreach $line (split(/\n/,$rtn)) {
        next if ($line=~/^#/);
        if ($line=~/VCS ERROR/) {
            $msg=Msg::new("HAD is not started on $sys->{sys}", 40, 3561, "$sys->{sys}");
            EDR::push_error_sys($sys,$msg);
            return 0;
        } elsif ($line=~/^(\S+)\s+SysState\s+(\S+)/) {
            push (@systems,"$1") if ($2 ne 'RUNNING');
        }
    }
 
    if (@systems) {
        $systems=join(',',@systems);
        $msg=Msg::new("Checking from $sys->{sys}, HAD is not in running state on the following systems:\n\t$systems", 40, 3562, "$sys->{sys}", "$systems");
        EDR::push_error_sys($sys,$msg);
        $warning=1;
    }

    $rtn=EDR::cmd_sys($sys, "_cmd_hagrp -state -sys $sys->{hostname} 2>&1");
    foreach $line (split(/\n/,$rtn)) {
        next if ($line=~/^#/);
        if ($line=~/^(\S+)\s+State\s+\S+\s+\|(\S+)\|/) {
            $group=$1;
            $state=$2;
            $parallel=EDR::cmd_sys($sys, "_cmd_hagrp -value $group Parallel 2>/dev/null");
            if ($parallel ne '0') {
                push(@offline_groups, $group) if ($state ne 'ONLINE');
            } else {
                # For failover service group
                $onlines=EDR::cmd_sys($sys, "_cmd_hagrp -state $group | _cmd_grep ONLINE | _cmd_wc -l 2>/dev/null");
                if ($onlines != 1) {
                    push(@failover_groups, $group);
                }
            }
        }
    }

    if (@offline_groups) {
        $groups=join("\n\t",@offline_groups);
        $msg=Msg::new("The following service groups are not in online state on $sys->{sys}:\n\t$groups", 40, 3563, "$sys->{sys}", "$groups");
        EDR::push_warning_sys($sys,$msg);
        $warning=1;
    }
 
    if (@failover_groups) {
        $groups=join("\n\t",@failover_groups);
        $msg=Msg::new("Checking from $sys->{sys}, the following Failover service groups are not online on one and only one system:\n\t$groups", 40, 3564, "$sys->{sys}", "$groups");
        EDR::push_warning_sys($sys,$msg);
        $warning=1;
    }
 
    return 0 if($warning);
    return 1;
}

sub register_postchecks_per_system {
    my ($prod,$sequence_id,$name,$desc,$handler);
    $prod=shift;

    $sequence_id=300;
    $name="lltstat";
    $desc=Msg::new("lltstat status", 40, 3565);
    $handler=\&postcheck_lltstat_sys;
    $prod->tsub("register_postcheck_item_per_system",$sequence_id,"$name",$desc,$handler);

    $sequence_id=320;
    $name="lltconfig";
    $desc=Msg::new("lltconfig status", 40, 3566);
    $handler=\&postcheck_lltconfig_sys;
    $prod->tsub("register_postcheck_item_per_system",$sequence_id,"$name",$desc,$handler);

    $sequence_id=330;
    $name="gabconfig";
    $desc=Msg::new("gabconfig ports status", 40, 3247);
    $handler=\&postcheck_gabconfig_sys;
    $prod->tsub("register_postcheck_item_per_system",$sequence_id,"$name",$desc,$handler);

    $sequence_id=335;
    $name="vxfen";
    $desc=Msg::new("vxfen status", 40, 3567);
    $handler=\&postcheck_vxfen_sys;
    $prod->tsub("register_postcheck_item_per_system",$sequence_id,"$name",$desc,$handler);

    $sequence_id=340;
    $name="hastatus";
    $desc=Msg::new("HAD status", 40, 3568);
    $handler=\&postcheck_hastatus_sys;
    $prod->tsub("register_postcheck_item_per_system",$sequence_id,"$name",$desc,$handler);

    return 1;
}

sub register_postchecks_per_cluster {
    my ($prod,$sequence_id,$name,$desc,$handler);
    $prod=shift;

    #$sequence_id=340;
    #$name="hastatus";
    #$desc=Msg::new("HAD status");
    #$handler=\&postcheck_hastatus;
    #$prod->tsub("register_postcheck_item_per_cluster",$sequence_id,"$name",$desc,$handler);

    return 1;
}

#
# VCS add node
#

sub opt_addnode {
    my $vcs = shift;
    my $cpic = Obj::cpic();
    my $prod = $cpic->prod();
    for my $sub (qw(addnode_messages addnode_get_cluster addnode_get_newnode addnode_compare_systems
                    addnode_configure_heartbeat addnode_configure_cluster addnode_start_cluster
                    addnode_poststart addnode_completion))
    {
        if ($prod->can($sub)) {
            $prod->tsub($sub);
        } else {
            $vcs->tsub($sub);
        }
    }    
}

sub web_opt_addnode {
    my $vcs = shift;
    my $cpic = Obj::cpic();
    my $prod = $cpic->prod();
    
    for my $sub (qw(web_addnode_messages web_addnode_get_cluster web_addnode_get_newnode addnode_compare_systems
                    web_addnode_configure_heartbeat addnode_configure_cluster addnode_start_cluster
                    addnode_poststart addnode_completion)) {
        if ($prod->can($sub)) {
            $prod->tsub($sub);
        } else {
            $vcs->tsub($sub);
        }
    }    
}

sub addnode_messages {
    my $prod = shift;
    my $pdfrs=Msg::get("pdfrs");
    my($msg);
    my $cpic = Obj::cpic();
    $prod = $cpic->prod();

    
    # display messages
    $msg = Msg::new("Following are the prerequisites to add a node to the cluster:\n", 40, 2563);
    $msg->print;
    $msg = Msg::new("\t* The cluster to which you want to add the node must have all required $prod->{abbr} $pdfrs installed\n", 40, 2564, "$prod->{abbr}", "$pdfrs");
    $msg->print;
    $msg = Msg::new("\t* New node must have all required $prod->{abbr} $pdfrs installed\n", 40, 2565, "$prod->{abbr}", "$pdfrs");
    $msg->print;
    $msg = Msg::new("\t* $prod->{abbr} must be running on the cluster to which you want to add a node\n", 40, 2566, "$prod->{abbr}");
    $msg->print;
    $msg = Msg::new("\t* There should be no prior running $prod->{abbr} processes on the new node\n", 40, 2567, "$prod->{abbr}");
    $msg->print;
    $msg = Msg::new("\t* New node must have the same $prod->{abbr} version as that of the existing cluster nodes\n", 40, 2568, "$prod->{abbr}");
    $msg->print;
    $msg = Msg::new("\nRefer to the $prod->{name} Installation Guide for more details\n\n", 40, 2569, "$prod->{name}");
    $msg->print;
    Msg::prtc();
}

sub web_addnode_messages {
    my $pdfrs=Msg::get("pdfrs");
    my($msg);
    my $cpic = Obj::cpic();
    my $prod = $cpic->prod();
    my $web = Obj::web();

    $msg = Msg::new("Following are the prerequisites to add a node to the cluster:\\n", 40, 3723)->{msg};
    $msg .= Msg::new("<ul><li>The cluster to which you want to add the node must have all required $prod->{abbr} $pdfrs installed</li>", 40, 3724, "$prod->{abbr}", "$pdfrs")->{msg};
    $msg .= Msg::new("<li>New node must have all required $prod->{abbr} $pdfrs installed</li>", 40, 3725, "$prod->{abbr}", "$pdfrs")->{msg};
    $msg .= Msg::new("<li>$prod->{abbr} must be running on the cluster to which you want to add a node</li>", 40, 3726, "$prod->{abbr}")->{msg};
    $msg .= Msg::new("<li>There should be no prior running $prod->{abbr} processes on the new node</li>", 40, 3727, "$prod->{abbr}")->{msg};
    $msg .= Msg::new("<li>New node must have the same $prod->{abbr} version as that of the existing cluster nodes</li></ul>", 40, 3728, "$prod->{abbr}")->{msg};
    $msg .= Msg::new("\\nRefer to the $prod->{name} Installation Guide for more details", 40, 3729, "$prod->{name}")->{msg};
    
    $web->tsub("web_script_form", "alert", $msg);
}

sub addnode_get_cluster {
    my ($ayn,$clustername,$clusterid,$conf,$done,$had,$llttab,$maincf,$msg,$node,$sys,$sysi,@systems,$syslist);
    my $prod = shift;
    my $edr = Obj::edr();
    my $cfg = Obj::cfg();
    my $cpic = Obj::cpic();  
    my $prod1 = $cpic->prod(); 
 
    $edr->{savelog} = 1;
    # ask for the running cluster by entering one of its nodes
    while(1) {
        Msg::title();
        if (Cfg::opt("responsefile")) {
            $node = ${$cfg->{clustersystems}}[0];
        } else {            
            $msg = Msg::new("Enter one node of the $prod1->{abbr} cluster to which you would like to add one or more new nodes:", 40, 3569, "$prod1->{abbr}");
            $node = $msg->ask();
            $node = EDR::despace($node);
        }
        next if ($edr->tsub("validate_systemnames", $node));
        $sysi = $node;
        $sys = tsub("Sys","new",$sysi); # initialize system objects

        $msg = Msg::new("Checking communication on $sysi", 40, 2554, "$sysi");
        $msg->left;
        if ($edr->tsub("transport_sys", $sys)) {
            Msg::right_done();
        } else {
            Msg::right_failed();
            Msg::n();
            for my $errmsg (@{$sys->{errors}}) {
                Msg::print("$errmsg\n");
            }
            undef($sys->{errors});
            next;            
        }

        $msg=Msg::new("Checking release compatibility on $sysi", 40, 2555, "$sysi");
        $msg->left;
        if ($edr->tsub("rel_padv_sys", $sys)) {
            Msg::right_done();
        } else {
            Msg::right_failed();
            Msg::n();
            for my $errmsg (@{$sys->{errors}}) {
                Msg::print("$errmsg\n");
            }
            undef($sys->{errors});
            next;                        
        }
        
        if (Cfg::opt("responsefile")) {
            @systems = @{$cfg->{clustersystems}};
        } else {            
            # get cluster systems
            $conf = $prod->get_config_sys($sys);
            if ($conf) {
                $clustername = $conf->{clustername};
                @systems = @{$conf->{systems}};
                if ($conf->{onenode}) {
                    # onenode cluster
                    Msg::n();
                    $msg = Msg::new("Cluster $clustername is a single node cluster, and llt and gab is not configured on the node", 40, 2572, "$clustername");
                    $msg->print();
                    $msg = Msg::new("To add a new node to single-node cluster, $clustername, a unique cluster ID is required", 40, 2573, "$clustername");
                    $msg->print();
                    $clusterid = $prod->ask_clusterid($clusterid);
                } else {
                    $clusterid = $conf->{clusterid};
                }
            } else {
                $msg = Msg::new("Can not find valid configuration information on $sysi", 40, 2574, "$sysi");
                $msg->print;
                next;            
            }
                    
            # show verification messages
            Msg::title();
            $msg = Msg::new("Following cluster information detected:\n", 40, 2575);
            $msg->bold;
            $msg = Msg::new("\tCluster Name: $clustername\n", 40, 2557, "$clustername");
            $msg->print;
            $msg = Msg::new("\tCluster ID: $clusterid\n", 40, 2576, "$clusterid");
            $msg->print;
            $syslist = join(" ",@systems);
            $msg = Msg::new("\tSystems: $syslist\n", 40, 2577, "$syslist");
            $msg->print;
            $msg = Msg::new("Is this information correct?", 40, 1102);
            $ayn = $msg->ayny;
            next if ($ayn eq "N");
        }
        
        # check max number in cluster
        if (scalar(@systems) == $prod->{max_number_in_cluster}) {
            $msg = Msg::new("The cluster, $clustername, already has $prod->{max_number_in_cluster} nodes. $prod->{abbr} supports only a maximum of $prod->{max_number_in_cluster} nodes in a cluster.", 40, 2578, "$clustername", "$prod->{max_number_in_cluster}", "$prod->{abbr}", "$prod->{max_number_in_cluster}");
            $msg->print;
            next;
        }
        
        $done = 1;

        # check communication
        for $sysi (@systems) {
            next if ($sysi eq $node); # already check it at the beginning
            $sys = tsub("Sys","new",$sysi); # initialize system objects
            $msg = Msg::new("Checking communication on $sysi", 40, 2554, "$sysi");
            $msg->left;
            if ($edr->tsub("transport_sys", $sys)) {
                Msg::right_done();
            } else {
                Msg::right_failed();
                $done = 0;                
            }
        }
        next unless($done);

        # check vcs runing state
        for $sysi (@systems) {
            $sys = Obj::sys($sysi);
            $had = $sys->proc("had51");
            $msg = Msg::new("Checking $prod->{abbr} running state on $sysi", 40, 2579, "$prod->{abbr}", "$sysi");
            $msg->left;
            if ($had->check_sys($sys,"poststart")) {
                Msg::right_done();
            } else {
                Msg::right_failed();
                $done = 0;
            }
        }
                
        if ($done) {
            unless (Cfg::opt("responsefile")) {
                $cfg->{vcs_clustername} = $clustername;
                $cfg->{vcs_clusterid} = $clusterid;
                $cfg->{clustersystems} = \@systems;                
            }
            sleep 3;
            last;
        } else {
            Msg::n();
            for $sysi (@systems) {
                $sys = Obj::sys($sysi);
                for my $errmsg (@{$sys->{errors}}) {
                    Msg::print("$errmsg\n");
                }
                undef($sys->{errors});
            }
        }
        
    } continue {
        if (Cfg::opt("responsefile")) {
            $msg = Msg::new("Failed to add node(s) using response file", 40, 3570);
            $msg->die();
        }
        $msg = Msg::new("Do you want to enter a node name for another cluster?", 40, 2580);
        $ayn = $msg->aynn;
        Msg::die() if ($ayn eq "N");
    }
}

sub web_addnode_get_cluster {
    my ($ayn,$clustername,$clusterid,$conf,$done,$errmsg,$had,$llttab,$maincf,$msg,$node,@systems);
    
    my $prod = shift;
    my $edr = Obj::edr();
    my $cfg = Obj::cfg();
    my $cpic = Obj::cpic();  
    my $prod1 = $cpic->prod();
    my $web = Obj::web();
 
    $edr->{savelog} = 1;
    # ask for the running cluster by entering one of its nodes
    
    while(1) {
        if (Cfg::opt("responsefile")) {
            $node = ${$cfg->{clustersystems}}[0];
        } else {
            $msg = Msg::new("Enter one node of the cluster to which you would like to add one or more new nodes", 40, 3571);
            $web->tsub("web_script_form", "selectcluster", $msg);
            $node = $web->param("sys");
            $node = EDR::despace($node);
        }

        $web->{stage} = Msg::new("Retrieving cluster information", 40, 3572);
        $web->set_progress_steps(2);
        
        next if ($edr->tsub("validate_systemnames", $node));
        my $sysi = $node;
        my $sys = tsub("Sys", "new", $sysi);        

        $web->display_left(Msg::new("Checking communication on $sysi", 40, 2554, "$sysi"));
        
        if ($edr->tsub("transport_sys", $sys)) {
            $web->display_right();
        } else {
            for $errmsg (@{$sys->{errors}}) {
                $web->addError($errmsg);
            }
            next;            
        }

        $web->display_left(Msg::new("Checking release compatibility on $sysi", 40, 2555, "$sysi"));        
        if ($edr->tsub("rel_padv_sys", $sys)) {
            $web->display_right();
        } else {
            for $errmsg (@{$sys->{errors}}) {
                $web->addError($errmsg);
            }
            next;                        
        }
        
        if (Cfg::opt("responsefile")) {
            @systems = @{$cfg->{clustersystems}};
        } else {            
            # get cluster systems
            $conf = $prod->get_config_sys($sys);
            if ($conf) {
                $clustername = $conf->{clustername};
                @systems = @{$conf->{systems}};
                $clusterid = $conf->{clusterid} unless ($conf->{onenode});
            } else {                
                $web->addError(Msg::new("Can not find valid configuration information on $sysi", 40, 2574, "$sysi")->{msg});
                next;            
            }
                    
            # show verification messages             
            $msg = Msg::new("Following cluster information detected:\n", 40, 2575)->{msg};            
            $msg .= Msg::new("\tCluster Name: $clustername\n", 40, 2557, "$clustername")->{msg};            
            $msg .= Msg::new("\tCluster ID: $clusterid\n", 40, 2576, "$clusterid")->{msg} unless ($conf->{onenode});            
            
            my $syslist = join(" ",@systems);            
            $msg .= Msg::new("\tSystems: $syslist\n", 40, 2577, "$syslist")->{msg};
            $msg = Msg::new($msg);
            
            my $ask_msg = Msg::new("Is this information correct?", 40, 1102);
            
            $ayn = $ask_msg->ayny_web('', '', $msg);   
            next if ($ayn eq "N");
        }
        
        # check max number in cluster
        if (scalar(@systems) == $prod->{max_number_in_cluster}) {
            $msg = Msg::new("The cluster, $clustername, already has $prod->{max_number_in_cluster} nodes. $prod->{abbr} supports only a maximum of $prod->{max_number_in_cluster} nodes in a cluster.", 40, 2578, "$clustername", "$prod->{max_number_in_cluster}", "$prod->{abbr}", "$prod->{max_number_in_cluster}");
            $web->addError($msg->{msg});
            next;
        }
        
        $done = 1;

        # check communication
        for $sysi (@systems) {
            next if ($sysi eq $node); # already check it at the beginning
            $sys = tsub("Sys", "new", $sysi); # initialize system objects            
#            $web->display_left(Msg::new("Checking communication on $sysi"));
            
            if ($edr->tsub("transport_sys", $sys)) {
#                $web->display_right();
            } else {
                for $errmsg (@{$sys->{errors}}) {
                    $web->addError($errmsg);
                }
                $done = 0;                
            }
        }
        next unless($done);

        # check vcs runing state
        for $sysi (@systems) {
            $sys = Obj::sys($sysi);
            $had = $sys->proc("had51");
            
#            $msg = Msg::new("Checking $prod->{abbr} running state on $sysi");
#            $web->display_left($msg);
            
            if ($had->check_sys($sys,"poststart")) {
#                $web->display_right();
            } else {
                $msg = Msg::new("$prod->{abbr} is not in running state on $sysi, the cluster needs to be running when adding node(s)", 40, 3573, "$prod->{abbr}", "$sysi");
                $web->addError($msg->{msg});
            }
        }
                
        if ($done) {
            unless (Cfg::opt("responsefile")) {
                $cfg->{vcs_clustername} = $clustername;
                $cfg->{vcs_clusterid} = $clusterid;
                $cfg->{clustersystems} = \@systems;                
            }
            last;
        }
        
    } continue {
        if (Cfg::opt("responsefile")) {
            $msg = Msg::new("Failed to add node(s) using response file", 40, 3570);
            $msg->die();
        }
    }
}

sub addnode_get_newnode {
    my ($ayn,$done,$had,$msg,$nodelist,@newnodes,$sys,$sysi,$syslist,@allsystems,@t_newnodes);
    my $prod = shift;
    my $edr = Obj::edr();
    my $cfg = Obj::cfg();
    
    # ask for new node to be added
    while (1) {
        @t_newnodes=();
        if (Cfg::opt("responsefile")) {
            @newnodes = @{$cfg->{newnodes}};
        } elsif (defined $cfg->{systems}) {
            @newnodes = @{$cfg->{systems}};
        } else {
            Msg::title();
            $msg = Msg::new("Enter the system names separated by spaces to add to the cluster:", 40, 2581);
            $nodelist = $msg->ask();
            $nodelist = EDR::despace($nodelist);
            @newnodes = split(/\s+/,$nodelist);
        }
        next if ($edr->tsub("validate_systemnames", @newnodes));
        
        # check each new node
        $done = 1;
        for $sysi (@newnodes) {
            $sys = tsub("Sys","new",$sysi); # initialize system objects
            if(EDR::inarr($sysi,@{$cfg->{clustersystems}})) {
                $msg = Msg::new("System $sysi is already a member of the cluster $cfg->{vcs_clustername}", 40, 2582, "$sysi", "$cfg->{vcs_clustername}");
                $msg->print;
                $done = 0;
                next;
            }
            
            $msg = Msg::new("Checking communication on $sysi", 40, 2554, "$sysi");
            $msg->left;
            if ($edr->tsub("transport_sys", $sys)) {
                Msg::right_done();
            } else {
                Msg::right_failed();
                $done = 0;
                next;
            }
            # only check time sync for the sys that passed transport_sys check
            push (@t_newnodes,$sys->{sys});

            $msg=Msg::new("Checking release compatibility on $sysi", 40, 2555, "$sysi");
            $msg->left;
            if ($edr->tsub("rel_padv_sys", $sys)) {
                Msg::right_done();
            } else {
                Msg::right_failed();
                $done = 0;
                next;
            }

            $had = $sys->proc("had51");
            if ($had->check_sys($sys,"poststart")) {
                $msg = Msg::new("$prod->{abbr} is already running on $sysi", 40, 3574, "$prod->{abbr}", "$sysi");
                $msg->print;
                $done = 0;
            }
            
            my $conf = $prod->get_config_sys($sys);
            if ($conf) {
                $msg = Msg::new("The node $sysi is already a member of cluster $conf->{clustername}", 40, 2583, "$sysi", "$conf->{clustername}");
                $msg->print;
                $done = 0;
            }
        }
        
        # check time sync
        push(@allsystems,@t_newnodes,@{$cfg->{clustersystems}});
        $sys = Obj::sys(${$cfg->{clustersystems}}[0]);
        $prod->tsub("check_timesync_sys", $sys, \@allsystems, 5);
        if($sys->{warnings}) {
            Msg::n();
            for my $warningmsg (@{$sys->{warnings}}) {
                Msg::bold("$warningmsg\n");
            }
            undef($sys->{warnings});
        }

        if ($done) {
            unless (Cfg::opt("responsefile")) {
                $syslist = join(" ",@newnodes);
                $msg = Msg::new("Do you want to add the system(s) $syslist to the cluster $cfg->{vcs_clustername}?", 40, 2584, "$syslist", "$cfg->{vcs_clustername}");
                $ayn = $msg->ayny;
                next if ($ayn eq "N");
                $cfg->{newnodes} = \@newnodes;
            }
            last;
        } else {
            Msg::n();
            for $sysi (@newnodes) {
                $sys = Obj::sys($sysi);
                for my $errmsg (@{$sys->{errors}}) {
                    Msg::print("$errmsg\n");
                }
                undef($sys->{errors});
            }            
        }

    } continue {
        if (Cfg::opt("responsefile")) {
            $msg = Msg::new("Failed to add node(s) using response file", 40, 3570);
            $msg->die();
        }
        undef($cfg->{systems});
        $msg = Msg::new("Do you want to enter other node name?", 40, 2585);
        $ayn = $msg->aynn;
        Msg::die() if ($ayn eq "N");
    }
}

sub web_addnode_get_newnode {
    my ($ayn,$done,$errmsg,$had,$msg,$nodelist,@newnodes,$sys,$sysi,$syslist,@allsystems,@t_newnodes);
    my $prod = shift;
    my $edr = Obj::edr();
    my $cfg = Obj::cfg();
    my $web = Obj::web();
    
    # ask for new node to be added
    while (1) {
        @t_newnodes = ();
        if (Cfg::opt("responsefile")) {
            @newnodes = @{$cfg->{newnodes}};
        } else {
            $web->reset_progress();
            $msg = Msg::new("Enter the system names separated by spaces to add to the cluster", 40, 3575);
            $web->tsub("web_script_form", "selectsystems", $msg);

            $nodelist = $web->param("sys");
            $nodelist = EDR::despace($nodelist);
            @newnodes = split(/\s+/,$nodelist);
        }
        $web->{stage} = Msg::new("Checking new cluster node", 40, 3576);
        $web->set_progress_steps(2);
        
        next if ($edr->tsub("validate_systemnames", @newnodes));
        
        # check each new node
        $done = 1;
        for $sysi (@newnodes) {
            $sys = tsub("Sys","new",$sysi); # initialize system objects
            if(EDR::inarr($sysi,@{$cfg->{clustersystems}})) {
                $msg = Msg::new("System $sysi is already a member of the cluster $cfg->{vcs_clustername}", 40, 2582, "$sysi", "$cfg->{vcs_clustername}");
                $web->addError($msg->{msg});
                
                $done = 0;                
                next;
            }
            
            $msg = Msg::new("Checking communication on $sysi", 40, 2554, "$sysi");
            $web->display_left($msg);
            if ($edr->tsub("transport_sys", $sys)) {
                $web->display_right();
            } else {
                for $errmsg (@{$sys->{errors}}) {
                    $web->addError($errmsg);
                }
                undef($sys->{errors});
                $done = 0;
                next;
            }
            # only check time sync for the sys that passed transport_sys check
            push (@t_newnodes, $sys->{sys});

            $msg=Msg::new("Checking release compatibility on $sysi", 40, 2555, "$sysi");
            $web->display_left($msg);
            if ($edr->tsub("rel_padv_sys", $sys)) {
                $web->display_right();
            } else {
                for $errmsg (@{$sys->{errors}}) {
                    $web->addError($errmsg);
                }
                undef($sys->{errors});
                $done = 0;
                next;
            }

            $had = $sys->proc("had51");
            if ($had->check_sys($sys,"poststart")) {
                $msg = Msg::new("$prod->{abbr} is already running on $sysi", 40, 3574, "$prod->{abbr}", "$sysi");
                $web->addError($msg->{msg});
                $done = 0;
            }
            
            my $conf = $prod->get_config_sys($sys);
            if ($conf) {
                $msg = Msg::new("The node $sysi is already a member of cluster $conf->{clustername}", 40, 2583, "$sysi", "$conf->{clustername}");
                $web->addError($msg->{msg});                
                $done = 0;
            }
        }

        push(@allsystems,@t_newnodes,@{$cfg->{clustersystems}});
        $sys = Obj::sys(${$cfg->{clustersystems}}[0]);
        $prod->tsub("check_timesync_sys", $sys, \@allsystems, 5);
        if($sys->{warnings}) {
            $web->tsub("web_script_form", "alert", join("\n",@{$sys->{warnings}}));
            undef($sys->{warnings});
        } 

        if ($done) {
            unless (Cfg::opt("responsefile")) {
                $syslist = join(" ",@newnodes);
                $msg = Msg::new("Do you want to add the system(s) $syslist to the cluster $cfg->{vcs_clustername}?", 40, 2584, "$syslist", "$cfg->{vcs_clustername}");
                $ayn = $msg->aynn_web;                
                next if ($ayn eq "N");
                $cfg->{newnodes} = \@newnodes;
            }
            last;
        }

    } continue {
        if (Cfg::opt("responsefile")) {
            $msg = Msg::new("Failed to add node(s) using response file", 40, 3570);
            $msg->die();
        }
        undef($cfg->{systems});
    }
}

sub addnode_compare_systems {
    my ($errmsg,$minpkgs,@misspkgs,$n,$msg,$pkgi,$pkg,$prodi,$sysi,$sys);
    my ($clus_prod,$clus_lic,$clus_ver,$failed,$installed_prodi,$node_prod,$node_lic,$node_ver);
    my $vcs = shift;
    my $cfg = Obj::cfg();
    my $cpic = Obj::cpic();
    my $edr = Obj::edr();
    my $prod = $cpic->prod();
    my $padv = $cpic->{padv};
    my @prodlist = qw(SFRAC51 SFCFSRAC51 SVS51 SFCFSHA51 SFCFS51 SFHA51 VCS51);

    $n = 0;
    # check installed product and version on first node of the cluster
    $sys = Obj::sys(${$cfg->{clustersystems}}[0]);
    # initialize patches version on Solaris, original called in $edr->rel_padv_sys 
    $edr->patches_sys($sys) unless ($sys->{patchvers});
    $msg = Msg::new("Checking installed product on cluster $cfg->{vcs_clustername}", 40, 2586, "$cfg->{vcs_clustername}");
    $msg->left();
    for $prodi (@prodlist) {
        $clus_prod = $Obj::pool{"Prod\::$prodi\::$padv"};
        next unless (defined($clus_prod));
        $clus_ver = $clus_prod->tsub("version_sys",$sys,1);
        if ($clus_ver) {
            $clus_lic = $cpic->prod_licensed_sys($sys,$prodi);
            if ($clus_lic) {
                Msg::right($clus_prod->{abbr}.$clus_ver);
                $installed_prodi = $prodi;
                last;
            }
        }
    }
    if (!$clus_ver) {
        Msg::right_failed();
        $msg = Msg::new("$prod->{abbr} is not installed properly on $sys->{sys}", 40, 2960, "$prod->{abbr}", "$sys->{sys}");
        $msg->die();
    }
    if (!$clus_lic) {
        Msg::right_failed();
        $msg = Msg::new("$clus_prod->{abbr} is installed on $sys->{sys}, but it does not have valid license", 40, 3730, "$clus_prod->{abbr}", "$sys->{sys}");
        $msg->die();
    }
    if ($clus_prod->tsub("configure_alternate_prod")) {
        $cpic->{prod} = $installed_prodi;
        $prod = $cpic->prod();
    }
    if ($clus_prod->{prod} ne $prod->{prod}) {
        my $upi = lc($clus_prod->{prod});
        $upi = "sfcfs" if ($upi eq "sfcfsha"); # no installsfcfsha script
        $msg = Msg::new("$clus_prod->{abbr} is installed on cluster $cfg->{vcs_clustername}, please use install$upi -addnode to add new node(s)", 40, 2962, "$clus_prod->{abbr}", "$cfg->{vcs_clustername}", "$upi");
        $msg->die();
    }
    
    Cfg::set_opt("installminpkgs");
    $minpkgs = $prod->getpkgs(0);
    $failed = 0;
    
    for $sysi (@{$cfg->{newnodes}}) {
        $sys = Obj::sys($sysi);
        # check installed product and version on new node
        $msg = Msg::new("Checking installed product on $sysi", 40, 2589, "$sysi");
        $msg->left;
        for $prodi (@prodlist) {
            $node_prod = $Obj::pool{"Prod\::$prodi\::$padv"};
            next unless (defined($node_prod));
            $node_ver = $node_prod->tsub("version_sys",$sys,1);
            if ($node_ver) {
                $node_lic = $cpic->prod_licensed_sys($sys,$prodi);
                if ($node_lic) {
                    Msg::right($node_prod->{prod}.$node_ver);
                    last;
                }
            }
        }
        if (!$node_ver) {
            Msg::right_failed();
            $errmsg .= Msg::new("$prod->{abbr} is not installed completely on $sysi. At least the minimal package set for $prod->{prod} should be installed prior to adding this node to a running cluster.\n", 40, 3731, "$prod->{abbr}", "$sysi", "$prod->{prod}")->{msg};
            $failed = 1;
            next;
        }
        if (!$node_lic) {
            Msg::right_failed();
            $errmsg .= Msg::new("$node_prod->{abbr} is installed on $sysi, but does not have valid license.\n", 40, 3732, "$node_prod->{abbr}", "$sysi")->{msg};
            $failed = 1;
            next;            
        }
        if ($node_prod->{prod} ne $prod->{prod}) {
            $errmsg .= Msg::new("$clus_prod->{abbr} is installed on cluster $cfg->{vcs_clustername}, whereas, $node_prod->{abbr} is installed on $sysi. Adding a node between different products is not supported.\n", 40, 3733, "$clus_prod->{abbr}", "$cfg->{vcs_clustername}", "$node_prod->{abbr}", "$sysi")->{msg};
            $failed = 1;
            next;
        }

        # make version compare between cluster and each new node
        if (EDR::compvers($clus_ver, $node_ver) == 0) {
            $msg = Msg::new("Checking installed packages on $sysi", 40, 2592, "$sysi");
            $msg->left;
            # packages integrality validation, at least the minimial package set required.
            for $pkgi (@{$minpkgs}) {
                $pkg = $sys->pkg($pkgi);
                push(@misspkgs, $pkgi) unless ($pkg->version_sys($sys));
            }
            if(scalar(@misspkgs) > 0) {
                Msg::right_failed();
                Msg::n();
                $msg = Msg::new("At a minimum, the following packages should be installed prior to adding $sysi to a running cluster:", 40, 3734, "$sysi");
                $msg->bold();
                $n=$cpic->tsub("list_packages",$sys,$n,\@misspkgs);
                $errmsg .= $msg->{msg};
                $errmsg .= "\n";
                $errmsg .= join(" ", @misspkgs);
                $errmsg .= "\n";
                $failed = 1;
                @misspkgs = ();
            } else {
                Msg::right_done();
            }
            
        } elsif (EDR::compvers($clus_ver, $node_ver) == 1) {
            $errmsg .= Msg::new("$prod->{abbr} installed on $sysi has a lower version than the members in cluster $cfg->{vcs_clustername}\n", 40, 3735, "$prod->{abbr}", "$sysi", "$cfg->{vcs_clustername}")->{msg};
            $failed = 1;
        } elsif (EDR::compvers($clus_ver, $node_ver) == 2) {
            $errmsg .= Msg::new("$prod->{abbr} installed on $sysi has a higher version than the members in cluster $cfg->{vcs_clustername}\n", 40, 3736, "$prod->{abbr}", "$sysi", "$cfg->{vcs_clustername}")->{msg};
            $failed = 1;
        }
    }
    if ($failed) {
        $msg = Msg::new($errmsg);
        $msg->die();
    }
    
    Cfg::unset_opt("installminpkgs");
    return 1;
}

# create llttab on new node
sub addnode_configure_heartbeat {
    my ($address,$ayn,$common_speed,$clus_conf,$diff,$key,$lmsg,$msg,$n,$netmask,$nic,$onenode,$port,$rhbn);
    my ($sys,$sysi);
    my $prod = shift;
    my $edr = Obj::edr();
    my $cfg = Obj::cfg();
    
    # reset system list in $cfg->{systems}
    $cfg->{systems} = [];
    my $sys1 = Obj::sys(${$cfg->{clustersystems}}[0]);
    $clus_conf = $prod->get_config_sys($sys1);
    if ($clus_conf->{onenode}) {
        # without llt/gab, ask for heartbeat confiuration on this node as well
        push (@{$cfg->{systems}}, @{$cfg->{clustersystems}});
        $onenode = 1;
    }
    push(@{$cfg->{systems}}, @{$cfg->{newnodes}});
    $edr->{systems}=$edr->tsub("init_sys_objects");
    
    while (1) {
        last if (Cfg::opt("responsefile"));
        Msg::title();
        if ($onenode) {
            $rhbn = $prod->tsub("hb_config_option");
        } else {
            if ($clus_conf->{lltoverudp}) {
                $cfg->{lltoverudp} = 1;
            }
            $rhbn=$prod->ask_hbnics_addnode($clus_conf, $sys1->{sys});
        }
        next if ($rhbn eq $edr->{msg}{back});
        if (!$cfg->{autocfgllt}) {
            # check link speed on new nodes
            $diff = $prod->check_link_speed($edr->{systems},$rhbn);
            unless ($onenode || $diff) {
                # check link speed on original cluster
                $diff = $prod->check_link_speed($cfg->{clustersystems},$clus_conf);
                unless ($diff) {
                    $sysi = ${$cfg->{newnodes}}[0];
                    $sys = Obj::sys($sysi);
                    $diff = 1 if ($sys1->{nics_comm_speed} ne $sys->{nics_comm_speed});                    
                }
            }
            if ($diff == 1) {
                Msg::n();
                $msg = Msg::new("The private NICs do not have same media speed.\n", 40, 2958);
                $msg->warning();
                $msg = Msg::new("It is recommended that the media speed be same for all the private NICs. Without this, LLT may not function properly. Consult your Operating System manual for information on how to set the Media Speed.\n", 40, 2381);
                $msg->bold;
            }                    
            if ($diff == 2) {
                Msg::n();
                $msg = Msg::new("$edr->{script} can not detect media speed for the selected private NICs properly. Consult your Operating System manual for information on how to set the Media Speed.\n", 40, 2959, "$edr->{script}");
                $msg->warning();
            }
            if ($diff) {    
                $msg = Msg::new("Do you want to continue with current heartbeat configuration?", 40, 2382);
                $ayn = $msg->ayny();
                next if ($ayn ne "Y");
            }
        }
        Msg::title();
        for $sys(@{$edr->{systems}}) {
            $sysi=$sys->{sys};
            $lmsg="";
            for $n (1..$prod->num_hbnics($rhbn)) {
                $key="lltlink$n";
                next unless ($rhbn->{$key}{$sysi});
                $lmsg.="\n\t\tlink$n=$rhbn->{$key}{$sysi}";
                if ($cfg->{lltoverudp}) {
                    $address=$rhbn->{"udplink$n"."_address"}{$sysi};
                    $netmask=$rhbn->{"udplink$n"."_netmask"}{$sysi};
                    $port=$rhbn->{"udplink$n"."_port"}{$sysi};
                    if (ip_is_ipv6($address)) {
                        $lmsg.=" over UDP6\n\t\t  ip $address prefix $netmask port $port";
                    } else {
                        $lmsg.=" over UDP\n\t\t  ip $address netmask $netmask port $port";
                    }                    
                }
            }
            $msg=Msg::new("\tPrivate Heartbeat NICs for $sysi: $lmsg", 40, 2386, "$sysi", "$lmsg");
            $msg->print;
            if ($prod->num_lopri_hbnics($rhbn) > 0) {
                $lmsg = "";
                for $n (1..$prod->num_lopri_hbnics($rhbn)) {
                    $key="lltlinklowpri$n";
                    $lmsg.="\n\t\tlink-lowpri$n=$rhbn->{$key}{$sysi}";
                    if ($cfg->{lltoverudp}) {
                        $address=$rhbn->{"udplinklowpri$n"."_address"}{$sysi};
                        $netmask=$rhbn->{"udplinklowpri$n"."_netmask"}{$sysi};
                        $port=$rhbn->{"udplinklowpri$n"."_port"}{$sysi};
                        if (ip_is_ipv6($address)) {
                            $lmsg.=" over UDP6\n\t\t  ip $address prefix $netmask port $port";
                        } else {
                            $lmsg.=" over UDP\n\t\t  ip $address netmask $netmask port $port";                            
                        }
                    }
                }
                if ($prod->num_lopri_hbnics($rhbn) > 1) {
                    $msg=Msg::new("\tLow-Priority Heartbeat NICs for $sysi: $lmsg", 40, 3412, "$sysi", "$lmsg");
                } else {
                    $msg=Msg::new("\tLow-Priority Heartbeat NIC for $sysi: $lmsg", 40, 3413, "$sysi", "$lmsg");
                }
                $msg->print;           
            }
        }

        Msg::n();
        $msg = Msg::new("Is this information correct?", 40, 1102);
        $ayn = $msg->ayny();
        last if ($ayn eq "Y");
    }

    delete($cfg->{autocfgllt});
    $prod->addnode_update_llttab($rhbn,$onenode);
    return 1;
}

sub addnode_update_llttab {
    my ($prod,$rhbn,$onenode) = @_;
    my $cfg = Obj::cfg();
    my $edr = Obj::edr();
    my ($conf,$n,$sys,$sysi);
    
    if ($cfg->{lltoverudp}) {
        $prod->set_hb_nics($rhbn,$edr->{systems});
        unless ($onenode) {
            $sys = Obj::sys(${$cfg->{clustersystems}}[0]);
            $conf = $prod->get_config_sys($sys);
            $prod->set_hb_nics($conf,$cfg->{clustersystems});
        }
        
        $cfg->{systems} = [];
        push (@{$cfg->{systems}}, @{$cfg->{clustersystems}});
        push(@{$cfg->{systems}}, @{$cfg->{newnodes}});
        $edr->{systems}=$edr->tsub("init_sys_objects");
        $prod->update_llttab();
        for $sys(@{$edr->{systems}}) {
            $sysi = $sys->{sys};
            EDR::copy_sys($sys, "$edr->{tmpdir}/llttab.$sysi",$prod->{llttab});
        }
    } else {
        $prod->set_hb_nics($rhbn,$edr->{systems}) unless (Cfg::opt("responsefile"));
        $prod->update_llttab();
        for $sys(@{$edr->{systems}}) {
            $sysi = $sys->{sys};
            EDR::copy_sys($sys, "$edr->{tmpdir}/llttab.$sysi",$prod->{llttab});
        }
    }
}

sub web_addnode_configure_heartbeat {
    my ($answer,$msg,$n,$onenode,$rtvalue,$sys,$sysi,$unique);
    my $prod = shift;
    my $web = Obj::web();
    my $edr = Obj::edr();
    my $cpic = Obj::cpic();
    my $cfg = Obj::cfg();
    my ($done);

    # reset system list in $cfg->{systems}
    $cfg->{systems} = [];
    my $sys1 = Obj::sys(${$cfg->{clustersystems}}[0]);
    my $clus_conf = $prod->get_config_sys($sys1);
    if ($clus_conf->{onenode}) {
        # without llt/gab, ask for heartbeat confiuration on this node as well
        push (@{$cfg->{systems}}, @{$cfg->{clustersystems}});
        $onenode = 1;
    }
    push(@{$cfg->{systems}}, @{$cfg->{newnodes}});
    $edr->{systems}=$edr->tsub("init_sys_objects");

    while (!$done) {
        if ($onenode) {
            $rtvalue=$web->tsub("web_script_form", "vcs_select_cluster");
            $unique = $web->{submit}{uniqueNicsPerSys};
        }
        if ($cfg->{autocfgllt}) {
            if (!$prod->tsub("web_autocfg_hbnics")) {
                $cfg->{autocfgllt} = 0;
                next;
            }
        } else {
            for $sys (@{$edr->{systems}}) {
                my $padv = $sys->padv;
                my $nics = $padv->tsub("systemnics_sys", $sys);
                $rtvalue = $web->tsub("web_script_form", "vcs_select_heartbeats", "addnode", $nics, $sys);
                last if ($rtvalue eq "back");            
                if ($sys->system1() && ($edr->nsystems() > 1) && !$unique && !$onenode) {
                    $msg = Msg::new("Are you using the same NICs for private heartbeat links on all systems?", 40, 2430);
                    $answer = $msg->ayny_web();
                    $unique = ($answer eq "Y") ? 0 : 1;
                    $web->{uniqueNicsPerSys} = $unique;
                }
                last unless $unique;
            }
            next if($rtvalue eq "back");
            # check link speed on new nodes
            my $diff = $prod->check_link_speed($edr->{systems},$web->{rhbn});
            unless ($onenode || $diff) {
                # check link speed on original cluster
                $diff = $prod->check_link_speed($cfg->{clustersystems},$clus_conf);
                unless ($diff) {
                    $sysi = ${$cfg->{newnodes}}[0];
                    $sys = Obj::sys($sysi);
                    $diff = 1 if ($sys1->{nics_comm_speed} ne $sys->{nics_comm_speed});                
                }
            }
            next if (!$prod->web_check_link_speed($diff));
        }           
        $done = 1;
        # delete this hash key so that it do not show in responsefile
        delete $cfg->{autocfgllt};        
    }
    
    $prod->addnode_update_llttab($web->{rhbn},$onenode);
    return 1;
}

sub addnode_configure_cluster {
    my ($addr,$gabtab,$id,$llthosts,$lltsys,$msg,$n,$nic,$nsysi,$firstnode,$sys,$sysi,$system);
    my $prod = shift;
    my $edr = Obj::edr();
    my $cfg = Obj::cfg();
    
    $firstnode = Obj::sys(${$cfg->{clustersystems}}[0]);
    $n = scalar(@{$cfg->{newnodes}}) + scalar(@{$cfg->{clustersystems}});
    # create gabtab
    $gabtab = "/sbin/gabconfig -c -n$n\n";
    EDR::writefile($gabtab,"$edr->{tmpdir}/gabtab");
    # create llthosts
    # new node should append to the llthosts, this will keep the sequence of old nodes
    if (EDR::file_sys($firstnode,$prod->{llthosts})) {
        $llthosts = EDR::cmd_sys($firstnode,"_cmd_cat $prod->{llthosts}");
        $llthosts.="\n" if ($llthosts);
        $n = scalar(@{$cfg->{clustersystems}});
    } else {
        $n = 0;
    }
    for $sysi (@{$cfg->{clustersystems}}, @{$cfg->{newnodes}}) {
        next if ($llthosts =~ /$sysi/); # sysi already exist in /etc/llthosts
        $lltsys = transform_system_name($sysi);
        next if ($llthosts =~ /$lltsys/); # hostname already exist in /etc/llthosts            
        $llthosts.="$n $lltsys\n";
        $n++;
    }
    EDR::writefile($llthosts,"$edr->{tmpdir}/llthosts");
    
    for $sysi (@{$cfg->{clustersystems}}, @{$cfg->{newnodes}}) {
        $sys = Obj::sys($sysi);
        # copy updated gabtab to cluster members
        if (EDR::file_sys($sys,$prod->{gabtab})) {
            EDR::cmd_sys($sys,"_cmd_mv $prod->{gabtab} $prod->{gabtab}.prev");
        }
        EDR::copy_sys($sys, "$edr->{tmpdir}/gabtab",$prod->{gabtab});
        # copy updated llthosts to cluster members
        if (EDR::file_sys($sys,$prod->{llthosts})) {
            EDR::cmd_sys($sys,"_cmd_mv $prod->{llthosts} $prod->{llthosts}.prev");
        }
        EDR::copy_sys($sys, "$edr->{tmpdir}/llthosts",$prod->{llthosts});
    }
    
    if ($cfg->{lltoverudp}) {
        # add llt links to original cluster members
        $id = scalar(@{$cfg->{clustersystems}});
        for $nsysi (@{$cfg->{newnodes}}) {
            for $sysi (@{$cfg->{clustersystems}}) {
                $sys = Obj::sys($sysi);
                for $n (1..$prod->num_hbnics_cfg($cfg)) {
                    if ($cfg->{"vcs_lltlink$n"}{$nsysi}) {
                        $nic = $cfg->{"vcs_lltlink$n"}{$sysi};
                        $addr = $cfg->{"vcs_udplink$n"."_address"}{$nsysi};
                        EDR::cmd_sys($sys,"_cmd_lltconfig -a set $id $nic $addr");
                    }
                }
                for $n (1..$prod->num_lopri_hbnics_cfg($cfg)) {
                    $nic = $cfg->{"vcs_lltlinklowpri$n"}{$sysi};
                    $addr = $cfg->{"vcs_udplinklowpri$n"."_address"}{$nsysi};
                    EDR::cmd_sys($sys,"_cmd_lltconfig -a set $id $nic $addr");
                }
            }
            $id++;
        }
    }
    
    for $sysi (@{$cfg->{newnodes}}) {
        $sys = Obj::sys($sysi);
        # create /etc/VRTSvcs/conf/sysname on new added nodes
        $system = transform_system_name($sysi);
        EDR::writefile("$system\n","$edr->{tmpdir}/sysname.$sysi");
        EDR::copy_sys($sys, "$edr->{tmpdir}/sysname.$sysi","$prod->{confdir}/sysname");
        
        # copy uuid from cluster to new node
        if ($prod->check_uuidconfig_pl()) {
            $prod->copy_uuid($firstnode->{sys},$sys->{sys});
        } else {
            $msg = Msg::new("Cannot find uuidconfig.pl for UUID configuration. Manually create UUID before starting VCS", 40, 2434);
            $msg->warning();
        }
        # link install: enable vcs
        $cfg->{vcs_allowcomms} = 1;
        $prod->vcs_enable_sys($sys);
        
        # on AIX, configure DLPI diriver
        $prod->tsub("update_pseconf_sys", $sys) if $prod->can("update_pseconf_sys");
    }
    
    $prod->addnode_config_CSSG();
    
    return 1;
}

sub addnode_start_cluster {
    my ($failures,$firstnode,$had,$msg,$n,$onenode,$pids,$proci,$proc,$sys,$sysi,$system,@syslist);
    my $prod = shift;
    my $edr = Obj::edr();
    my $cpic = Obj::cpic();
    my $cfg = Obj::cfg();
    my $web = Obj::web();
    
    Msg::title();
    $web->{stage} = Msg::new("Stoping $prod->{abbr}", 40, 3578, "$prod->{abbr}");
    $web->{stage}->log();
        
    # stop 'had' on single node cluster
    if (scalar(@{$cfg->{clustersystems}}) == 1) {
        # one node cluster
        $sysi = ${$cfg->{clustersystems}}[0];
        $sys = Obj::sys($sysi);
        $pids = EDR::pids_sys($sys,"bin/had -onenode");
        if (@$pids) {
            $onenode = 1;
            $web->tsub("web_script_form", "stopprocess"); 
            $web->set_progress_steps(1);
            $had = $sys->proc("had51");
            $msg=Msg::new("Stopping $had->{proc} on $sysi", 40, 2600, "$had->{proc}", "$sysi");
            $web->display_left($msg);
            if ($cpic->tsub("proc_stop_sys", $sys, $had)) {
                CPIC::proc_stop_passed_sys($sys, $had);
                $web->display_right();
            } else {
                CPIC::proc_stop_failed_sys($sys, $had);
                $web->display_right("failed");
            }
            
            # Linux: remove ONENODE=yes in /etc/sysconfig/vcs
            # SunOS: remove manifest for system/vcs-onenode
            $prod->set_onenode_cluster_sys($sys,0) if ($prod->can("set_onenode_cluster_sys"));
        }
    }
    if ($onenode) {
        # llt/gab do not start for single node cluster, need to start it
        push(@syslist, @{$cfg->{clustersystems}});
        # enable llt/gab/vxfen for previous onenode cluster
        $prod->vcs_enable_sys($sys);
    }
    Msg::title();
    $web->{stage} = Msg::new("Starting $prod->{abbr}", 40, 3579, "$prod->{abbr}");
    $web->{stage}->log();
    # start llt/gab on new added node
    push(@syslist, @{$cfg->{newnodes}});
    $web->tsub("web_script_form", "startprocess", 1); 
    for $proci qw(llt51 gab51) {
        for $sysi (@syslist) {
            $sys = Obj::sys($sysi);
            $proc = $sys->proc($proci);
            $msg=Msg::new("Starting $proc->{proc} on $sysi", 40, 1259, "$proc->{proc}", "$sysi");
            if ($cpic->tsub("proc_start_sys", $sys, $proc)) {
                CPIC::proc_start_passed_sys($sys, $proc);
                $web->display_status($msg);
            } else {
                CPIC::proc_start_failed_sys($sys, $proc);
                $web->display_status($msg,1);
            }
        }
    }
    
    # todo: checking gab port seems to be a more precise way, added later
    $failures=$cpic->failures("startfailmsg");
    unless ($#$failures < 0) {
        Msg::n();
        $msg = Msg::new("LLT and GAB startup did not complete successfully", 40, 3580);
        $msg->bold();
        $msg = Msg::new("Please check your heartbeat link configurations", 40, 3581);
        $msg->die();
    }
    
    # restart vxfen/had for single node cluster
    if ($onenode) {
        Msg::n();
        $msg = Msg::new("Restarting cluster $cfg->{vcs_clustername}", 40, 2601, "$cfg->{vcs_clustername}");
        $msg->bold;
        $sysi = ${$cfg->{clustersystems}}[0];
        $sys = Obj::sys($sysi);
        for $proci qw(vxfen51 had51) {
            # do not start vxfen for VCS/SFHA
            next if (($proci eq "vxfen51") && ($cpic->{prod} =~ /(VCS|SFHA)/));
            $proc = $sys->proc($proci);
            $msg=Msg::new("Starting $proc->{proc} on $sysi", 40, 1259, "$proc->{proc}", "$sysi");
            if ($cpic->tsub("proc_start_sys", $sys, $proc)) {
                CPIC::proc_start_passed_sys($sys, $proc);
                $web->display_status($msg);
            } else {
                CPIC::proc_start_failed_sys($sys, $proc);
                $web->display_status($msg,1);
            }
        }
    }
    
    # check vcs status on first node of cluster
    $firstnode = Obj::sys(${$cfg->{clustersystems}}[0]);
    $had = $firstnode->proc("had51");
    unless ($had->check_sys($firstnode, "poststart")) {
        $msg = Msg::new("Cluster $cfg->{vcs_clustername} must be active before adding a node", 40, 3582, "$cfg->{vcs_clustername}");
        EDR::push_error_sys($firstnode,$msg);
        $msg->die();    
    }
    
    # reset system list in $cfg->{systems}
    $cfg->{systems} = [];
    push(@{$cfg->{systems}}, @{$cfg->{clustersystems}});
    push(@{$cfg->{systems}}, @{$cfg->{newnodes}});
    $cpic->{systems}=$edr->tsub("init_sys_objects");
    
    $prod->haconf_makerw();
    
    # update main.cf
    for $sysi (@{$cfg->{newnodes}}) {
        $system = transform_system_name($sysi);
        EDR::cmd_sys($firstnode,"$prod->{bindir}/hasys -add $system");
    }
    # configure vxss
    $prod->addnode_configure_vxss();
    
    $prod->addnode_update_CSSG();
    
    $prod->haconf_dumpmakero();
    
    $prod->addnode_configure_fencing();
    
    # start 'had' on new nodes
    for $sysi (@{$cfg->{newnodes}}) {
        $sys = Obj::sys($sysi);
        # reset onenode cluster mode to false on new node to remove a possible previous old config
        $prod->set_onenode_cluster_sys($sys,0) if ($prod->can("set_onenode_cluster_sys"));
        EDR::copy_file($firstnode,$sys,$prod->{maincf});
        $proc = $sys->proc("had51");
        $msg=Msg::new("Starting $proc->{proc} on $sysi", 40, 1259, "$proc->{proc}", "$sysi");
        if ($cpic->tsub("proc_start_sys", $sys, $proc)) {
            CPIC::proc_start_passed_sys($sys, $proc);
            $web->display_status($msg);
        } else {
            CPIC::proc_start_failed_sys($sys, $proc);
            $web->display_status($msg,1);
        }         
    }
    $prod->tsub("poststart_errors",$cpic) if ($cpic->sys_error); 
    EDR::reset_errors_warnings();
    return 1;
}

sub addnode_configure_vxss {
    my ($at,$cred,$mode,$firstnode,$n,$sys,$sysi,$system);
    my $prod = shift;
    my $cpic = Obj::cpic();
    my $cfg = Obj::cfg();
    
    $firstnode = Obj::sys(${$cfg->{clustersystems}}[0]);
    $mode = EDR::cmd_sys($firstnode,"$prod->{bindir}/haclus -value SecureClus");
    # no secure cluster
    return 0 if ($mode ne "1");
    # start AB, establish trush, create pd/principal on new node
    $prod->tsub("check_rootbroker");
    $prod->tsub("create_vxsscredentials");

    # modify VxSS service group
    $n = $#{$cfg->{clustersystems}};
    for $sysi (@{$cfg->{newnodes}}) {
        $n++;
        $sys = Obj::sys($sysi);
        $system = transform_system_name($sysi);
        EDR::cmd_sys($sys, "_cmd_touch $prod->{secure}");
        EDR::cmd_sys($firstnode, "$prod->{bindir}/hagrp -modify VxSS SystemList -add $system $n");
        EDR::cmd_sys($firstnode, "$prod->{bindir}/hagrp -modify VxSS AutoStartList -add $system");
        EDR::cmd_sys($firstnode, "$prod->{bindir}/hauser -add root\@$sys->{sys}$prod->{rootfqdn} -priv Administrator");
    }
    return 1;
}

sub addnode_configure_fencing {
    my($firstnode,$msg,$sys,$sysi,$proc,$proci,$vxfenmode);
    my($ret,$security,$cps_aref,$ports_href,$conf,$uuid,$node_cnt);
    my ($vxfen_mechanism,$vxfendg,$cpsab,$cpsabport,$service_enabled,$vxfen);
    my $prod = shift;
    my $cpic = Obj::cpic();
    my $cfg = Obj::cfg();
    my $vxfen_pkg = Obj::pkg("VRTSvxfen51",$prod->{padv});
    
    $firstnode = Obj::sys(${$cfg->{clustersystems}}[0]);
    $service_enabled = 1;
    $vxfen = $firstnode->proc("vxfen51");
    if ($vxfen->check_service_sys($firstnode)==0) {
        $service_enabled = 0;
    }
    if ($service_enabled) {
        if (EDR::file_sys($firstnode,$prod->{vxfenmode})) {
            $conf = EDR::cmd_sys($firstnode,"_cmd_cat $prod->{vxfenmode} 2>/dev/null");
            if ($conf =~ /vxfen_mode\s*=\s*(\w*)/) {
                $vxfenmode = $1;
                Msg::log("Fencing works in $vxfenmode mode for cluster $cfg->{vcs_clustername}");
            } else {
                $msg = Msg::new("Could not get vxfen mode from first node, $firstnode, of the cluster, $cfg->{vcs_clustername}. Fencing will not be started on the newly added node", 40, 3583, "$firstnode", "$cfg->{vcs_clustername}");
                $msg->print();
                return;
            }
        } else {
            # only vxfendg, vxfenmode=scsi3
            if (EDR::file_sys($firstnode, $prod->{vxfendg})) {
                $vxfenmode = "scsi3";
            } else {
                # no vxfenmode and vxfendg, do not start vxfen
                return;
            }
        }
   
        $vxfen_mechanism = ""; 
        $vxfendg = "";
        if ($vxfenmode eq "customized") {
            if ($conf =~ /vxfen_mechanism\s*=\s*cps\s*/) {
                $vxfen_mechanism = "cps";
                Msg::log("Fencing works in $vxfenmode mode with vxfen_mechanism=$vxfen_mechanism"); 
                ($ret, $security, $cps_aref, $ports_href) = $vxfen_pkg->addnode_get_cp_client_conf(
                    $firstnode, $prod->{vxfenmode});
                if ($ret) { 
                    $vxfenmode = "disabled";
                    $msg = Msg::new("Failed to read CP Client configuration from $prod->{vxfenmode} on first node, $firstnode, of cluster, $cfg->{vcs_clustername}. Fencing will be started in disabled mode on the newly added node.", 40, 2605, "$prod->{vxfenmode}", "$firstnode", "$cfg->{vcs_clustername}");
                    $msg->print();
                } else {
                    my $cnt = 1;
                    my %ports = %$ports_href;
                    Msg::log("Existing CP client side configuration"); 
                    Msg::log("security=$security"); 
                    foreach $sys(@$cps_aref) {  
                        Msg::log("cp$cnt=$sys->{sys}:$ports{$sys}");
                        $cnt++;
                    }
                    $uuid = $prod->addnode_get_cluster_uuid();
                    if ($uuid == -1) {
                        $vxfenmode = "disabled";  
                        $msg = Msg::new("Failed to get the UUID for the cluster, $cfg->{vcs_clustername}. CP client configuration cannot be completed without UUID. Fencing will be started in disabled mode on the newly added node.", 40, 2606, "$cfg->{vcs_clustername}");
                        $msg->print;
                    } else { 
                        Msg::log("Cluster uuid=$uuid");
                    }

                    if ($security) {
                        ($cpsab, $cpsabport) = $vxfen_pkg->addnode_get_cpsab_info();
                    }
                }
                if ($conf =~ /vxfendg\s*=\s*(\w+)/) {
                    $vxfendg = $1;
                    Msg::log("Fencing works in $vxfenmode mode with vxfendg=$vxfendg");
                } 
            } else {
                $msg = Msg::new("Fencing works in customized mode for cluster, $cfg->{vcs_clustername}. Configure fencing manually after add node procedure completes.", 40, 2607, "$cfg->{vcs_clustername}");
                $msg->print;
                return;
            }
        }

        $node_cnt = @{$cfg->{clustersystems}};
        for $sysi (@{$cfg->{newnodes}}) {
            $sys = Obj::sys($sysi);
            if ($vxfenmode eq "scsi3" || 
                    (($vxfenmode eq "customized") && ($vxfen_mechanism eq "cps"))) {
                if ($vxfenmode ne "scsi3") {
                    $ret = $vxfen_pkg->addnode_configure_cp_agent($firstnode, $sys, $node_cnt);
                    if ($ret) {
                        $msg = Msg::new("Failed to add new node $sysi into SystemList of the VCS group containing CoordPoint type resource. Manually add the node into SystemList after add node procedure completes.", 40, 2608, "$sysi");
                        $msg->warning();
                    }

                    if ($security) {
                        $ret = $prod->addnode_create_vcs_user_sys($sys);
                        if ($ret) {
                            $msg = Msg::new("Failed to create the VCS user, _HA_VCS_$sys->{sys}, on $sys->{sys}. Manually create VCS user before proceeding further.", 40, 2609, "$sys->{sys}", "$sys->{sys}");
                            $msg->warning(); 
                            Msg::prtc(); 
                        }

                        if ($cpsab ne "") {
                            $ret = $vxfen_pkg->addnode_setup_trust_with_cpsab($sys, $cpsab, $cpsabport);
                            if ($ret) {
                                $msg = Msg::new("Failed to setup trust between $sys->{sys} and $cpsab->{sys}. Manually resolve the issue before proceeding further.", 40, 2610, "$sys->{sys}", "$cpsab->{sys}");
                                $msg->warning();
                                Msg::prtc();
                            }
                        }
                    }
 
                    $ret = $vxfen_pkg->addnode_configure_cpc_sys($sys, $cfg->{vcs_clustername}, $node_cnt,
                                                                 $uuid, $security, $cps_aref, $ports_href);
                    if ($ret) {
                        $msg = Msg::new("Failed to add CP client $sysi to one or more CP Server(s). Manually configure fencing on $sysi after addnode procedure completes.", 40, 2611, "$sysi", "$sysi"); 
                        $msg->warning(); 
                        $node_cnt++;
                        next;
                    } 
                    # copy related config files if non_scsi3 fencing
                    $vxfen_pkg->addnode_configure_nonscsi3_sys($sys, $firstnode);
                    $node_cnt++;
                }
                # copy vxfenmode and vxfendg from cluster to new node
                if ($vxfenmode eq "scsi3") {
                    EDR::copy_file($firstnode,$sys,$prod->{vxfendg});
                } 
                EDR::copy_file($firstnode,$sys,$prod->{vxfenmode}) if (EDR::file_sys($firstnode,$prod->{vxfenmode}));
                if (($vxfenmode eq "scsi3") || ($vxfendg ne "")) {
                    # check VM running state on new node
                    my $vm = Obj::prod("VM51",$prod->{padv});
                    if ($vm->vold_status_sys($sys) ne "enabled") {
                        Msg::n();
                        $msg = Msg::new("VM must be started before configuring fencing", 40, 2612);
                        $msg->print();
                        my $startprocs = $vm->tsub("startprocs_sys", $sys);
                        for $proci (@$startprocs) {
                            $proc = $sys->proc($proci);
                            $msg=Msg::new("Starting $proc->{proc} on $sysi", 40, 1259, "$proc->{proc}", "$sysi");
                            $msg->left();
                            if ($cpic->tsub("proc_start_sys", $sys, $proc)) {
                                CPIC::proc_start_passed_sys($sys, $proc);
                                Msg::right_done();
                            } else {
                                CPIC::proc_start_failed_sys($sys, $proc);
                                Msg::right_failed();
                            }                        
                        }
                    }
                    if ($vm->vold_status_sys($sys) ne "enabled") {
                        $msg = Msg::new("Failed to start VM processes. Manually configure fencing on $sysi after addnode procedure completes.", 40, 2613, "$sysi");
                        $msg->warning();
                    }
                }
            } elsif ($vxfenmode eq "disabled") {
                EDR::cmd_sys($sys, "_cmd_cp /etc/vxfen.d/vxfenmode_disabled $prod->{vxfenmode}");
            }
            $proc = $sys->proc("vxfen51");
            $proc->enable_sys($sys);
            $msg=Msg::new("Starting $proc->{proc} on $sysi", 40, 1259, "$proc->{proc}", "$sysi");
            $msg->left();
            if ($cpic->tsub("proc_start_sys", $sys, $proc)) {
                CPIC::proc_start_passed_sys($sys, $proc);
                Msg::right_done();
            } else {
                CPIC::proc_start_failed_sys($sys, $proc);
                Msg::right_failed();
            }          
        }
    } else {
        # vxfen svc is disabled on existing cluster nodes.
        # it should be disabled on newly added nodes too.
        if ( $vxfen->can("disable_service_sys")) {
            for $sysi (@{$cfg->{newnodes}}) {
                $sys = Obj::sys($sysi);
                $vxfen->disable_service_sys($sys);
            }
        }
    }
}

sub addnode_get_cluster_uuid {
    my $prod = shift;
    my ($uuid_hash, $nuuid, @uuids, $uuid, $msg);

    if (!$prod->tsub("check_uuidconfig_pl")) {
        return -1;
    }

    $uuid_hash = $prod->tsub("get_uuid");
    $nuuid = keys %{$uuid_hash};
    if ($nuuid != 1) {
         return -1;
    }

    @uuids = keys %{$uuid_hash};
    $uuid = shift @uuids;

    return $uuid;
}

# This function starts and then stops VCS (had) in
# onenode mode.
# This function is used just to get the VCS user
# created which is needed to start CPS based fencing
# in secure mode.
# This function should be rewritten if there a easier 
# way create the VCS user. 
sub addnode_create_vcs_user_sys {
    my ($prod, $sys) = (@_);
    my ($maincf, $retry, $dn_str, $vcs_user_str, $msg, $sysname);
    my $edr = Obj::edr();
    my $cfg=Obj::cfg();

    $sysname =transform_system_name($sys->{sys});
    $vcs_user_str = EDR::cmd_sys($sys, "/opt/VRTScps/bin/cpsat showcred | grep _HA_VCS_$sysname");
    $dn_str = EDR::cmd_sys($sys, "/opt/VRTScps/bin/cpsat listpd -t local | _cmd_grep 'HA_SERVICES'");
    if (($vcs_user_str eq "") || ($dn_str eq "")) {      
        $maincf="include \"types.cf\"\n";
        $maincf.="\ncluster $cfg->{vcs_clustername} ("; 
        $maincf.="\n\tSecureClus = 1\n\t)\n\n";
        $maincf.="system $sysname(\n\t)\n\n";
        EDR::writefile($maincf, "$edr->{tmpdir}/main.cf", 1);
        EDR::copy_sys($sys, "$edr->{tmpdir}/main.cf", $prod->{maincf}); 
        EDR::cmd_sys($sys, "_cmd_cp $prod->{confdir}/types.cf $prod->{configdir}");

        EDR::cmd_sys($sys, "$prod->{bindir}/hastart -onenode");
        $retry = 10;
        while($retry) {
            $vcs_user_str = EDR::cmd_sys($sys, "/opt/VRTScps/bin/cpsat showcred | grep _HA_VCS_$sysname");
            if ($vcs_user_str eq "") {
                sleep(5);
                $retry--;
            } else {
               $retry = 0;
            } 
        }

        $retry = 10;
        while($retry) {
            $dn_str = EDR::cmd_sys($sys, "/opt/VRTScps/bin/cpsat listpd -t local | _cmd_grep 'HA_SERVICES'");
            if ($dn_str eq "") {
                sleep(5);
                $retry--;
            } else {
                $retry = 0;
            }
        }     

        EDR::cmd_sys($sys, "$prod->{bindir}/hastop -local");
        sleep(2);

        if ($vcs_user_str eq "") {
            Msg::log("Failed to create the VCS user, _HA_VCS_$sysname, on $sys->{sys}. Cannot proceed with fencing configuration.");
            return 1;
        } 

        if ($dn_str eq "") {
            Msg::log("Failed to create the domain HA_SERVICES on $sys->{sys}. Cannot proceed with fencing configuration.");
            return 1;
        }
    } 
    return 0;
}

sub get_publicnic_sys {
    my ($def,$rpn,$nicl,$en,$msg);
    my ($prod,$sys) = @_;
    my $sysi = $sys->{sys};
    while(1){
        $rpn=$sys->padv->publicnics_sys($sys);
        $rpn=EDR::arruniq(@$rpn);
        if ($#$rpn<0) {
            $msg = Msg::new("No active NIC devices have been discovered on $sysi", 40, 2327, "$sysi");
            $msg->warning;
        } else {
            $nicl=join(" ",@$rpn);
            $msg = Msg::new("Active NIC devices discovered on $sysi: $nicl", 40, 2328, "$sysi", "$nicl");
            $msg->print;
        }
        $def = $$rpn[0];
        $msg = Msg::new("Enter the NIC for the $prod->{abbr} to use on $sysi:", 40, 2631, "$prod->{abbr}", "$sysi");
        $en=$msg->ask($def);
        if (!EDR::nic_sys($sys, $en)) {
            $msg = Msg::new("$en is not a valid NIC name", 40, 2334, "$en");
            $msg->print;
        }else{
            return $en;
        }
    }
}

sub addnode_config_CSSG {
    return "" if (Cfg::opt("responsefile"));
    my ($all,$ask_gconic,$ask_csgnic,$ayn,$csgnic,$firstnode,$gco,$gcoip,$gconic,$msg,$n,$rtn,$smtp,$snmp,$sys,$sysi,$system,$vip,$padv);
    my $prod = shift;
    my $cfg = Obj::cfg();

    $firstnode = Obj::sys(${$cfg->{clustersystems}}[0]);
    $rtn = EDR::cmd_sys($firstnode,"$prod->{bindir}/hagrp -list 2>/dev/null |_cmd_grep ClusterService");
    return 1 if ($rtn eq "");

    # smtp
    $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -display ntfr -attribute SmtpServer 2>/dev/null |_cmd_grep -v Resource | _cmd_awk '{print \$4}'");
    if($rtn ne ""){
        $msg = Msg::new("SMTP notification is configured in the cluster $cfg->{vcs_clustername}", 40, 2616, "$cfg->{vcs_clustername}");
        $msg->print;
        $smtp = 1;
        $ask_csgnic = 1;
    }
    # snmp
    $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -display ntfr -attribute SnmpConsoles 2>/dev/null |_cmd_grep -v Resource | _cmd_awk '{print \$4}'");
    if($rtn ne ""){
        $msg = Msg::new("SNMP notification is configured in the cluster $cfg->{vcs_clustername}", 40, 2617, "$cfg->{vcs_clustername}");
        $msg->print;
        $snmp = 1;
        $ask_csgnic = 1;
    }
    # webip
    $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -list 2>/dev/null |_cmd_grep webip");
    if($rtn ne ""){
        $msg = Msg::new("Cluster Virtual IP is configured on the cluster $cfg->{vcs_clustername}", 40, 2618, "$cfg->{vcs_clustername}");
        $msg->print();
        $vip = 1;
        $ask_csgnic = 1;
    }
    # gco
    $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -list 2>/dev/null |_cmd_grep wac");
    if($rtn ne ""){
        $msg = Msg::new("Global Cluster Option is configured on the cluster $cfg->{vcs_clustername}.", 40, 2619, "$cfg->{vcs_clustername}");
        $msg->print();
        $gcoip = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -dep wac 2>/dev/null |_cmd_grep -v Group |_cmd_awk '{print \$3}'");
        if($gcoip ne "webip") {
            $gco = 1;
            $gconic = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -dep $gcoip  |_cmd_grep -v Group |_cmd_awk '{print \$3}'|_cmd_grep -v $gcoip");
            if ($gconic eq "gconic") {
                $ask_gconic = 1;
            } else {
                $ask_csgnic = 1;
            }
            $gconic = "";
        }
    }
        
    $n = scalar(@{$cfg->{newnodes}});

    $all = 0;    
    if ($ask_csgnic) {
        Msg::n();
        $msg = Msg::new("A public NIC device is required by following services on each of the newly added nodes:", 40, 2620);
        $msg->bold();
        if($smtp) {
            $msg = Msg::new("SMTP notification", 40, 2621);
            $msg->print();
        }
        if($snmp) {
            $msg = Msg::new("SNMP notification", 40, 2622);
            $msg->print();
        }
        if($vip) {
            $msg = Msg::new("Cluster Virtual IP", 40, 2623);
            $msg->print();
        }
        if($gco && !$ask_gconic) {
            $msg = Msg::new("Global Cluster Option", 40, 2624);
            $msg->print();
        }
        for $sysi (@{$cfg->{newnodes}}) {
            $sys = Obj::sys($sysi);
            if ($all) {
                $cfg->{vcs_csgnic}{$sysi} = $csgnic;
                next;
            }
            $padv=$sys->padv;
            $csgnic = $prod->get_publicnic_sys($sys);

            if (($n>1) && ($sysi eq ${$cfg->{newnodes}}[0])) {
                $msg = Msg::new("Is $csgnic the public NIC used by other new nodes?", 40, 2625, "$csgnic");
                $ayn = $msg->ayny_web();
                $all = 1 if ($ayn eq "Y");
            }
            $cfg->{vcs_csgnic}{$sysi} = $csgnic;
        }
    }
    
    $all = 0;
    if ($ask_gconic) {
        Msg::n();
        $msg = Msg::new("A public NIC device is required by Global Cluster Option on each of the newly added nodes:", 40, 2626);
        $msg->bold();
        for $sysi (@{$cfg->{newnodes}}) {
            $sys = Obj::sys($sysi);
            if ($all) {
                $cfg->{vcs_gconic}{$sysi} = $gconic;
                next;
            }
            $padv=$sys->padv;
            $gconic = $prod->get_publicnic_sys($sys);
            if (($n>1) && ($sysi eq ${$cfg->{newnodes}}[0])) {
                $msg = Msg::new("Is $gconic the public NIC used by other new nodes?", 40, 2625, "$gconic");
                $ayn = $msg->ayny_web();
                $all = 1 if ($ayn eq "Y");
            }
            $cfg->{vcs_gconic}{$sysi} = $gconic;
        }
    }
}

sub addnode_update_CSSG {
    my ($csgnic,$firstnode,$gconic,$global_csgnic,$global_gconic,$msg,$n,$rtn,$sysi,$system);
    my $prod = shift;
    my $cfg = Obj::cfg();
    
    $firstnode = Obj::sys(${$cfg->{clustersystems}}[0]);
    $rtn = EDR::cmd_sys($firstnode,"$prod->{bindir}/hagrp -list 2>/dev/null |_cmd_grep ClusterService");
    return 1 if ($rtn eq "");
    
    # update SystemList AutoStartList
    $n = $#{$cfg->{clustersystems}};
    for $sysi (@{$cfg->{newnodes}}) {
        $n++;
        $system = transform_system_name($sysi);
        EDR::cmd_sys($firstnode, "$prod->{bindir}/hagrp -modify ClusterService SystemList -add $system $n");
        EDR::cmd_sys($firstnode, "$prod->{bindir}/hagrp -modify ClusterService AutoStartList -add $system");
    }
    # update csgnic resource
    $global_csgnic = 0;
    if (defined($cfg->{vcs_csgnic})) {
        $msg = Msg::new("Updating csgnic resource", 40, 2627);
        $msg->left();
        if (EDR::hashvaleq($cfg->{vcs_csgnic})) {
            $sysi = ${$cfg->{newnodes}}[0];
            $csgnic = $cfg->{vcs_csgnic}{$sysi};
            $rtn = EDR::cmd_sys($firstnode,"$prod->{bindir}/hares -display csgnic -attribute Device |_cmd_grep $csgnic | _cmd_awk '{print \$3}'");
            $global_csgnic = 1 if ($rtn eq "global");
        }
        if (!$global_csgnic) {
            $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -local csgnic Device");
            for $sysi (@{$cfg->{newnodes}}) {
                $system = transform_system_name($sysi);            
                $csgnic = $cfg->{vcs_csgnic}{$sysi};
                $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -modify csgnic Device $csgnic -sys $system");
            }
        }
        Msg::right_done();
    }
    
    # update webip resource
    $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -list 2>/dev/null |_cmd_grep webip");
    if ($rtn ne "") {
        $msg = Msg::new("Updating webip resource", 40, 2628);
        $msg->left();
        if (!$global_csgnic) {
            $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -local webip Device");
            for $sysi (@{$cfg->{newnodes}}) {
                $system = transform_system_name($sysi);            
                $csgnic = $cfg->{vcs_csgnic}{$sysi};
                $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -modify webip Device $csgnic -sys $system");
            }
        }
        Msg::right_done();
    }

    # update gconic resource
    $global_gconic = 0;    
    if (defined($cfg->{vcs_gconic})) {
        $msg = Msg::new("Updating gconic resources", 40, 2629);
        $msg->left();
        if (EDR::hashvaleq($cfg->{vcs_gconic})) {
            $sysi = ${$cfg->{newnodes}}[0];
            $gconic = $cfg->{vcs_gconic}{$sysi};
            $rtn = EDR::cmd_sys($firstnode,"$prod->{bindir}/hares -display gconic -attribute Device |_cmd_grep $gconic | _cmd_awk '{print \$3}'");
            $global_gconic = 1 if ($rtn eq "global");
        }
        if (!$global_gconic) {
            $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -local gconic Device");
            for $sysi (@{$cfg->{newnodes}}) {
                $system = transform_system_name($sysi);
                $gconic = $cfg->{vcs_gconic}{$sysi};
                $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -modify gconic Device $gconic -sys $system");    
            }            
        }
        Msg::right_done();
    }   
        
    # update gcoip resource
    $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -list 2>/dev/null |_cmd_grep gcoip");
    if ($rtn ne "") {
        $msg = Msg::new("Updating gcoip resource", 40, 2630);
        $msg->left();
        if (defined($cfg->{vcs_gconic})) {
            if (!$global_gconic) {
                $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -local gcoip Device");
                for $sysi (@{$cfg->{newnodes}}) {
                    $system = transform_system_name($sysi);            
                    $gconic = $cfg->{vcs_gconic}{$sysi};
                    $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -modify gcoip Device $gconic -sys $system");
                }            
            }
        }elsif (defined($cfg->{vcs_csgnic})) {
            if (!$global_csgnic) {
                $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -local gcoip Device");
                for $sysi (@{$cfg->{newnodes}}) {
                    $system = transform_system_name($sysi);            
                    $gconic = $cfg->{vcs_csgnic}{$sysi};
                    $rtn = EDR::cmd_sys($firstnode, "$prod->{bindir}/hares -modify gcoip Device $gconic -sys $system");
                }            
            }
        }
        Msg::right_done();
    }
}

sub addnode_poststart {
    return 1;
}

sub addnode_completion {
    my ($errors,$errmsg,$msg);
    my $prod = shift;
    my $cpic = Obj::cpic();
    
    Msg::n();
    $errors = $cpic->failures("stopfailmsg", "startfailmsg");
    if ($#$errors < 0) {
        $msg = Msg::new("Addnode completed successfully", 40, 2632);
    } else {
        $msg = Msg::new("Addnode did not complete successfully", 40, 2633);
    }
    $msg->bold();
    
    $cpic->tsub("create_summaryfile");
    $cpic->tsub("edr_completion");
    return "";
}

sub num_hbnics{
    my ($prod,$nics, $nr_hbnics);
    ($prod,$nics) = @_;
    $nr_hbnics=1;
    while (defined($nics->{"lltlink$nr_hbnics"})) {
        $nr_hbnics++;
    }
    $nr_hbnics--;
    return $nr_hbnics;
}

sub num_lopri_hbnics{
    my ($prod,$nics, $nr_lowpri_hbnics);
    ($prod,$nics) = @_;
    $nr_lowpri_hbnics=1;
    while (defined($nics->{"lltlinklowpri$nr_lowpri_hbnics"})) {
        $nr_lowpri_hbnics++;
    }
    $nr_lowpri_hbnics--;
    return $nr_lowpri_hbnics;
}

sub num_hbnics_cfg{
    my ($prod,$cfg, $nr_hbnics);
    ($prod,$cfg) = @_;
    $nr_hbnics=1;
    while (defined($cfg->{"vcs_lltlink$nr_hbnics"})) {
        $nr_hbnics++;
    }
    $nr_hbnics--;
    return $nr_hbnics;
}

sub num_lopri_hbnics_cfg{
    my ($prod,$cfg, $nr_lowpri_hbnics);
    ($prod,$cfg) = @_;
    $nr_lowpri_hbnics=1;
    while (defined($cfg->{"vcs_lltlinklowpri$nr_lowpri_hbnics"})) {
        $nr_lowpri_hbnics++;
    }
    $nr_lowpri_hbnics--;
    return $nr_lowpri_hbnics;
}

sub set_lowpri_for_slow_links{
    my ($prod, $edr, $cfg, $cpic, $sys, $sysi, $link, $key, $key_low,$key_udp_addr,$key_udp_mask,$key_udp_port,$new_key,$new_key_udp_addr,$new_key_udp_mask,$new_key_udp_port,$nic, $speed_string, $new_rhbn_link_idx, $new_rhbn_lowpri_link_idx, $rhbn, $new_rhbn, $list, $least_hipri_links);
    my @links_to_lowpri;
    my %linkspeeds;
    my @sorted_links;
    ($prod,$rhbn) = @_;
    $edr = Obj::edr();
    $cfg = Obj::cfg();
    $cpic = Obj::cpic();

    $least_hipri_links = 1;
    if ( $cpic->{prod} =~ /SFRAC/ || $cpic->{prod} =~ /(SVS|SFCFS)/) {
        $least_hipri_links = 2;
    }

    if ( $least_hipri_links >= $prod->num_hbnics($rhbn)) {
        return $rhbn;
    }
    
    %linkspeeds = ();
    for $link (1..$prod->num_hbnics($rhbn)) {
        $key = "lltlink$link";
        $key_low = "lltlink$link"."_low";
        if ( defined ($rhbn->{$key_low})) {
            $linkspeeds{$key} = $rhbn->{$key_low};
        } else {
            $linkspeeds{$key} = 0;
        }
    }
    @sorted_links = sort {$linkspeeds{$b} <=> $linkspeeds{$a}} keys(%linkspeeds); # sort numerically descending

    for $link ($least_hipri_links..$#sorted_links) {
        if ($linkspeeds{$sorted_links[$link]} < $linkspeeds{$sorted_links[$least_hipri_links - 1]}) {
            push @links_to_lowpri, $sorted_links[$link];
        }
    }

    # lltlink#_high/lltlink#_low is not needed since we have set slower links as low pri links.
    $new_rhbn_link_idx = 1;
    $new_rhbn_lowpri_link_idx = 1;
    for $link (1..$prod->num_hbnics($rhbn)) {
        $key = "lltlink$link";
        if ( grep {$key eq $_} @links_to_lowpri ) {
            # low-pri
            $new_key = "lltlinklowpri$new_rhbn_lowpri_link_idx";
            foreach $sys (@{$edr->{systems}}) {
                $sysi = $sys->{sys};
                $new_rhbn->{$new_key}{$sysi} = $rhbn->{$key}{$sysi};
                if ($cfg->{lltoverudp}) {
                    $key_udp_addr = "udplink$link"."_address";
                    $key_udp_mask = "udplink$link"."_netmask";
                    $key_udp_port = "udplink$link"."_port";
                    $new_key_udp_addr = "udplinklowpri$new_rhbn_lowpri_link_idx"."_address";
                    $new_key_udp_mask = "udplinklowpri$new_rhbn_lowpri_link_idx"."_netmask";
                    $new_key_udp_port = "udplinklowpri$new_rhbn_lowpri_link_idx"."_port";
                    $new_rhbn->{$new_key_udp_addr}{$sysi} = $rhbn->{$key_udp_addr}{$sysi};
                    $new_rhbn->{$new_key_udp_mask}{$sysi} = $rhbn->{$key_udp_mask}{$sysi};
                    $new_rhbn->{$new_key_udp_port}{$sysi} = $rhbn->{$key_udp_port}{$sysi};
                }
            }
            $new_rhbn_lowpri_link_idx ++;
        } else {
            # hi-pri
            $new_key = "lltlink$new_rhbn_link_idx";
            foreach $sys (@{$edr->{systems}}) {
                $sysi = $sys->{sys};
                $new_rhbn->{$new_key}{$sysi} = $rhbn->{$key}{$sysi};
                if ($cfg->{lltoverudp}) {
                    $key_udp_addr = "udplink$link"."_address";
                    $key_udp_mask = "udplink$link"."_netmask";
                    $key_udp_port = "udplink$link"."_port";
                    $new_key_udp_addr = "udplink$new_rhbn_link_idx"."_address";
                    $new_key_udp_mask = "udplink$new_rhbn_link_idx"."_netmask";
                    $new_key_udp_port = "udplink$new_rhbn_link_idx"."_port";
                    $new_rhbn->{$new_key_udp_addr}{$sysi} = $rhbn->{$key_udp_addr}{$sysi};
                    $new_rhbn->{$new_key_udp_mask}{$sysi} = $rhbn->{$key_udp_mask}{$sysi};
                    $new_rhbn->{$new_key_udp_port}{$sysi} = $rhbn->{$key_udp_port}{$sysi};
                }
            }
            $new_rhbn_link_idx ++;
        }
    }
    # low-pri links specified by user.
    for $link (1..$prod->num_lopri_hbnics($rhbn)) {
        $key = "lltlinklowpri$link";
        $new_key = "lltlinklowpri$new_rhbn_lowpri_link_idx";
        foreach $sys (@{$edr->{systems}}) {
            $sysi = $sys->{sys};
            $new_rhbn->{$new_key}{$sysi} = $rhbn->{$key}{$sysi};
            if ($cfg->{lltoverudp}) {
                $key_udp_addr = "udplinklowpri$link"."_address";
                $key_udp_mask = "udplinklowpri$link"."_netmask";
                $key_udp_port = "udplinklowpri$link"."_port";
                $new_key_udp_addr = "udplinklowpri$new_rhbn_lowpri_link_idx"."_address";
                $new_key_udp_mask = "udplinklowpri$new_rhbn_lowpri_link_idx"."_netmask";
                $new_key_udp_port = "udplinklowpri$new_rhbn_lowpri_link_idx"."_port";
                $new_rhbn->{$new_key_udp_addr}{$sysi} = $rhbn->{$key_udp_addr}{$sysi};
                $new_rhbn->{$new_key_udp_mask}{$sysi} = $rhbn->{$key_udp_mask}{$sysi};
                $new_rhbn->{$new_key_udp_port}{$sysi} = $rhbn->{$key_udp_port}{$sysi};
            }
        }
        $new_rhbn_lowpri_link_idx ++;
    }
    return $new_rhbn;
}

package Prod::VCS51::AIX;
@Prod::VCS51::AIX::ISA = qw(Prod::VCS51::Common);

sub init_plat {
    my $prod=shift;
    $prod->{pkgs}=[ qw(VRTSveki51 VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTScps51 VRTSvcsag51 VRTSvcsea51) ];
    $prod->{minpkgs}=[ qw(VRTSveki51 VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTSvcsag51) ];
    $prod->{recpkgs}=[ qw(VRTSveki51 VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTSvcsag51 VRTSvcsea51) ];
    $prod->{upgradevers}=[qw(4.0.4 5.0 5.1)];
    $prod->{zru_releases}=[qw(5.0.3 5.1)];
    $prod->{deleted_agents} = [ qw(ServiceGroupHB ClusterMonitorConfig ClusterConnectorConfig VRTSWebApp) ];

    $prod->{obsoleted_pkgs_of_previous_releases} = [ qw(
        VRTScsocw.rte VRTSvcsApache VRTScmc VRTSccacm VRTSvcsw.rte
        VRTScspro VRTSvcsdb.rte VRTSvcsor.rte VRTSvcssy.rte
        VRTSvcsvr VRTScmccc.rte VRTScmcs.rte VRTSacclib52 VRTSacclib.rte
        VRTScscm.rte VRTSweb.rte VRTScscw.rte VRTScssim.rte
        VRTSvcsea51 VRTScutil VRTScutil.rte VRTSjre15.rte VRTSvcs.doc VRTSvcs.man
        VRTSvcs.msg.en_US VRTSvcsag51 VRTSvcsag.rte VRTScps51
        VRTSvcs51 VRTSvcs.rte VRTSvxfen51 VRTSvxfen.rte VRTSgab51
        VRTSgab.rte VRTSllt51 VRTSllt.rte VRTSveki51 VRTScpi.rte
        VRTSvsvc SYMClma VRTSspt51 VRTSat50 VRTSat.server VRTSat.client
        VRTSsmf VRTSpbx VRTSicsco VRTSjre.rte VRTSperl510 VRTSperl.rte
        VRTSvlic32
    ) ];
    # For UXRT 5.1SP1, VRTSacclib need to be fresh on AIX and Solaris.
    $prod->{obsoleted_but_need_refresh_when_upgrade_pkgs} = [ qw(VRTSacclib52) ];
}

sub update_pseconf_sys {
    my (@l,$dlpi,$line,$npse,$pse,$llt,$uen);
    my ($prod,$sys)=(@_);
    my $edr=Obj::edr();
    Msg::log("Updating pse.conf on $sys->{sys}");
    $pse=EDR::cmd_sys($sys, "_cmd_cat /etc/pse.conf");
    @l=split(/\n/,$pse);
    for $line(@l) {
        if (($line=~/^#d+/) && ($line=~/\/dev\/dlpi\/en/)) {
            $line=~s/^#d\+/d+/;
            $uen=1;
        }
        next if (($line=~/^d/) && ($line=~/llt/));
        $npse.="$line\n";
    }
    if ($uen) {
        EDR::writefile($npse, "$edr->{tmpdir}/pse.conf.$sys->{sys}");
        EDR::copy_sys($sys, "$edr->{tmpdir}/pse.conf.$sys->{sys}", "/etc/pse.conf");
        $dlpi=EDR::cmd_sys($sys, "_cmd_strload -q -d /usr/lib/drivers/pse/dlpi | _cmd_awk '{print \$2}'");
        chomp($dlpi);
        if ($dlpi eq "no") {
            EDR::cmd_sys($sys, "_cmd_strload -f /etc/dlpi.conf");
        }
        $llt=EDR::cmd_sys($sys, "_cmd_strload -q -d /usr/lib/drivers/pse/llt | _cmd_awk '{print \$2}'");
        if ($llt eq "no") {
            EDR::cmd_sys($sys, "_cmd_strload -d /usr/lib/drivers/pse/llt");
        }
        Msg::log("pse.conf updated");
    } else {
        Msg::log("pse.conf correct without update");
    }
}

package Prod::VCS51::HPUX;
@Prod::VCS51::HPUX::ISA = qw(Prod::VCS51::Common);

sub init_plat {
    my $prod=shift;
    $prod->{pkgs}=[ qw(VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTScps51 VRTSvcsag51 VRTSvcsea51) ];
    $prod->{minpkgs}=[ qw(VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTSvcsag51) ];
    $prod->{recpkgs}=[ qw(VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTSvcs51 VRTSvcsag51 VRTSvcsea51) ];
    $prod->{upgradevers}=[qw(3.5 4.1 5.0 5.1)];
    $prod->{zru_releases}=[qw(3.5 4.1 5.0 5.1)];
    $prod->{deleted_agents} = [ qw(ServiceGroupHB ClusterMonitorConfig CampusCluster ClusterConnectorConfig VRTSWebApp) ];

    $prod->{obsoleted_pkgs_of_previous_releases} = [ qw(
        VRTScsocw VRTSvcsApache VRTScmc VRTSccacm VRTSvcsw
        VRTScspro VRTSvcsdb VRTSvcsor VRTSvcssy VRTSvcsvr VRTScmccc
        VRTScmcs VRTScscm VRTSweb VRTScscw VRTScssim
        VRTSvcsea51 VRTScutil VRTSjre15 VRTSvcsdc VRTSvcsmn
        VRTSvcsmg VRTSvcsdr51 VRTSvcsag51 VRTScps51 VRTSvcs51
        VRTSvxfen51 VRTSgab51 VRTSllt51 VRTScpi VRTSvsvc SYMClma
        VRTSspt51 VRTSat50 VRTSsmfh VRTSpbx VRTSicsco VRTSjre
        VRTSperl510 VRTSvlic32
    ) ];
}

sub padv_mend_old_cf_sys {
    my ($prod, $sys, $path) = (@_);
    my $typecf = EDR::readfile_sys($sys, "$path/types.cf");
    if ($typecf =~ /\s*static\s+int\s+Operations\s+=\s+None/) {
        $typecf =~ s/(\s*static\s+)int(\s+Operations\s+=\s+None)/$1str$2/g;
        EDR::writefile_sys($sys, $typecf, "$path/types.cf");
    }
    return 1;
}

sub padv_maincf_upgrade_sys{
    my ($prod,$sys,$treeref_result,$treeref_newsys) = @_;
    # modify vxatd path if upgrade from 4.1 on HPUX platform
    Prod::VCS51::Common::dynupgrade_modify_attr_value($treeref_result,"ProcessOnOnly","PathName","/opt/VRTSat/bin/pa20_64/vxatd","/opt/VRTSat/bin/vxatd");
    return $treeref_result;    
}

package Prod::VCS51::Linux;
@Prod::VCS51::Linux::ISA = qw(Prod::VCS51::Common);

sub init_plat {
    my $prod=shift;
    $prod->{pkgs}=[ qw(VRTSatClient50 VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTScps51 VRTSvcsag51 VRTSvcsdr51 VRTSvcsea51) ];
    $prod->{minpkgs}=[ qw(VRTSatClient50 VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTSvcsag51) ];
    $prod->{recpkgs}=[ qw(VRTSatClient50 VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTSvcsag51 VRTSvcsdr51 VRTSvcsea51) ];
    $prod->{upgradevers}=[qw(4.1.40 5.0 5.1)];
    $prod->{zru_releases}=[qw(4.1.40 5.0 5.1)];
    $prod->{deleted_agents} = [ qw(ServiceGroupHB ClusterMonitorConfig CampusCluster ClusterConnectorConfig SANVolume VRTSWebApp) ];
    $prod->{rename_attrs} = {"Apache" => {"Address" => "HostName",
                                          "Postdirective" => "DirectiveAfter",
                                          "Predirective" => "DirectiveBefore",
                                          "ServerRoot" => "httpdDir"}
                            };
    $prod->{extra_types}=["vcsApacheTypes.cf"];

    $prod->{obsoleted_pkgs_of_previous_releases} = [ qw(
        VRTScsocw VRTSvcsApache VRTScmc VRTSccacm VRTSvcsw
        VRTScspro VRTSvcsdb VRTSvcsor VRTSvcssy VRTSvcsvr VRTScmccc
        VRTScmcs VRTScscm VRTSweb VRTScscw VRTScssim
        VRTSvcsea51 VRTScutil VRTSjre15 VRTSvcsdc VRTSvcsmn
        VRTSvcsmg VRTSvcsdr51 VRTSvcsag51 VRTScps51 VRTSvcs51
        VRTSvxfen51 VRTSgab51 VRTSllt51 VRTScpi VRTSvsvc SYMClma
        VRTSspt51 VRTSat50 VRTSatClient50 VRTSsmf VRTSpbx VRTSicsco
        VRTSjre VRTSperl510 VRTSvlic32
    ) ];

}

sub is_vlan_nic_sys {
    my ($prod,$sys,$nic) = @_;
    return EDR::file_sys($sys, "/proc/net/vlan/$nic");
}

sub convert_nic2lltlink_sys {
    my ($prod,$sys,$nic,$lowpri,@ifc,$msg,$cfg);
    ($prod,$sys,$nic,$lowpri) = @_;
    $cfg=Obj::cfg();
    return "" unless ($nic);

    if ($lowpri) {
        $lowpri="-lowpri";
    } else {
        $lowpri="";
    }

    if ($prod->is_vlan_nic_sys($sys, $nic) || $sys->padv()->is_bonded_nic_sys($sys, $nic)) {
        # VLAN interface. Cant use MAC here.
        return "link$lowpri $nic $nic - ether - -\n";
    }
    $_ = EDR::cmd_sys($sys, "_cmd_ip -o link show dev $nic");
    if (EDR::cmdexit()) {
        $msg=Msg::new("NIC $nic does not exist on system $sys->{sys}. LLT may not come up. Exiting installation.", 40, 2634, "$nic", "$sys->{sys}");
        $msg->die;
    }
    chomp;
    @ifc = split " ";
    my $idx = EDR::arrpos("link/ether", @ifc);
    if ($idx == -1) {
        # NIC without a MC?
        return "";
    }
    # The next index will be the MAC
    return "link$lowpri $nic eth-$ifc[$idx+1] - ether - -\n";
}

# get the original NIC name from a link/link-lowpri devname in /etc/llttab (For LLT over Ethernet)
sub convert_linkdev2nic_sys {
    my($prod,$mac,$dev,$found,$nic,$nici,$linkdev,$output,$rnics,$sys,$h);
    ($prod,$sys,$linkdev)=(@_);
    return "" if (!$linkdev);
    $rnics = $sys->padv()->systemnics_sys($sys,1);
    $found = 0;
    $mac = $linkdev;
    $h = "[A-Fa-f0-9]";
    if ( $mac !~ /eth-$h$h:$h$h:$h$h:$h$h:$h$h:$h$h/) {
        return $mac;
    }
    $mac=~s/eth-//;
    
    for $nici (@$rnics) {
        next unless $nici;
        $output = EDR::cmd_sys($sys, "_cmd_ifconfig $nici");
        if ( $output =~ /[^A-Fa-f0-9]$mac[^A-Fa-f0-9]/i) {
            $found = 1;
            $nic = $nici;
            last;
        }
    }
    if ( !$found) {
        $nic = $linkdev; # return original value if no NIC with specified MAC found.
    }
    return $nic;
}

sub set_onenode_cluster_sys {
    my($prod,$sys, $onenode) = @_;
    my $initconf = "/etc/sysconfig/vcs";
    my $conf = EDR::cmd_sys($sys,"_cmd_cat $initconf 2>/dev/null");
    return 0 unless ($conf);
    if ($onenode) {
        return 1 if ($conf =~ /ONENODE\s*=\s*yes\W+/);
        $conf =~ s/ONENODE\s*=.*/ONENODE=yes/;
    } else {
        return 1 if ($conf =~ /ONENODE\s*=\s*no\W+/);
        $conf =~ s/ONENODE\s*=.*/ONENODE=no/;
    }
    $conf .= "\n";
    EDR::writefile_sys($sys,$conf,$initconf);
    return 1;    
}

sub checknicconf_sys {
    my ($cnffile,$distro,$msg,$name,$rtn,$line,$nic,$mac,$masternics,$slavenics);
    my @ifcmd_out_all_lines;
    my %iflist;
    my ($prod,$sys) = @_;

    # only check for suse linux
    unless ($sys->{padv} =~ /SLES10/) {
        return 0;
    }
    ($masternics,$slavenics)=$sys->{padv}->bondednics_sys($sys);
    $rtn = EDR::cmd_sys($sys, "_cmd_ip -o link show");
    foreach $line (split("\n",$rtn)) {
        if ($line=~/\d+:\s+(\w*):.*link\/ether\s+(\w{2}:\w{2}:\w{2}:\w{2}:\w{2}:\w{2})\s+/){
            $nic=$1;
            $mac=$2;
        } else {
            next;
        }

        next if ($prod->is_vlan_nic_sys($sys,$nic));
        next if ($nic =~ /bond\d+/);
        next if (EDR::inarr($nic,@{$masternics}));
        next if (EDR::inarr($nic,@{$slavenics}));

       if (exists ($iflist{$mac})) {
            $iflist{$mac}.=" $nic";
        } else {
            $iflist{$mac} = $nic;
        }
    }

    my $fp = 0;
    $rtn= EDR::cmd_sys($sys, "_cmd_grep '^FORCE_PERSISTENT_NAMES' /etc/sysconfig/network/config");
    if( $rtn =~ /^FORCE_PERSISTENT_NAMES\s+=\s+no/ ) {
        $fp= 0;
    } else {
        $fp= 1;
    }
    
    if ($fp) {
        # for sles10, use rule based framework to find the PERSISTENT_NAMEs
        my (%filelist, $mac_addr, $content, @lines);
        $content= EDR::cmd_sys($sys, "_cmd_grep '^.*SUBSYSTEM.*==.*net.*ACTION.*==.*add' /etc/udev/rules.d/30-net_persistent_names.rules");
        @lines= split(/\n+/, $content);
        foreach(@lines) {
            # remove comments
            s/#.*$//;

            # parse lines similar to: 
            #   SUBSYSTEM=="net", ACTION=="add", SYSFS{address}=="00:0c:29:3c:04:fd", IMPORT="/lib/udev/rename_netiface %k eth0"
            #   SUBSYSTEM=="net", ACTION=="add", SYSFS{address}=="00:0c:29:3c:04:07", IMPORT="/lib/udev/rename_netiface %k eth1"
            if (/^.*?SUBSYSTEM\s*?==\s*?\"net\"\s*?,\s*?ACTION\s*?==\s*?\"add\"\s*?,\s*?SYSFS\{address\}\s*?==\s*?\"(\w{2}(:\w{2}){5})\".*?IMPORT.*\%k\s+(\w*?)\".*?$/) {
                $filelist{$1}= $3;
            }
        }
            
        # make sure each nic name has the right mac address
        foreach $mac_addr( keys %iflist) {
            # find the undefined NIC
            if( !$filelist{$mac_addr} ) {
                $msg = Msg::new("PERSISTENT_NAME is not set for all the NICs. Manually set it before the next reboot.", 40, 2635);
                EDR::push_warning_sys($sys,$msg);
                next;
            }
            if ($iflist{$mac_addr} ne $filelist{$mac_addr}) {
                $msg = Msg::new("WRONG configuration. NIC with MAC address $mac_addr has name $iflist{$mac_addr}, but config file says $filelist{$mac_addr}", 40, 3584, "$mac_addr", "$iflist{$mac_addr}", "$filelist{$mac_addr}");
                EDR::push_error_sys($sys,$msg);
                next;
            }
            # MAC address correctly set 
            Msg::log("$iflist{$mac_addr} on $sys->{sys} configured correctly");
        }

    } else {

        #EDR::copy_sys($sys,"/etc/sysconfig/network/ifcfg-eth-id-*", EDR::tmpdir(), 1);
        $rtn = EDR::cmd_sys($sys,"_cmd_ls /etc/sysconfig/network/ifcfg-eth-id-*");
        for $cnffile (split("\n",$rtn)) {
            my ($addr);

            Msg::log("Processing file $cnffile on $sys->{sys}");
            $addr = lc($cnffile);
            $addr = substr ($addr, -17); # Last 17 chars are the MAC
            if (!defined($iflist{$addr})) {
                # A file has already been found for this NIC.
                # We should proceed only if the current file is the higher
                # priority one, i.e., the one with the file name in caps.
                # But since the array has been lexicographiaclly sorted,
                # that cannot be. So we can safely skip this file.
                Msg::log("File $cnffile from $sys->{sys} skipped. This happens when there are two files for the same MAC address, and the current file is the one with lowercase filename.");
                next;
            }
            undef $name;
            $rtn = EDR::cmd_sys($sys,"_cmd_cat $cnffile |_cmd_grep '^PERSISTENT_NAME'");
            if ($rtn =~ /PERSISTENT_NAME\s+=\s+(.*)/) {
                $name = $1;
                if ($iflist{$addr}!~ /$name/) {
                    $msg = Msg::new("WRONG configuration. NIC with MAC address $addr has name $iflist{$addr}, but config file says $name.", 40, 3585, "$addr", "$iflist{$addr}", "$name");
                    EDR::push_error_sys($sys,$msg);
                } else {
                    Msg::log("$name on $sys->{sys} configured correctly");
                }
            } else {
                $msg = Msg::new("No PERSISTENT_NAME set for NIC with MAC address $addr (present name $iflist{$addr}), though config file exists.", 40, 2638, "$addr", "$iflist{$addr}");
                EDR::push_warning_sys($sys,$msg);
            }
            $iflist{$addr} = undef; # File for this guy found & processed
        }
        
        foreach (keys (%iflist)) {
            if (defined($iflist{$_})) {
                $msg = Msg::new("Configuration file for NIC with MAC address $_ (present name $iflist{$_}) not found", 40, 2639, "$_", "$iflist{$_}");
                EDR::push_warning_sys($sys,$msg);
            }
        }
    }
    
}

# On Linux(not cleare on other platforms), prod team request to back up and restore dns lock files
# 1. for files in /var/VRTSvcs/lock/ with the name like "_online_lock", rename to "._online_lock" (5.0*)
# 2. for files in /var/VRTSvcs/lock/ with the name like "._online_lock", backup directly (5.1)
sub backup_vcs_lock_files_sys {
    my ($prod,$sys,$lockfilepath);
    ($prod,$sys) = @_;
    $lockfilepath= EDR::tmpdir()."/VRTSvcs_lock";

    if (EDR::file_sys($sys,"/var/VRTSvcs/lock/*_online_lock")){
        EDR::cmd_sys($sys,"_cmd_mkdir -p $lockfilepath");
        EDR::cmd_sys($sys, "_cmd_cp /var/VRTSvcs/lock/*_online_lock $lockfilepath/");
    }
}

sub restore_vcs_lock_files_sys {
    my ($prod,$sys,$lockfilepath,$oldlockfile,$newlockfile,$files,@files);
    ($prod,$sys) = @_;
    $lockfilepath= EDR::tmpdir()."/VRTSvcs_lock";

    return 1 unless EDR::dir_sys($sys,$lockfilepath);
    $files = EDR::cmd_sys($sys, "cd $lockfilepath; _cmd_ls *._online_lock *[^.]_online_lock 2>/dev/null");
    @files = split(/\s+/, $files);
    return 1 unless (@files);
    EDR::cmd_sys($sys,"_cmd_mkdir -p /var/VRTSvcs/lock");
    for $oldlockfile (@files) {
        $newlockfile = $oldlockfile;
        $newlockfile =~ s/_online_lock/\._online_lock/ unless ($oldlockfile =~ /\._online_lock/);
        EDR::cmd_sys($sys, "_cmd_cp $lockfilepath/$oldlockfile /var/VRTSvcs/lock/$newlockfile") unless EDR::file_sys($sys,"/var/VRTSvcs/lock/$newlockfile");
    }
}

package Prod::VCS51::SunOS;
@Prod::VCS51::SunOS::ISA = qw(Prod::VCS51::Common);

sub init_plat {
    my $prod=shift;
    $prod->{pkgs}=[ qw(VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTScps51 VRTSvcsag51 VRTSvcsea51) ];
    $prod->{minpkgs}=[ qw(VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTSvcsag51) ];
    $prod->{recpkgs}=[ qw(VRTSat50 VRTSllt51 VRTSgab51 VRTSvxfen51 VRTSamf51 VRTSvcs51 VRTSvcsag51 VRTSvcsea51) ];
    $prod->{deleted_agents} = [ qw(Disk DiskReservation NFSLock ServiceGroupHB
                                   CampusCluster ClusterMonitorConfig CFSQlogckd
                                   ClusterConnectorConfig SANVolume VRTSWebApp) ];

    $prod->{obsoleted_pkgs_of_previous_releases} = [ qw(
        VRTScsocw VRTSvcsApache VRTScmc VRTSccacm VRTSvcsw
        VRTScspro VRTSvcsdb VRTSvcsor VRTSvcssy VRTSvcsvr VRTScmccc
        VRTScmcs VRTSacclib52 VRTScscm VRTSweb VRTScscw VRTScssim
        VRTSvcsea51 VRTScutil VRTSjre15 VRTSvcsdc VRTSvcsmn
        VRTSvcsmg VRTSvcsag51 VRTScps51 VRTSvcs51 VRTSvxfen51
        VRTSgab51 VRTSllt51 VRTScpi VRTSvsvc SYMClma VRTSspt51
        VRTSat50 VRTSsmf VRTSpbx VRTSicsco VRTSjre VRTSperl510
        VRTSvlic32
    ) ];
    # For UXRT 5.1SP1, VRTSacclib need to be fresh on AIX and Solaris.
    $prod->{obsoleted_but_need_refresh_when_upgrade_pkgs} = [ qw(VRTSacclib52) ];
}

sub set_onenode_cluster_sys {
    my($prod,$sys,$onenode) = @_;
    my $manifest = "vcs-onenode.xml";
    my $fmri = "svc:/system/vcs-onenode:default";
    my $rootpath = Cfg::opt("rootpath");
    my $smfprofile_upgrade = "$rootpath/var/svc/profile/upgrade";
    if ($onenode) {
        EDR::cmd_sys($sys,"_cmd_cp -rp $rootpath/etc/init.d/$manifest $rootpath/var/svc/manifest/system/$manifest");
        if ($rootpath) {
            EDR::cmd_sys($sys,"_cmd_echo svccfg import /var/svc/manifest/system/$manifest >> $smfprofile_upgrade");
            EDR::cmd_sys($sys,"_cmd_echo svcadm enable $fmri >> $smfprofile_upgrade");
        } else {
            EDR::cmd_sys($sys,"_cmd_svccfg import /var/svc/manifest/system/$manifest");
        }
    } else {
        return 1 unless (EDR::file_sys($sys,"$rootpath/var/svc/manifest/system/$manifest"));
        if ($rootpath) {
            EDR::cmd_sys($sys,"_cmd_echo svcadm disable -s $fmri >> $smfprofile_upgrade");
            EDR::cmd_sys($sys,"_cmd_echo svccfg delete $fmri >> $smfprofile_upgrade");
            #EDR::cmd_sys($sys,"_cmd_rm -f $rootpath/var/svc/manifest/system/$manifest");
        } else {
            EDR::cmd_sys($sys,"_cmd_svcadm disable -s $fmri");
            EDR::cmd_sys($sys,"_cmd_svccfg delete $fmri");
            EDR::cmd_sys($sys,"_cmd_rm -f /var/svc/manifest/system/$manifest");
        }
    }
    return 1;
}

# precheck for main.cf upgrade
sub padv_maincf_upgrade_precheck_sys {
    my($haconf,$grp_name,$res_name);
    my($grp_container_name,$grp_container_type,$grp_errors,$grp_container_settings,$container_settings);
    my($prod,$sys,$treeref,$msg,$padv,$vcspkg,$vcsvers);
    my($attr,$con_type,@grps,@grp_syslist,@reses,$res_type,$rtn,$sysi);
    my($container_opts,$grp_zone_name,$grp_zone_settings,$rootpath,$zone_errors,$zone_name);

    ($prod,$sys,$treeref) = @_;
    $rootpath=Cfg::opt("rootpath") || "";
    
    #return unless (EDR::file_sys($sys, $prod->{maincf}));

    # Check containerization setting in main.cf
    $padv = $sys->{padv};
    $vcspkg=$Obj::pool{"Pkg\::VRTSvcs51\::$padv"};
    if (defined $vcspkg) {
        $vcsvers=$vcspkg->version_sys($sys);
    }
    if ($vcsvers && (EDR::compvers($vcsvers,"5.1") == 2) && $sys->{zone}) {
        Msg::log("Checking the zone setting informations on $sys->{sys}");

        # Retrieve Service Group and Resource hierarchy informations from main.cf of 5.0 version

        $container_settings='';

        # get group list
        @grps = Prod::VCS51::Common::dynupgrade_get_grplist($treeref);
        for $grp_name (@grps) {
            next unless ($grp_name);

            $grp_container_name="";
            $grp_container_type="";
            $grp_container_settings="";
            $grp_errors="";

            $grp_zone_name="";
            $zone_errors="";
            $grp_zone_settings="";
            $container_opts="";
            $zone_name="";

            # get resources list
            @reses = Prod::VCS51::Common::dynupgrade_get_res_from_grp($treeref, $grp_name);
            for $res_name (@reses) {
                next unless ($res_name);

                $res_type = Prod::VCS51::Common::dynupgrade_get_restype_of_resname($treeref,$res_name);
                if ($res_type eq "Zone") {
                    $zone_name = Prod::VCS51::Common::dynupgrade_get_attrvalue_of_resname($treeref, $res_name, "ZoneName");
                    next unless ($zone_name);
                    $grp_zone_settings.="    $grp_name   Zone Name $zone_name\n";
                    if ($grp_zone_name && ($grp_zone_name ne $zone_name)) {
                        $zone_errors=1;
                    } else {
                        $grp_zone_name = $zone_name;
                    }
                    next;
                }

                # get resource's attributes
                $attr = Prod::VCS51::Common::dynupgrade_get_attrvalue_of_resname($treeref, $res_name, "ContainerName");
                EDR::set_value_sys($sys, "maincf_res_container_name,$res_name", $attr);
                next unless $attr;
                $grp_container_settings.="    $grp_name   $res_name  ContainerName  $attr\n";
                if ($grp_container_name) {
                    if ($grp_container_name ne $attr) {
                        $grp_errors=1;
                    }
                } else {
                    $grp_container_name = $attr;
                }

                $con_type = Prod::VCS51::Common::dynupgrade_get_attrvalue_of_typename($treeref,$res_type,"ContainerType");
                EDR::set_value_sys($sys, "maincf_res_container_type,$res_name", $con_type);
                next unless $con_type;
                $grp_container_settings.="    $grp_name   $res_name  ContainerType  $con_type\n";
                if ($grp_container_type) {
                    if ($grp_container_type ne $con_type) {
                        $grp_errors=1;
                    }
                } else {
                    $grp_container_type = $con_type;
                }
            }

            if (($grp_errors) || ($zone_errors) ){
                if ($grp_errors) {
                    $msg=Msg::new("In the file '$rootpath$prod->{maincf}' on $sys->{sys}, the ContainerName and ContainerType value of all resources in group '$grp_name' are not consistent :\n$grp_container_settings", 40, 3737, "$rootpath$prod->{maincf}", "$sys->{sys}", "$grp_name", "$grp_container_settings");
                    EDR::push_error_sys($sys,$msg);
                }
                if ($zone_errors) {
                    $msg=Msg::new("In the file '$rootpath$prod->{maincf}' on $sys->{sys}, the group '$grp_name' contains more than 1 Zone :\n$grp_zone_settings", 40, 3738, "$rootpath$prod->{maincf}", "$sys->{sys}", "$grp_name", "$grp_zone_settings");
                    EDR::push_error_sys($sys,$msg);
                }
            } elsif ($grp_container_name || $grp_container_type) {
                $grp_container_name||='\"\"';
                EDR::set_value_sys($sys, "maincf_grp_container_name,$grp_name", $grp_container_name);
                EDR::set_value_sys($sys, "maincf_grp_container_type,$grp_name", $grp_container_type);
            } elsif ($grp_zone_name) {
                EDR::set_value_sys($sys, "maincf_grp_container_name,$grp_name", $grp_zone_name);
                EDR::set_value_sys($sys, "maincf_grp_container_type,$grp_name", "Zone");
            }
        }

        if ($container_settings) {
            Msg::log("Container Settings:\n$container_settings\n");
            EDR::set_value_sys($sys, 'maincf_upgrade,21_container_settings', $container_settings);
        }
    }

    return 1;
}

# Update containerization settings to main.cf.
sub padv_maincf_upgrade_sys{
    my ($prod,$sys,$treeref_result,$treeref_newsys) = @_;
    my ($grp_name,$grp_container_name,$grp_container_type,@grp_syslist,$sysi);
    my (@reses,$res_name,$res_type,$container_opts);
    my ($container_settings,@container_settings);

    foreach $grp_name (sort keys %{$sys->{maincf_grp_container_name}}) {
        next unless ($sys->{maincf_grp_container_name}{$grp_name});
        next unless Prod::VCS51::Common::dynupgrade_get_node_from_tree($treeref_result,"GRP",$grp_name);
        $grp_container_name = $grp_container_type = "";
        $grp_container_name = $sys->{maincf_grp_container_name}{$grp_name};
        $grp_container_type = $sys->{maincf_grp_container_type}{$grp_name} || "Zone";
        @grp_syslist = Prod::VCS51::Common::dynupgrade_get_systemlist_of_grpname($treeref_result,$grp_name);
        for $sysi (@grp_syslist) {
            $container_settings.="hagrp -modify $grp_name ContainerInfo Type $grp_container_type Name $grp_container_name Enabled 1 -sys $sysi\n";
        }
        # get resources list
        @reses = Prod::VCS51::Common::dynupgrade_get_res_from_grp($treeref_result, $grp_name);
        for $res_name (@reses) {
            $res_type = $container_opts = "";
            $res_type = Prod::VCS51::Common::dynupgrade_get_restype_of_resname($treeref_result,$res_name);
            next if ($res_type eq "Zone");
            $container_opts = Prod::VCS51::Common::dynupgrade_get_attrvalue_of_typename($treeref_newsys,$res_type,"ContainerOpts");
            if($sys->{maincf_res_container_name}{$res_name}) {
                if ($sys->{maincf_res_container_type}{$res_name} eq "Zone") {
                    $container_settings.="hares -override $res_name ContainerOpts\nhares -modify $res_name ContainerOpts RunInContainer 1 PassCInfo 0\n"
                        if ($container_opts !~ /RunInContainer\s+1\s+PassCInfo\s+0/);
                } else {
                    $container_settings.="hares -override $res_name ContainerOpts\nhares -modify $res_name ContainerOpts RunInContainer 0 PassCInfo 1\n"
                        if ($container_opts !~ /RunInContainer\s+0\s+PassCInfo\s+1/);
                }
            } else {
                $container_settings.="hares -override $res_name ContainerOpts\nhares -modify $res_name ContainerOpts RunInContainer 0 PassCInfo 0\n"
                    if ($container_opts =~ /RunInContainer\s+\d+\s+PassCInfo\s+\d+/);
            }
        }
    }
    Msg::log("Upgrade main.cmd for user configuration: $container_settings\n");
    @container_settings = split (/\n+/,$container_settings);
    $treeref_result = Prod::VCS51::Common::dynupgrade_import_lines2tree(\@container_settings, $treeref_result);
    return $treeref_result;
}

package Prod::VCS51::SolSparc;
@Prod::VCS51::SolSparc::ISA = qw(Prod::VCS51::SunOS);

sub init_padv {
    my $prod=shift;
    $prod->{upgradevers}=[qw(4.1.2 5.0 5.1)];
    $prod->{zru_releases}=[qw(4.1.2 5.0 5.1)];
}

package Prod::VCS51::Solx64;
@Prod::VCS51::Solx64::ISA = qw(Prod::VCS51::SunOS);

sub init_padv {
    my $prod=shift;
    $prod->{upgradevers}=[qw(5.0 5.1)];
    $prod->{zru_releases}=[qw(5.0 5.1)];
}

1;