#!/var/opt/STORtools/bin/perl

require  "subroutines.pm";
&st_globals();

$PROGNAME = "sedr";

$SIG{INT} = 'clean_up';
$MONITOR = "FALSE";
####################################################################
# StorEdgeDr main()
####################################################################
    &check_root_access( $PROGNAME );
    &proc_cli();
    my $input = "";
    while ($input !~ /Q|q/) {
        $input = &get_input;
        if ($input == 1) {
            &system_clear();
            &sedr();
        } elsif ($input == 2) {
            &log_manager;
        } elsif ($input ne "q") {
            print "\n";
            print "$input is an invalid option!\n";
        }
    }    

###############################################################
# clean up all log files
###############################################################
sub clean_logs {
    while (1) {
        print "\nWould you like to remove all log files? [y, <n>]: ";
        $ans = &get_response("n");
        if ($ans =~ /y/i) {
            system "/usr/bin/ksh -c '/usr/bin/rm ${LOGSEDR}/sedr.* 2>/dev/null'"; 
            last;
        } elsif ( $ans =~ /n/i) {
            last;
        } else {
            print"\n";
            print"$ans is an invalid option!\n";
        }
    }
    &continue;
}

###############################################################
# show mess_sum
###############################################################
sub show_mess_sum {
    
   &print_banner;
   $file_id = &get_dfileid;

   if ($file_id == -1) 
   {
       print "Log File Does Not Exist.\n"; 
   } 
   else 
   {

      $sum_files = `/usr/bin/ksh -c '/bin/ls -1t ${LOGSEDR}/sedr.log.$file_id 2>/dev/null'`;
      @arr_files = split(/\n/,$sum_files);
      print "\nChecking Latest sedr summary Logs:\n";
      foreach $sum_file (@arr_files) 
      {
         if (-e $sum_file) 
	 {
            print "Press <return> to view $sum_file";
            $cont = <STDIN>;
	    system "/usr/bin/more $sum_file";
            print "\n";
         }
      }
   }
   &continue;
}

###############################################################
# display the installation verification submenus and take input
###############################################################
sub get_input{
    &print_banner;
    print " [1] Start StorEdgeDr\n";
    print " [2] StorEdgeDr Log Manager\n";
    print "\n";
    print " [q] Quit\n";
    print "\nSelect an operation to perform: ";
    chop($stdin=<STDIN>);
    return $stdin;
}

###############################################################
# display the installation verification submenus and take input
###############################################################
sub log_manager {
    my $input = "";

    while ($input !~ /Q|q/) {
        $input = &print_logscreen;
        if ($input == 1) 
	{
            &show_mess_sum;
        } 
	elsif ($input == 2)
	{
            &show_full_logs();
        } 
	elsif ($input == 3) 
	{
            &show_summary();
        } 
	elsif ($input == 4)
	{
            &show_diskx();
        }
	elsif ($input == 5) 
	{
            &show_pci_hba();
        } 
	elsif ($input == 6)
	{
            &clean_logs;
        } 
	elsif ( $input ne "q" ) 
	{
            print "\n";
            print "$input is an invalid option!\n";
        }
    }
}


###############################################################
# get the file_id
###############################################################
sub get_fileid {
   $all_lbf = `/usr/bin/ksh -c '/bin/ls -1t ${LOGSEDR}/sedr.log.*.str.lbf* 2>/dev/null'`;
    @arr_allfiles = split(/\n/,$all_lbf);

    if ($#arr_allfiles < 0) {
        return -1; 
    } else {
        $match = $arr_allfiles[0];
        ($latest_id) = $match =~ /sedr.log.(\d+)/;
        return $latest_id;
    }
}
###############################################################
# get the dfile_id
###############################################################
sub get_dfileid {
   $all_dex = `/usr/bin/ksh -c '/bin/ls -1t ${LOGSEDR}/sedr.log.*.str.dex* 2>/dev/null'`;
    @arr_allfiles = split(/\n/,$all_dex);

    if ($#arr_allfiles < 0) {
        return -1; 
    } else {
        $match = $arr_allfiles[0];
        ($latest_id) = $match =~ /sedr.log.(\d+)/;
        return $latest_id;
    }
}

###############################################################
# choose_cnum
###############################################################
sub choose_cnum {
    my @cur_menu;
    my @allcnums = &get_cnums;
    my $x = 0;
    my $ans;
    foreach $cnum (@allcnums) {
       $x++;
       @cur_menu = (@cur_menu, "  [$x] $cnum\n");
    }
    if ($x == 0) {
        print "Log Files Do Not Exist.\n";
        return "q";
    } 
    @cur_menu = (@cur_menu,
                 "\n  [a] All\n",
                 "  [q] Quit\n",
                 "\nSelect [ 1 - $x, a or <q> ]: ");
choose_cnum_start:
    print @cur_menu;
    $ans = &get_response("q");
    if ($ans =~ /^q/i) { return "q"; }
    if ($ans =~ /^a/i) { return "a"; }
    if ($ans lt 1 or $ans gt $x) {
        print "\n$ans is an invalid option!\n";
        goto choose_cnum_start;
    }
    return $allcnums[$ans - 1];
}

sub get_cnums {
    my %cnames;

     open CNAMES, "ls $LOGSEDR |";
     while (<CNAMES>) {
         chomp;
#         $cnames{$1} = $cnames{$1} + 1 if /sedr.log.\d+\.str.lbf.(c\d+)/;
          $cnames{$1} = $cnames{$1} + 1 if /sedr.log.$file_id.str.lbf.(c\d+)/;
     }
     return sort (keys %cnames);
}


###############################################################
# show the failures in the lbf log
###############################################################
sub show_full_logs {
    &print_banner;
    $file_id = &get_fileid;

    if ($file_id == -1) { 
        print "Log File Does Not Exist.\n"; 
    } else {
    my $cnum = &choose_cnum;
    if ($cnum eq "a") {
          $lbf_files = `/usr/bin/ksh -c '/bin/ls -1t ${LOGSEDR}/sedr.log.$file_id.str.lbf.c* 2>/dev/null'`;
    } elsif ($cnum eq "q") {    
        print "\nPress <return> to continue.";
        $cont = <STDIN>;
        return; 
    } else {
        $lbf_files = `/usr/bin/ksh -c '/bin/ls -1t ${LOGSEDR}/sedr.log.$file_id.str.lbf.$cnum 2>/dev/null'`;
    }
        @arr_files = split(/\n/,$lbf_files);
        print "\nChecking Latest Loop Integrity Logs:\n";
        foreach $lbf_file (@arr_files) {
            if (-e $lbf_file) {
                print "Press <return> to view $lbf_file";
                $cont = <STDIN>;
                system "/usr/bin/more $lbf_file";
                print "\n";
             }
        }
    }
    &continue;
}

sub show_summary {
    &print_banner;
    $file_id = &get_fileid;

    if ($file_id == -1) { 
        print "Log File Does Not Exist.\n"; 
    } else {
        my $cnum = &choose_cnum;
        if ($cnum eq "a") {
            $lbf_files = `/usr/bin/ksh -c '/bin/ls -1t ${LOGSEDR}/sedr.log.$file_id.str.lbf.c* 2>/dev/null'`;
        } elsif ($cnum eq "q") {
            print "\nPress <return> to continue.";
            $cont = <STDIN>;
            return;
        } else {
            $lbf_files = `/usr/bin/ksh -c '/bin/ls -1t ${LOGSEDR}/sedr.log.$file_id.str.lbf.$cnum 2>/dev/null'`;
        }
        @arr_files = split(/\n/,$lbf_files);
        print "\nChecking Latest Loop Integrity Logs:\n";
        foreach $lbf_file (@arr_files) {
            if (-e $lbf_file) {
                $failures = `/bin/grep -i fail $lbf_file`;
                print "Press <return> to view $lbf_file";
                $cont = <STDIN>;
                if ($failures eq "") {
                    print "No failures found.\n\n";
                } else {
                    system "/usr/bin/grep -i fail $lbf_file | more";
                    print "\n";
                }
            }
        }
    }
    &continue;
}

#########################################################
# display disk exerciser log file           #
#########################################################
sub show_pci_hba {
    &print_banner;
    $dfile_id = &get_dfileid;

    if ($dfile_id == -1) { 
        print "Log File Does Not Exist.\n"; 
    } else {
        $log_files = `/usr/bin/ksh -c '/bin/ls -1t ${LOGSEDR}/sedr.log.$dfile_id.str.hba_test.* 2>/dev/null'`;
        @arr_files = split(/\n/,$log_files);
        print "\nChecking Latest HBA test Log:\n";
        foreach $log_file (@arr_files) {
            if (-e $log_file) {
                print "Press <return> to view $log_file";
                $cont = <STDIN>;
                system "/usr/bin/more $log_file";
                print "\n";
            }
        }
    }
    &continue
}

#########################################################
# display disk exerciser log file           #
#########################################################
sub show_diskx {
    &print_banner;
    $dfile_id = &get_dfileid;

    if ($dfile_id == -1) { 
        print "Log File Does Not Exist.\n"; 
    } else {
        $log_files = `/usr/bin/ksh -c '/bin/ls -1t ${LOGSEDR}/sedr.log.$dfile_id.str.dex.* 2>/dev/null'`;
        @arr_files = split(/\n/,$log_files);
        print "\nChecking Latest Disk Exerciser Log:\n";
        foreach $log_file (@arr_files) {
            if (-e $log_file) {
                print "Press <return> to view $log_file";
                $cont = <STDIN>;
                system "/usr/bin/more $log_file";
                print "\n";
            }
        }
    }
    &continue
}

sub print_logscreen {
    &print_banner;
    print " [1] Storage Messages Summary\n";
    print " [2] Fibre Loop Test - Full Logs\n";
    print " [3] Fibre Loop Test - Failures only\n";
    print " [4] Disk Exerciser Logs\n";
    print " [5] HBA Test Logs\n";
    print " [6] Remove Log Files\n";
    print "\n";

    print " [q] Quit\n";
    print "\nSelect an operation to perform: ";
    chop($stdin=<STDIN>);
    return $stdin;
}

###############################################################
# cont_test
###############################################################
sub cont_test {
    $cont = 1;

    while($cont) {
        print "Would you like to continue: [y, <n>]: ";
        $tmp = &get_response('n');
        if ($tmp =~ /^n/i) {
            $cont = 0;
        } elsif ($tmp =~ /^y/i) {
            $cont = 0;
        } else {
            print "Please enter yes or no\n";
        }
    }
    return ($tmp); 
}

###############################################################
# continue
###############################################################
sub continue {
    print "\nPress <return> for previous menu ...";
    $cont = <STDIN>;
}


###############################################################
# prints the header banner for this program
###############################################################
sub print_banner {
    system_clear();
    &st_header();
    print "Menu: StorEdgeDr\n";
    print "----------------\n\n";
}


sub proc_cli {

    $HBA = "";
    $DUAL_PORT = $PRIMARY;
    $MENU_DUAL_PORT = $PRIMARY;

    $usage[0] = "$PROGNAME [-help] \n";
    $usage[1] = "help - Display usage\n";

    $arg = shift @ARGV;
    while($arg) {
        if ($arg =~ /help/)  
	{
            die @usage;
	}
        # Get the next value.
        # The last command in the loop
        $arg = shift @ARGV;
    }


}


sub sedr {

    $ERRFILE_NAME = "sedr.log.$$.err";
    $LOGFILE = "$LOGSEDR/sedr.log.$$";
    $ERRFILE = "$LOGSEDR/$ERRFILE_NAME";

    &open_logfile($ERRFILE_NAME, $LOGSEDR);

    system "/usr/bin/touch $ERRFILE";

    $| = 1;         # turn off write buffer to print .

    # find the controllers present (list of c-names)

    &set_color( "ON" );
    &make_port_list;


    if( -e $LOGFILE ) {

        print "$LOGFILE exists, overwrite( <y>, n = append ):";

        $resp = <STDIN>;

        if($resp =~ /^y/i) {
            open(LOGHAND, ">${LOGFILE}") or die "$LOGFILE failed to open: $!";
            print "\nCreating file $LOGFILE\n";
        } else  {
            open(LOGHAND, ">>${LOGFILE}") or die "$LOGFILE failed to open: $!";
            print "\nappending to file $LOGFILE\n";
        }
    }
    else
    {
       open(LOGHAND, ">${LOGFILE}") or die "$LOGFILE failed to open: $!";
    }

    # Assign defaults all true, run all tests

    $LBF_ITERS = 1000;  
    $LBF_KBYTES = 64;
    $CON_RUN_TIME = 1;
    $DEX_RUN_TIME = 10;

    $RD_CHOICE = "ro";
    $MIN_RUN_TIME = 1;     # min time 1 minute
    $MAX_RUN_TIME = 1440;  # max time 24 hours
    $SHOW_LOGS="verbose";
    $NO_LOGS="quiet";
    $VIEW_LOGS = $SHOW_LOGS;
    $HBA_MODE = "skip";
    $RUN_HBA_OFFLINE = "no";

    print "CAUTION: DO NOT run this test while other loop diagnostics\n";
    print "         or I/O are running on the loop to be tested.\n\n";


    #get which loop(s) to work against
    get_test_list();
    return if ($#TEST_LP_LIST == -1);
    #set menu options given the list of ports to run against
    set_menu_options();

    while(1) {
	
        $res = display_menu();

        if($res eq "std") {
            &sedr_test();
        } elsif ($res eq "exit") {
            last;
        }
	print "\nPress return to continue.....\n";
	<STDIN>



    }
    close(LOGHAND);
    
}

sub display_menu {

    system_clear();
    &st_header();
    print "Menu: StorEdgeDr\n";
    print "----------------\n\n";
    print " \n";
    
    while (1) {

        print ("\n\n");
        print ("Options                         Value\n");
        print ("===========================     ====================\n");
        print "[p] Port(s) under test:          ";
        foreach $e (@TEST_LP_LIST) 
	{
           print "$CNUM{$e} ";
        }
        print "\n";


	if($SHOW_T300 eq "m,")
	{
	   print "[m] T300 Dual Port Mode          $MENU_DUAL_PORT\n";
	}
	if($SHOW_LBF_ITERS eq "c,")
	{
           print "[c] Data Path Pass Count         $LBF_ITERS pass(es)\n";
	}
	if($SHOW_CON_RUN_TIME eq "t,")
	{
           print "[t] Data Path Run Time           $CON_RUN_TIME minute(s)\n";
	}
        print "[d] dex Duration                 $DEX_RUN_TIME minute(s)\n";
        print "[h] HBA test                     $HBA_MODE\n";  
	if($HBA_MODE eq "execute")
	{
	   if($SHOW_OFFLINE_HBA eq "o,")
	   {
	      print "[o] RUN OFFLINE HBA TEST         $RUN_HBA_OFFLINE\n"; 
	   }
	}       
 
        print "[v] Verbose                      $VIEW_LOGS\n";  

        print "\n[r] Run\n";
        print "\n[q] Quit\n\n";

        print "Enter Selection [<r>,p,${SHOW_T300}${SHOW_LBF_ITERS}${SHOW_CON_RUN_TIME}d,h,${SHOW_OFFLINE_HBA}v,q]: ";

        $choice = &get_response("r");               # user response

        if($choice =~ /^q/i ) {         # Q or q : quit 
            return "exit";
        }

        if ( $choice =~ /^m/i ) {
            $MENU_DUAL_PORT = &select_dual_port_mode();
            next;
        }

        if($choice =~ /^p/i ) {      
            get_test_list();
	    set_menu_options();

            next;
        }

        if($choice =~ /^t/i ) {      
            $CON_RUN_TIME = &get_time( $CON_RUN_TIME );
            next;
        }

	if($choice =~ /^c/i ) {      
            $LBF_ITERS = &get_iterations( $LBF_ITERS );
            next;
        }

        if($choice =~ /^d/i ) {      
            $DEX_RUN_TIME = &get_time( $DEX_RUN_TIME );
            next;
        }

        if($choice =~ /^h/i ) {      
            if ( $HBA_MODE eq "execute" ) { 
                $HBA_MODE = "skip";
            } else {
                $HBA_MODE = "execute";
            } 
            next;
        }

        if($choice =~ /^v/i ) {      
            if($VIEW_LOGS eq $SHOW_LOGS) {
                $VIEW_LOGS = $NO_LOGS;
            } else {
                $VIEW_LOGS = $SHOW_LOGS;
            }
            next;
        }

        if($choice =~ /^o/i ) 
	{      
	   if($RUN_HBA_OFFLINE eq "no")
	   {
	      $RUN_HBA_OFFLINE = "yes";
	   }
	   else
	   {
	      $RUN_HBA_OFFLINE = "no";
	   }
           next;
        }


        if($choice =~ /^r/i ) {
            return "std";
        }

        print "\n";
        print "$choice is an invalid option!\n";
        
    }

}

####################################################################
# check iterations
####################################################################
sub get_iterations {

    my $run_time = $_[0];

    while ( 1 ) {
        print "Please enter iterations in decimal [between $MIN_RUN_TIME and $MAX_RUN_TIME, <$run_time>]: ";
        $run_time = &get_response( $run_time );
        chomp($run_time);
        if(($run_time < $MIN_RUN_TIME) || ($run_time > $MAX_RUN_TIME)) {
            print "$run_time is an invalid count!\n";
			$run_time = $_[0];
            next;
        }
        last;
    }
    return($run_time);

}
####################################################################
# check time
####################################################################
sub get_time {

    my $run_time = $_[0];

    while ( 1 ) {
        print "Please enter run-time in minutes [between $MIN_RUN_TIME and $MAX_RUN_TIME, <$run_time>]: ";
        $run_time = &get_response( $run_time );
        chomp($run_time);

	$int_run_time=int($run_time);

        if(($run_time < $MIN_RUN_TIME) || 
	   ($run_time > $MAX_RUN_TIME) ||
	   ("$run_time" ne "$int_run_time")) 
	{
            print "$run_time is an invalid time!\n";
			$run_time = $_[0];
            next;
        }
        last;
    }
    return($run_time);

}


# This will fork a process that will montior the log
# file passed as arg 1
# needs to get killed at end of test.

        
####################################################################
# run lbf and dex
# new logfile created, sedr.pid.lbfdex.cnum
####################################################################

sub sedr_test {

   my $test_name;
   my $verbose;
   my $expected_dma_errors;
      $test_status = 0;
   my $start_date;
   my $skip_loop_test;

   # add a check to see if loop test will be offline(qlc port)
   $skip_loop_test = 0;
   foreach $port (@TEST_LP_LIST) 
   {
       if( $port =~ /qlc/ )
       {
          # this port is a qlc port
	  print "\nWARNING: The loop test will offline this I/O path:\n($CNUM{$port}) $port\n";
          print "It should not be run on storage arrays currently in use.\n";
	  print "If you choose no, all loop tests will be bypassed. \n";
	  print "Do you wish to run the loop tests? [y, <n>]: ";
          $usr_choice = "";
          while ($usr_choice eq "") 
	  {
            $usr_choice = &get_response;
            if ($usr_choice =~ /^y/i) 
	    {
               # continue in for loop
            } 
	    elsif (($usr_choice =~ /^n/i) | ($usr_choice eq "")) 
	    {  
	       print "\nUser decided not to run loop test with current selected ports.\n";
	       $skip_loop_test = 1;
	       last;
            } 
	    else 
	    {
               print "Please choose [<y>, n]:";
               $usr_choice = "";
            }
	  }
	  if($skip_loop_test == 1)
	  {
	     # break out of for loop
	     last;
	  }

       }
   }


   foreach $port (@TEST_LP_LIST) 
   {
      print LOGHAND "\n ======== Testing port: ${port}  \n  ";

      &set_hba_port_type();

      &set_storage_type();
      
      if ($HBA_MODE eq "execute") 
      {
         # run hba tests
         &test_hba();
      }

      if ( $test_status != 0 ) {
         print "Test status: $test_status\n";
         print "Replace HBA and restart testing.\n";
         return;
      }

       if( $skip_loop_test == 0 )
       {
          loop_test();

	  if ( $test_status != 0 ) 
	  {
	     print "\n";
	     print "\n";
             print "Detected a loop problem.\n\n";
	     print "${RED} ****** Test FAILED ****** ${RC}\n";


             return;
	  }
       }
    
      # Storage test for both SBUS and PCI

      storage_test();

   } # end for each
  
}

sub monitor_log { 

    my $log = $_[0];

    # and can be tested elsewhere to ensure kills do not
    # do anything the parent should really do.

    if($pid = fork()) {
        return($pid);
    } else {

        $MONITOR = "TRUE";  # this variable will set this process copy as unique
        $SIG{INT} = 'clean_up';
        while( ! -e $log ) {
            sleep 5;
        }
        open(LOG, "$log") || die "sedr:  monitor_log cannot open [$log]";

        while ( 1 ) {
            if ( $line = <LOG> ) {
                if ( $line =~ /fail/i ) {
                    print "\n$line";
                }
            }
            sleep 1;
        }
        exit;

    }

    return(-1);

}
            

sub killMonitor {

    foreach $i ( @MONITOR_PIDS ) {
        `kill $i`;

    }

}

####################################################################
# control c interrupt handler
####################################################################

sub clean_up {

    if($MONITOR eq "TRUE" ) { 
    # don't do any cleanup.. This is just a monitor version
        # see monitor_log().
        exit;
    }
    killMonitor();
    kill_pid_files($LOGDIR, "k", "TRUE");  # kill processes silently
    exit;
}

####################################################################
# faux_logger to /var/adm/messages
# messages to var adm messages should have the form:
#  date: [A5000:DIAG:<status>:<error number>] error message
#   INPUTS:
#   $status     WARNING or ERROR or FAIL or INFO
#   $error-number   4 digit error code
#   $error-message  1 line message string
####################################################################

sub faux_logger {

    ($status, $error_number, $message) = @_;
    print "$status : $message\n";
    logger($status, $error_number, $message);

}

sub data_path_error {
print <<END;

  FC-AL connect to the $STORAGE failed.
  Check GBIC connections. Make sure that the bail is in the correct postion.
  Check MIA connection. If all are secure, replace one item at a time starting 
  with a single GBIC. Re-run the test.  If replacing the GBICs does not 
  correct the problem, change the MIA and re-start the test. If replacing
  the MIA does not correct the problem, replace the FC-AL cable and re-test.
  If the problem persists try replacing either the ${STORAGE} FC-AL
  board or the FC-AL HBA adapter. During each re-test always make sure the
  the error reported is consistent before changing a new part. For instance,
  if changing the GBIC consistently produces a 5A's pattern failure and the
  original GBIC consistantly produced a 7E's pattern failure, both GBICs may
  be faulty.
       
END
}

# Run HBA test on a given HBA port.
sub test_hba
{
    &st_sync();

    print "BEGIN HBA TEST on loop $port\n";

    if ( $HBA eq "S" ) 
    {

        # SBUS Host Bus Adapter Test
        # currently using lbf 
        # want to change to SUNvts test in future

        $test_name = "SBUS HBA test(lbf)";

        
	if($RUN_HBA_OFFLINE eq "no")
	{
	   print "\nThe ${test_name} is an offline test.\n";
	   print "User selected not to run offline HBA test \n";
	   print "\nEND OF TESTING HBA on loop $port\n";

	   return;
	}
	else
	{
	   print "\nThe ${test_name} is an offline test.\n";
	   print "User selected to run offline HBA test. \n";
	}

        $DMA_ERRORS = &check_dma_errors( $expected_dma_errors, $verbose );

        print "\n";
        print LOGHAND "\n ======== Start of ${test_name}:    ";
        print LOGHAND `/bin/date`,"\n";
        system "/usr/bin/date >> $ERRFILE";
        &faux_logger("INFO", 2012, "start of ${test_name} from ${PROGNAME}");

        $SIG{INT} = 'IGNORE';

        # start up lbf 

        my $lbf_loop_cnt = 5;

        print "\n*** Setting INTERNAL LOOPBACK for $port ***\n";
        `/usr/sbin/luxadm -e internal_loopback $port`;
        sleep 1;
        
	for ( my $tc=0; $tc < $lbf_loop_cnt; $tc++ ) {
           # $lbf_cmd = "$BINDIR/lbf -q -t ffffffff -n 1000 -k 64 -c $port 2>>$ERRFILE >> $LOGFILE.str.lbf.$CNUM{$port}";
             $lbf_cmd = "$BINDIR/lbf -t ffffffff -n 1000 -k 64 -c $port 2>>$ERRFILE >> $LOGFILE.str.hba_test.$CNUM{$port}";

            system "/usr/bin/ksh -c '${lbf_cmd} & /bin/echo \$! > /tmp/pid_$$'";
            #$lbf_cmd = "$BINDIR/lbf -q -t 00000000 -n 1000 -k 64 -c $port 2>>$ERRFILE >> $LOGFILE.str.lbf.$CNUM{$port}";
	    $lbf_cmd = "$BINDIR/lbf -t 00000000 -n 1000 -k 64 -c $port 2>>$ERRFILE >> $LOGFILE.str.hba_test.$CNUM{$port}";

            system "/usr/bin/ksh -c '${lbf_cmd} & /bin/echo \$! > /tmp/pid_$$'";
            $pid = `/bin/cat /tmp/pid_$$`;  

            chomp $pid;
            push(@pidList, $pid);
            `/usr/bin/rm /tmp/pid_$$`;
            &make_pid_file($LOGDIR, $CNUM{$port}, $pid);
            $monPid = monitor_log("$LOGFILE.str.hba_test.$CNUM{$port}");
            push(@MONITOR_PIDS, $monPid);

            $SIG{INT} = 'clean_up';
        
            # wait for all to complete
            while(wait2comp(@pidList)) {   
                sleep 5;
                print ".";
            }
        } # end of 5 loop
        print "\n";
        killMonitor();
        if ( $VIEW_LOGS eq $SHOW_LOGS ) { 
            sleep 1;
            system "/usr/bin/cat $LOGFILE.str.hba_test.$CNUM{$port}";
        }
        
	print "\n*** Resetting INTERNAL LOOPBACK for $port ***\n";
        `/usr/sbin/luxadm -e no_loopback $port`;
        sleep 1;
        print LOGHAND " ======== End of ${test_name}:      ";
        print "\n";
        print LOGHAND `/bin/date`, "\n";
        system "/bin/date >> $ERRFILE";
        &faux_logger("INFO", 2013, "end of ${test_name} from ${PROGNAME}");
        `/usr/bin/ksh -c '/usr/bin/kill $tailPid 2>/dev/null'`;
        `/usr/bin/ksh -c '/usr/bin/rm $LOGDIR/pids/tail.$tailPid 2>/dev/null'`;
        
        system "/bin/date >> $ERRFILE";
        &faux_logger("INFO", 2013, "${test_name} from ${PROGNAME} finished");
        print "\n";
        $verbose = 1;
        $dma_errors = &check_dma_errors( $DMA_ERRORS, $verbose );
        if ( $dma_errors ) {
            print "${RED}A scan of /var/adm/messages detected $dma_errors DMA error
(s).${RC}\n";
            print "${RED}Replace offending FC100 HBA immediately.${RC}\n";
        }

        $test_status = &sedr_summary_test( "hba_test", $test_name );
	print LOGHAND " ======== ${test_name} finished:    ";
        print LOGHAND `/bin/date`, "\n";
	if($test_status)
	{
           print LOGHAND " ======== Test Status: FAIL\n";
	}
	else
	{
           print LOGHAND " ======== Test Status: PASS\n";
	}

    } 
    elsif ( ( $HBA eq "21" ) || ( $HBA eq "22" ) )

    {

        # PCI Host Bus Adapter Test

        $test_name = "PCI HBA test";

        print "\n";
        print LOGHAND "\n ======== Start of ${test_name}:   ";
        print LOGHAND `/bin/date`,"\n";
        system "/usr/bin/date >> $ERRFILE";
        &faux_logger("INFO", 2012, "start of ${test_name} from ${PROGNAME}");
        $test_port = $port;
        if ( $test_port =~ /ifp/ ) 
	{
            $test_port =~ s/ifp\@(\d),0:0/ifp\@$1:devctl/;
        } 
	elsif ( $test_port =~ /scsi/ ) 
	{
            $test_port =~ s/scsi\@(\d),0:0/scsi\@$1:devctl/;
        }
        system "/usr/bin/ksh -c '$BINDIR/hba_test $test_port 2>>$ERRFILE >> $LOGFILE.str.hba_test.$CNUM{$port} & /bin/echo \$! > /tmp/pid_$$'";
        $SIG{INT} = 'clean_up';
    	$pid = `/bin/cat /tmp/pid_$$`;
	`/bin/rm /tmp/pid_$$`;

        # wait for all to complete
        while(wait2comp($pid)) 
	{   
            sleep 5;
            print ".";
        }
        print "\n";
        killMonitor();
        if ( $VIEW_LOGS eq $SHOW_LOGS ) { 
            sleep 1;
            system "/usr/bin/cat $LOGFILE.str.hba_test.$CNUM{$port}";
        }
        print LOGHAND " ======== End of ${test_name}:   ";
        print "\n";
        print LOGHAND `/bin/date`, "\n";
        system "/bin/date >> $ERRFILE";
        &faux_logger("INFO", 2013, "end of ${test_name} from ${PROGNAME}");
        `/usr/bin/ksh -c '/usr/bin/kill $tailPid 2>/dev/null'`;
        `/usr/bin/ksh -c '/usr/bin/rm $LOGDIR/pids/tail.$tailPid 2>/dev/null'`;
        
        system "/bin/date >> $ERRFILE";
        &faux_logger("INFO", 2013, "${test_name} from ${PROGNAME} finished");
        print "\n";
        $test_status = &sedr_summary_test( "hba_test", $test_name );
	print LOGHAND " ======== ${test_name} finished:   ";
	print LOGHAND `/bin/date`, "\n";
	if($test_status)
	{
           print LOGHAND " ======== Test Status: FAIL\n";
	}
	else
	{
           print LOGHAND " ======== Test Status: PASS\n";
	}

    
    }
    print "\nEND OF TESTING HBA on loop $port\n";

}



#run a loop test with a given port path

sub loop_test
{

   undef @pidList;


   $start_date = `/usr/bin/date '+%m/%d/%Y:%H:%M'`;
   chomp $start_date;


   &st_sync();
   print "\n\nBEGIN LOOP TEST\n";


   if ( ($HBA eq "S" ) || ($HBA eq "22"))
   {

        # SBUS Host data path test


LBF_DATA_PATH_TEST:


        $test_name = "Data Path test(lbf)";

        print "\n";
        print LOGHAND "\n ======== Start of ${test_name}:   ";
        print LOGHAND `/bin/date`,"\n";
        system "/usr/bin/date >> $ERRFILE";
        &faux_logger("INFO", 2012, "start of ${test_name} from ${PROGNAME}");

        $SIG{INT} = 'IGNORE';

        # start up lbf 

#        $lbf_cmd = "$BINDIR/lbf -q -a -k $LBF_KBYTES -n $LBF_ITERS $port 2>>$ERRFILE >> $LOGFILE.str.lbf.$CNUM{$port}";
        $lbf_cmd = "$BINDIR/lbf -a -k $LBF_KBYTES -n $LBF_ITERS $port 2>>$ERRFILE >$LOGFILE.str.lbf.$CNUM{$port}";
        system "/usr/bin/ksh -c '${lbf_cmd} & /bin/echo \$! > /tmp/pid_$$'";
        $pid = `/bin/cat /tmp/pid_$$`;  
        chomp $pid;
        push(@pidList, $pid);
        `/usr/bin/rm /tmp/pid_$$`;
        &make_pid_file($LOGDIR, $CNUM{$port}, $pid);
        $monPid = monitor_log("$LOGFILE.str.lbf.$CNUM{$port}");
        push(@MONITOR_PIDS, $monPid);

        $SIG{INT} = 'clean_up';
    
        # wait for all to complete
        while(wait2comp(@pidList)) {   
            sleep 5;
            print ".";
        }
        print "\n";
        killMonitor();
        if ( $VIEW_LOGS eq $SHOW_LOGS ) { 
            sleep 1;
            system "/usr/bin/cat $LOGFILE.str.lbf.$CNUM{$port}";
        }
        print LOGHAND " ======== End of ${test_name}:     ";
        print "\n";
        print LOGHAND `/bin/date`, "\n";
        system "/bin/date >> $ERRFILE";
        &faux_logger("INFO", 2013, "end of ${test_name} test from ${PROGNAME}");
        `/usr/bin/ksh -c '/usr/bin/kill $tailPid 2>/dev/null'`;
        `/usr/bin/ksh -c '/usr/bin/rm $LOGDIR/pids/tail.$tailPid 2>/dev/null'`;
        
        system "/bin/date >> $ERRFILE";
        &faux_logger("INFO", 2013, "${test_name} from ${PROGNAME} finished");
        print "\n";
        print "Scanning message logs for errors.\n";
        $messages = `$BINDIR/mess_sum  -q -s $start_date | grep errors`;
        if ( ! $messages ) {
            print "No errors detected in message logs.\n";
        }
        $test_status = &sedr_summary_test( "lbf", $test_name );
	print LOGHAND " ======== ${test_name} finished:   ";
        print LOGHAND `/bin/date`, "\n";
	if($test_status)
	{
           print LOGHAND " ======== Test Status: FAIL\n";
	}
	else
	{
           print LOGHAND " ======== Test Status: PASS\n";
	}



        if ( ( $test_status != 0 ) || ( $messages ) ) {
            if ( $test_status != 0 ) {
                print "   Test status: $test_status\n";
            } 
            if ( $messages ) {
               print "\nA scan of the systems logs detected errors.\n";
               print "Please inspect the log file to determine the cause of\n";
               print "error(s).\n";
               print "\nPlease press <return> to check system logs:";
               &get_response;
               system "$BINDIR/mess_sum -s $start_date";
            } else {
               print "\nA scan of the systems logs did not detect any errors.\n";
               print "Problem most likely a data path problem.\n";
            }
            &data_path_error();

            print "filtr is the best method for isolating data path FRUs.\n";
            print "Invoke filtr on $port [ <y>, n ]? ";
            $choice = &get_response( "y" );
            if ( $choice eq "y" ) {
                #&get_test_list( "lbf" );
                #system "${BINDIR}/filtr -slog -m $port";
		system "${BINDIR}/filtr -slog $port";

            } else {
                print "filtr skipped.\n";
            }
            print "rerun the data path test? [ <y>, n ]? ";
            $choice = &get_response( "y" );
            if ( $choice eq "y" ) {
	       
                goto LBF_DATA_PATH_TEST;
            }

            return;
        }

    } 
    elsif ( $HBA eq "21" ) 
    {

        # PCI Host data path test
        $test_name = "PCI Data Path test(dex)";

        print "\n";
        print LOGHAND "\n ======== Start of ${test_name}:   ";
        print LOGHAND `/bin/date`,"\n";
        system "/usr/bin/date >> $ERRFILE";

        &faux_logger("INFO", 2010, "start of ${test_name} $RD_CHOICE test from ${PROGNAME}");
        undef @pidList;
        $SIG{INT} = 'IGNORE';
        $dex_cnt = 0;

        #start up dex for each controller


	# Build up an array of drives so pass into sedr
	# get all t300 and a5k drives

	$SELECTED_DRIVES{$port} = $T300_DRIVES{$port};
        $SELECTED_DRIVES{$port} .= " $A5000_DRIVES{$port}" if ($A5000_DRIVES{$port} ne "");

	#Must check to see if alternate or primary drives were selected
        @drives = split / /, $SELECTED_DRIVES{ $port };
        $SELECTED_DRIVES{ $port } = "";
        foreach $drive ( @drives ) 
	{
             if (($T300{$drive} eq "p") && ($DUAL_PORT eq $PRIMARY)) 
	     {
                 ############################################
                 # Primary drive and we are in primary mode #
                 # so allow testing to it.                  #
                 ############################################
                 $SELECTED_DRIVES{ $port } .= $drive;
                 $SELECTED_DRIVES{ $port } .= " ";
             }
	     elsif (($T300{$drive} eq "a") && ($DUAL_PORT eq $ALTERNATE)) 
	     {
                 ################################################
                 # Alternate drive and we are in alternate mode #
                 # so allow testing to it.                      #
                 ################################################
                 $SELECTED_DRIVES{ $port } .= $drive;
                 $SELECTED_DRIVES{ $port } .= " ";
             } 
	     elsif ($T300{$drive} eq "") 
	     {
                 ############################################
                 # Not a T300 drive, so allow testing to it #
                 ############################################
                 $SELECTED_DRIVES{ $port } .= $drive;
                 $SELECTED_DRIVES{ $port } .= " ";
             } else 
	     {
                 print "Skipping drive $drive";
                 print " (primary T300 LUN)" if (($T300{$drive} eq "p") && ($DUAL_PORT eq $ALTERNATE));
                 print " (alternate T300 LUN)" if (($T300{$drive} eq "a") && ($DUAL_PORT eq $PRIMARY));
                 print "\n";
             }
        }

        @drives = split / /, $SELECTED_DRIVES{ $port };

        if ( ! @drives ) {
            print "\n";
            print "Program ERROR!\n";
            print "Invalid Dual Port Mode: $DUAL_PORT for $CNUM{ $port}\n";
	    print "Exiting program \n";
            exit(-1);
        }
#        $dex_cmd = "$BINDIR/seq_dex $CNUM{$port} $RD_CHOICE $CON_RUN_TIME  @drives 2>>$ERRFILE >> $LOGFILE.str.dex.${dex_cnt}";
        $dex_cmd = "$BINDIR/seq_dex $CNUM{$port} $RD_CHOICE $CON_RUN_TIME  @drives 2>>$ERRFILE >> $LOGFILE.str.lbf.$CNUM{$port}";

        system "/usr/bin/ksh -c '${dex_cmd} & /bin/echo \$! > /tmp/pid_$$'";
        $pid = `/bin/cat /tmp/pid_$$`;  
        push(@pidList, $pid);
        `/bin/rm /tmp/pid_$$`;
        chomp $pid;
        &make_pid_file($LOGDIR, $CNUM{$port}, $pid);
        $dex_cnt += 1;

        $SIG{INT} = 'clean_up';

        # wait for all to complete
        while(wait2comp(@pidList)) {   
            sleep 5;
            print ".";
        }
        print "\n";
        killMonitor();
        if ( $VIEW_LOGS eq $SHOW_LOGS ) { 
            sleep 1;
            system "/usr/bin/cat $LOGFILE.str.lbf.$CNUM{$port}";
        }
        print LOGHAND " ======== End of ${test_name}:   ";
        print LOGHAND `/bin/date`, "\n";
        system "/bin/date >> $ERRFILE";
        print "\n";
        &faux_logger("INFO", 2011, "end of ${test_name} $RD_CHOICE test from ${PROGNAME}");
        `/usr/bin/ksh -c '/usr/bin/kill $tailPid 2>/dev/null'`;
        `/usr/bin/ksh -c '/usr/bin/rm $LOGDIR/pids/tail.$tailPid 2>/dev/null'`;
        
        system "/bin/date >> $ERRFILE";
        &faux_logger("INFO", 2011, "${test_name} $RD_CHOICE test from ${PROGNAME} finished");
        print "\n";
        print "Scanning message logs for errors.\n";
        $messages = `$BINDIR/mess_sum  -q -s $start_date | grep errors`;
        if ( ! $messages ) {
            print "No errors detected in message logs.\n";
        }
        $test_status = &sedr_summary_test( "lbf", $test_name );
	print LOGHAND " ======== ${test_name} finished:   ";
        print LOGHAND `/bin/date`, "\n";
	if($test_status)
	{
           print LOGHAND " ======== Test Status: FAIL\n";
	}
	else
	{
           print LOGHAND " ======== Test Status: PASS\n";
	}

        if ( ( $test_status != 0 ) || ( $messages ) ) {
            if ( $test_status != 0 ) {
                print "   Test status: $test_status\n";
            } 
            if ( $messages ) {
               print "\nA scan of the system logs detected errors.\n";
               print "Please inspect the log file to determine the cause of\n";
               print "error(s). Possible ${STORAGE} problem.\n";
               print "\nPlease press <return> to check system logs:";
               &get_response;
               system "$BINDIR/mess_sum  -s $start_date";
            } else {
               print "\nA scan of the system logs did not detect any errors.\n";
               print "Problem most likely a data path problem.\n";
               &data_path_error();
            }
            return;
        }

    }
    print "END OF LOOP TEST\n\n";
}


# run a storage test on a given port path
sub storage_test
{

    
    &st_sync();

    print "START STORAGE TEST \n";
    $test_name = "Storage test(dex)";

    $start_date = `/usr/bin/date '+%m/%d/%Y:%H:%M'`;
    chomp $start_date;

    print "\n";
    print LOGHAND "\n ======== Start of ${test_name}:     ";
    print LOGHAND `/bin/date`,"\n";
    system "/usr/bin/date >> $ERRFILE";

    &faux_logger("INFO", 2010, "start of ${test_name} $RD_CHOICE test from ${PROGNAME}");
    undef @pidList;
    $SIG{INT} = 'IGNORE';
    $dex_cnt = 0;

    # Build up an array of drives so pass into sedr
    # get all t300 and a5k drives

    $SELECTED_DRIVES{$port} = $T300_DRIVES{$port};
    $SELECTED_DRIVES{$port} .= " $A5000_DRIVES{$port}" if ($A5000_DRIVES{$port} ne "");

    #Must check to see if alternate or primary drives were selected
    @drives = split / /, $SELECTED_DRIVES{ $port };
    $SELECTED_DRIVES{ $port } = "";
    foreach $drive ( @drives ) 
    {
         if (($T300{$drive} eq "p") && ($DUAL_PORT eq $PRIMARY)) 
         {
             ############################################
             # Primary drive and we are in primary mode #
             # so allow testing to it.                  #
             ############################################
             $SELECTED_DRIVES{ $port } .= $drive;
             $SELECTED_DRIVES{ $port } .= " ";
         }
         elsif (($T300{$drive} eq "a") && ($DUAL_PORT eq $ALTERNATE)) 
         {
             ################################################
             # Alternate drive and we are in alternate mode #
             # so allow testing to it.                      #
             ################################################
             $SELECTED_DRIVES{ $port } .= $drive;
             $SELECTED_DRIVES{ $port } .= " ";
         } 
         elsif ($T300{$drive} eq "") 
         {
             ############################################
             # Not a T300 drive, so allow testing to it #
             ############################################
             $SELECTED_DRIVES{ $port } .= $drive;
             $SELECTED_DRIVES{ $port } .= " ";
         } else 
         {
             print "Skipping drive $drive";
             print " (primary T300 LUN)" if (($T300{$drive} eq "p") && ($DUAL_PORT eq $ALTERNATE));
             print " (alternate T300 LUN)" if (($T300{$drive} eq "a") && ($DUAL_PORT eq $PRIMARY));
             print "\n";
         }
    }

    @drives = split / /, $SELECTED_DRIVES{ $port };

    if ( ! @drives ) {
        print "\n";
        print "Program ERROR!\n";
        print "Invalid Dual Port Mode: $DUAL_PORT for $CNUM{ $port}\n";
	print "Exiting program. \n";
        exit(-1);
    }

    $dex_cmd = "$BINDIR/seq_dex $CNUM{$port} $RD_CHOICE $DEX_RUN_TIME  @drives 2>>$ERRFILE >> $LOGFILE.str.dex.${dex_cnt}";
    system "/usr/bin/ksh -c '${dex_cmd} & /bin/echo \$! > /tmp/pid_$$'";
    $pid = `/bin/cat /tmp/pid_$$`;  
    push(@pidList, $pid);
    `/bin/rm /tmp/pid_$$`;
    chomp $pid;
    &make_pid_file($LOGDIR, $CNUM{$port}, $pid);
    $dex_cnt += 1;

    $SIG{INT} = 'clean_up';

    # wait for all to complete
    while(wait2comp(@pidList)) {   
        sleep 5;
        print ".";
    }
    print "\n";

    killMonitor();
    if ( $VIEW_LOGS eq $SHOW_LOGS ) { 
        sleep 1;
        for ( $cnt=0; $cnt < $dex_cnt; $cnt++ ) {
            system "/usr/bin/cat $LOGFILE.str.dex.${cnt}";
        }
    }

    print LOGHAND " ======== End of ${test_name}:       ";
    print LOGHAND `/bin/date`, "\n";
    system "/bin/date >> $ERRFILE";
    print "\n";
    &faux_logger("INFO", 2011, "end of ${test_name} $RD_CHOICE test from ${PROGNAME}");
    `/usr/bin/ksh -c '/usr/bin/kill $tailPid 2>/dev/null'`;
    `/usr/bin/ksh -c '/usr/bin/rm $LOGDIR/pids/tail.$tailPid 2>/dev/null'`;

    system "/bin/date >> $ERRFILE";
    &faux_logger("INFO", 2011, "${test_name} $RD_CHOICE test from ${PROGNAME} finished");
    print "\n";
    print "Scanning message logs for errors.\n";
    $messages = `$BINDIR/mess_sum  -q -s $start_date | grep errors`;
    if ( ! $messages ) {
        print "No errors detected in message logs.\n";
    }
    $test_status = &sedr_summary_test( "dex", $test_name );
    
    print LOGHAND " ======== ${test_name} finished:     ";
    print LOGHAND `/bin/date`, "\n";
    if($test_status)
    {
       print LOGHAND " ======== Test Status: FAIL\n";
    }
    else
    {
       print LOGHAND " ======== Test Status: PASS\n";
    }

    if ( ( $test_status != 0 ) || ( $messages ) ) {
        if ( $test_status != 0 ) {
            print "   Test status: $test_status\n";
        } 
        if ( $messages ) {
           print "\nA scan of the system logs detected errors.\n";
           print "Please inspect the log file to determine the cause of\n";
           print "error(s). Possible ${STORAGE} problem.\n";
           print "\nPlease press <return> to check system logs:";
           &get_response;
           system "$BINDIR/mess_sum  -s $start_date";
        } else {
           print "\nA scan of the system logs did not detect any errors.\n";
           print "Problem most likely a data path problem.\n";
           &data_path_error();
        }
    }
    print "\n\nEND OF STORAGE TEST \n";
    return;
}  




###############################################################################
#
# Set what type of HBA port this is.
# possible values are "S" for Socal or Usocal
#                     "21" for PCI 2100 board
#                     "22" for PCI 2202 board
###############################################################################

sub set_hba_port_type
{
   if ($port =~ /socal/ )
   {
      $HBA = "S" ;
   }
   elsif ($port =~ /usoc/ )
   {
      $HBA = "S" ;
   }
   elsif ($port =~ /qlc/ )
   {
      # PCI 2202 board
      $HBA = "22" ;
   }
   elsif ($port =~ /ifp/ )
   {
      # PCI 2100 board
      $HBA = "21" ;
   }
   else
   {
      print "Can't determine Host Bus Adapter type from path: \n";
      print "$port\n";
      print "Exiting program \n";
      exit;
   }
   return;

} # end set hba port type


###############################################################################
#
# Set storage type given a global port path
#   possible return values are T300 or A5000
#
###############################################################################

sub set_storage_type
{
   if ($A5000_DRIVES{$port}) 
   {
      $STORAGE = "A5000" ;
      $DUAL_PORT = $NONE;

   }
   if ($T300_DRIVES{$port}) 
   {
      $STORAGE = "T300" ;
      $DUAL_PORT = $MENU_DUAL_PORT;
   }
   if ($OTHER_DRIVES{$port}) 
   {
       print "SEDR doesn't run on this type of port $port \n";
       exit;
   }
   return;
} # end set storage type
###############################################################################


# this subroutine will loop through all loops and will explore to see what
# menu option variables need to be 
sub set_menu_options
{
   $SHOW_T300 = "";
   $SHOW_LBF_ITERS = "";
   $SHOW_CON_RUN_TIME = "";
   $SHOW_OFFLINE_HBA = "";

   foreach $port (@TEST_LP_LIST) 
   {
      if ($T300_DRIVES{$port}) 
      {
         $SHOW_T300 = "m," ;
	 if($DUAL_PORT eq $NONE)
	 {
	    $DUAL_PORT = $PRIMARY;
	 }

      }

      if ($port =~ /socal/ )
      {
	 # Can run lbf, so show lbf iters.
         $SHOW_LBF_ITERS = "c," ;
	 $SHOW_OFFLINE_HBA = "o," ;
      }
      elsif ($port =~ /usoc/ )
      {
	 # Can run lbf, so show lbf iters.
	 $SHOW_OFFLINE_HBA = "o," ;
         $SHOW_LBF_ITERS = "c," ;

      }
      elsif ($port =~ /qlc/ )
      {
         # PCI 2200 board
	 # Can run lbf, so show lbf iters.
	 $SHOW_LBF_ITERS = "c," ;

      }
      elsif ($port =~ /ifp/ )
      {
         # PCI 2100 board
	 $SHOW_CON_RUN_TIME = "t," ;
      }


   }# end for each
} # end set menu options 
###############################################################################


###############################################################################
#
#  summary test: display test results.
#
###############################################################################



sub sedr_summary_test {

   my $which_test = $_[0];
   my $test_name = $_[1];

   my %RC;
   my @summary;
   my $lp; 
   my $grlp;
   $RETCODE = 0;

   @summary = ("\n","$test_name Summary\n\n");

   $grlp = `/usr/bin/ksh -c '/bin/egrep -i fail $LOGFILE.str.${which_test}.$CNUM{$port} 2> /dev/null'`;
   if ($grlp eq "") {
       $RC{$port} = "PASS";
   } else {
       $RC{$port} = "++++ FAIL ++++";
       $RETCODE = -1;
   }
   @summary = (@summary, "LOOP: ($CNUM{$port}) $port \t $RC{$port} \n");

   print @summary;
   if ($RETCODE == 0) {
       print "${GREEN} ****** Test Passed ****** ${RC}\n";
   } else {
       print "${RED} ****** Test FAILED ****** ${RC}\n";
   }
   return $RETCODE;

}


