#!/usr/bin/env perl # # Licensed Materials - Property of IBM # # 2145-VC6, 2076-VM1 # # Copyright IBM Corp. 2011, All Rights Reserved. # # US Government Users Restricted Rights - Use, duplication or # disclosure restricted by GSA ADP Schedule Contract with # IBM Corp. # #======================================================================= # # Description: # # Attempt to recover "running" status of any SVC (IBM 2145) paths found # to be "offline" or missing, based on conf file from earlier invocation # #======================================================================= # use strict; #----------------------------------------------------------------------- # enhanced cache flushing #----------------------------------------------------------------------- $| = 1; #----------------------------------------------------------------------- # other global variables #----------------------------------------------------------------------- my $debug = 0; my $confile = "/etc/svc_health_check.conf"; my $logfile = "/var/log/svc_health_check.log"; #----------------------------------------------------------------------- # main #----------------------------------------------------------------------- sub main { my $prodcodeRef = shift; my $func = "main"; #------------------------------------------------------------------- # processing started - write 'started' message to log #------------------------------------------------------------------- syslogwrite( "svc_health_check (${$prodcodeRef}) started ..." ); #------------------------------------------------------------------- # identify all device UIDs, prodcodes and details from conf file #------------------------------------------------------------------- my %confUidPaths = (); my %confPathDtls = (); get_conf_dtls( \%confUidPaths, \%confPathDtls ); #------------------------------------------------------------------- # identify all device UIDs, prodcodes and details from 'multipathd' #------------------------------------------------------------------- my %uidPaths = (); my %pathDtls = (); get_curr_dtls( $prodcodeRef, \%uidPaths, \%pathDtls ); #------------------------------------------------------------------- # compare current situation with prior situation (from conf file) # update_conf: flag to indicate whether conf file requires update #------------------------------------------------------------------- my $update_conf = compare_bef_now_devs( $prodcodeRef, \%confUidPaths, \%confPathDtls, \%uidPaths, \%pathDtls ); #------------------------------------------------------------------- # write conf file according to current situation, if changed # update_conf: flag to indicate whether conf file requires update #------------------------------------------------------------------- write_conf_file( $prodcodeRef, \%uidPaths, \%pathDtls, $update_conf ); #------------------------------------------------------------------- # processing complete - write 'complete' message to log #------------------------------------------------------------------- syslogwrite( "svc_health_check (${$prodcodeRef}) ... completed" ); return( 0 ); } #======================================================================= # # Generic functions # #======================================================================= #----------------------------------------------------------------------- # identify device UIDs, prodcodes, etc from prior run's conf file #----------------------------------------------------------------------- sub get_conf_dtls { my $confUidPathsRef = shift; my $confPathDtlsRef = shift; my $func = "get_conf_dtls"; printdbg( ); if ( -r $confile ) { printdbg( \$func, "Collecting dev paths from conf file, '$confile':" ); open( CFILE, "<$confile" ) or abort_check( \$confile, "open input" ); #--------------------------------------------------------------- # eg conf file data (UID,hcil,dev): # 360050768019181c8b000000000000008,4:0:0:0,sda # 360050768019181c8b000000000000008,4:0:1:0,sdc # 360050768019181c8b000000000000008,5:0:0:0,sdr #--------------------------------------------------------------- foreach my $line ( ) { chomp( $line ); if ( my ( $uid, $hcil, $dev ) = $line =~ /^(\w+),(\d+:\d+:\d+:\d+),([^,\s]+)$/ ) { printdbg( \$func, "uid='$uid' dev='$dev' hcil='$hcil'" ); push( @{${$confUidPathsRef}{$uid}{paths}}, $dev ); ${$confPathDtlsRef}{$dev}{uid} = $uid; ${$confPathDtlsRef}{$dev}{hcil} = $hcil; } } close( CFILE ) or abort_check( \$confile, "close input" ); syslogwrite( sprintf( " NOTE: Sourced device path list from file, " . "'$confile' - %d devices", scalar( keys( %{$confUidPathsRef} ) ) ) ); } else { printdbg( \$func, "There is no conf file, '$confile'" ); } return( 0 ); } #----------------------------------------------------------------------- # identify device UIDs, prodcodes, etc from 'multipathd' command line #----------------------------------------------------------------------- sub get_curr_dtls { my $prodcodeRef = shift; my $uidPathsRef = shift; my $pathDtlsRef = shift; my $func = "get_curr_dtls"; #------------------------------------------------------------------- # # multipathd -k'show paths format "%w,%s,%i,%d,%t,%o,%T"' # uuid ,vend/prod/rev ,hcil ,dev ,dm_st ,dev_st ,chk_st # 3600507630affc0710000000000000328,IBM,2107900 ,1:0:0:0,sdh ,active,running,ready # 3600507630affc0710000000000000329,IBM,2107900 ,1:0:0:1,sdi ,active,running,ready # 36005076303ffc7480000000000000030,IBM,2107900 ,1:0:1:0,sdj ,active,running,ready # 36005076303ffc7480000000000000031,IBM,2107900 ,1:0:1:1,sdk ,active,running,ready # 360050768019181c8b000000000000006,IBM,2145 ,1:0:7:0,sdaa,active,running,ready # 36005076303ffc7480000000000000030,IBM,2107900 ,0:0:6:0,sdae,active,running,ready # 36005076303ffc7480000000000000031,IBM,2107900 ,0:0:6:1,sdaf,active,running,ready # 360050768019181c8b000000000000007,IBM,2145 ,1:0:7:1,sdac,active,running,ready # 360050768019181c8b000000000000007,IBM,2145 ,1:0:6:1,sdz ,active,running,ready # 360050768019181c8b000000000000006,IBM,2145 ,1:0:6:0,sdy ,active,running,ready # Key: # hcil logical host (hba), controller, lun id, bus (unused) # dm_st path state from kernel: 'active' or 'failed' # dev_st path state from multipathd: 'running', 'faulty', ... # chk_st unsure: 'ready', ... #------------------------------------------------------------------- my $command = "/sbin/multipathd -k'show paths format \"%w,%s,%i,%d,%t,%o,%T\"'"; my $cmd_failed = 0; printdbg( ); printdbg( \$func, "Collecting dev paths from live system (using: $command) ..." ); open ( MULTIPATHD, "$command 2>&1 |" ); #------------------------------------------------------------------- # loop over multipathd -k'show paths format ...' lines #------------------------------------------------------------------- foreach my $line ( ) { if ( $line =~ /ux_socket_connect: Connection refused/i or $line =~ /multipath-tools/i or $line =~ /CLI commands reference/i ) { $cmd_failed = 1; last; } chomp( $line ); #--------------------------------------------------------------- # split values to fields, removing any trailing space characters #--------------------------------------------------------------- my ( $uid, $vend, $prod, $hcil, $dev, $dm_st, $dev_st, $chk_st ) = split( / *, */, $line ); if ( $uid !~ /^uuid/ ) { printdbg( \$func, sprintf "'%s' '%s' '%s' '%s' '%s' '%s' ('%s' '%s')", $uid, $dev, $hcil, $dm_st, $dev_st, $chk_st, $vend, $prod ); ${$uidPathsRef}{$uid}{vend} = $vend; ${$uidPathsRef}{$uid}{prod} = $prod; push( @{${$uidPathsRef}{$uid}{paths}}, $dev ); ${$pathDtlsRef}{$dev}{uid} = $uid; ${$pathDtlsRef}{$dev}{hcil} = $hcil; ${$pathDtlsRef}{$dev}{dm_st} = $dm_st; ${$pathDtlsRef}{$dev}{dev_st} = $dev_st; ${$pathDtlsRef}{$dev}{chk_st} = $chk_st; } } #------------------------------------------------------------------- # loop over multipathd -k'show paths format ...' lines #------------------------------------------------------------------- close( MULTIPATHD ); if ( $cmd_failed ) { printdbg( \$func, "Failed to collect dev paths from live system (using: $command)" ); get_curr_dtls_alt( $prodcodeRef, $uidPathsRef, $pathDtlsRef ); } return( 0 ); } #----------------------------------------------------------------------- # identify device UIDs, etc from alternative 'multipathd' command line #----------------------------------------------------------------------- sub get_curr_dtls_alt { my $prodcodeRef = shift; my $uidPathsRef = shift; my $pathDtlsRef = shift; my $func = "get_curr_dtls_alt"; #------------------------------------------------------------------- # # multipathd -k'show multipaths topology' # mpathe (360050768019181c8b00000000000002b) dm-20 IBM,2145 # [size=2.0G][features=0 ][hwhandler=0 ] # \_ round-robin 0 [prio=100][active] # \_ 1:0:0:1 sdh 8:112 [active][ready] # \_ 0:0:0:1 sdb 8:16 [active][ready] # \_ round-robin 0 [prio=20][enabled] # \_ 1:0:1:1 sdj 8:144 [active][ready] # \_ 0:0:1:1 sdd 8:48 [active][ready] #------------------------------------------------------------------- my $command = "/sbin/multipathd -k'show multipaths topology'"; printdbg( ); printdbg( \$func, "Collecting dev paths from live system (using: $command) ..." ); my ( $uid, $vend, $prod ) = undef; open ( MULTIPATHD, "$command 2>&1 |" ); #------------------------------------------------------------------- # loop over multipathd -k'show multipaths topology' lines #------------------------------------------------------------------- foreach my $line ( ) { chomp( $line ); #--------------------------------------------------------------- # decipher the fields from the multipathd output #--------------------------------------------------------------- if ( $line =~ /^\S+\s\((\w+)\)\s+\S+\s+(\S+),(\w+)/ ) { $uid = $1; ${$uidPathsRef}{$uid}{vend} = $2; ${$uidPathsRef}{$uid}{prod} = $3; } elsif ( defined( $uid ) ) { my ( $hcil, $dev, $dm_st, $dev_st, $chk_st ) = undef; if ( $line =~ /^\s..\s(\d+:\d+:\d+:\d+)\s(\S+)\s+\d+:\d+\s+\[(\w+)\]\[(\w+)\]/ ) { ( $hcil, $dev, $dm_st, $dev_st ) = ( $1, $2, $3, $4, $5 ); } elsif ( $line =~ /^...\s(\d+:\d+:\d+:\d+)\s(\S+)\s+\d+:\d+\s+\[?(\w+)[\[\]\s]+(\w+)/ ) { ( $hcil, $dev, $dm_st, $dev_st ) = ( $1, $2, $3, $4, $5 ); } elsif ( $line =~ /^....\s(\d+:\d+:\d+:\d+)\s(\S+)\s+\d+:\d+\s+(\w+)s(\w+)\s(\w+)/ ) { ( $hcil, $dev, $dm_st, $dev_st, $chk_st ) = ( $1, $2, $3, $4, $5, $6 ); } if ( defined( $hcil ) ) { printdbg( \$func, sprintf "'%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s'", $uid, ${$uidPathsRef}{$uid}{vend}, ${$uidPathsRef}{$uid}{prod}, $hcil, $dev, $dm_st, $dev_st, $chk_st ); push( @{${$uidPathsRef}{$uid}{paths}}, $dev ); ${$pathDtlsRef}{$dev}{uid} = $uid; ${$pathDtlsRef}{$dev}{hcil} = $hcil; ${$pathDtlsRef}{$dev}{dm_st} = $dm_st; ${$pathDtlsRef}{$dev}{dev_st} = $dev_st; ${$pathDtlsRef}{$dev}{chk_st} = $chk_st; } } } #------------------------------------------------------------------- # loop over multipathd -k'show multipaths topology' lines #------------------------------------------------------------------- close( MULTIPATHD ); return( 0 ); } #----------------------------------------------------------------------- # compare current with any prior situation, per conf file contents # # recovery conditions handled: # # A : One or more statuses (dm_st) are not "active": # = A path has lost "good" status and should be recovered # > Write 'running' to the sysfs block device state file # + No change to conf file # # B : One or more hcils have changed: # = HBA module reloaded - sdX may have changed, too # > Check path state, as in A, above # + Conf file needs to be updated, accordingly # # C : One or more paths are missing: # = Path has been removed by OS # > Write 'c i l" to host 'h' scan file for this missing path # + Conf file needs to reflect the intended good state # # D : An entire disk is missing, ie all paths are gone: # = Device has been removed by OS # > Write 'c i l" to host 'h' scan file for each missing path # + Conf file needs to reflect the intended good state # #----------------------------------------------------------------------- sub compare_bef_now_devs { my $prodcodeRef = shift; my $confUidPathsRef = shift; my $confPathDtlsRef = shift; my $uidPathsRef = shift; my $pathDtlsRef = shift; my $update_conf = 0; # flag to indicate whether conf file requires update my $func = "compare_bef_now_devs"; printdbg( ); printdbg( \$func, "Compare conf file with live and attempt recoveries of bad paths ..." ); my %already_processed = (); #------------------------------------------------------------------- # loop over conf file data, to confirm no losses #------------------------------------------------------------------- foreach my $uid ( sort( keys( %{$confUidPathsRef} ), keys( %{$uidPathsRef} ) ) ) { #--------------------------------------------------------------- # skip repeat instances of uid, seen before and now and ... # be sure it's the required product #--------------------------------------------------------------- if ( defined( $already_processed{$uid} ) || ${$uidPathsRef}{$uid}{prod} ne ${$prodcodeRef} ) { ; # nothing to do } else { printdbg( ); printdbg( \$func, "="x79 ); #----------------------------------------------------------- # can the uid still be seen? # yes, the uid still exists # no, the uid is no longer seen - attempt all path recovery #----------------------------------------------------------- if ( defined( @{${$uidPathsRef}{$uid}{paths}} ) && scalar( @{${$uidPathsRef}{$uid}{paths}} ) ) { $update_conf += current_device( $prodcodeRef, $confUidPathsRef, $confPathDtlsRef, $uidPathsRef, $pathDtlsRef, \$uid); } else { printdbg( \$func, "uid $uid (${$prodcodeRef}) disk no longer seen - Recover ..." ); #------------------------------------------------------- # D : An entire disk is missing, ie all paths are gone: # attempt recovery of each of the old device paths # method: echo "c i l" to hostN's scan file #------------------------------------------------------- for my $dev ( @{${$confUidPathsRef}{$uid}{paths}} ) { recover_device( \$uid, \${$confPathDtlsRef}{$dev}{hcil} ); $update_conf ++; } } #----------------------------------------------------------- # can the uid still be seen or not? #----------------------------------------------------------- $already_processed{$uid} = 1; printdbg( \$func, "="x79 ); } #--------------------------------------------------------------- # skip repeat instances of uid, seen before and now #--------------------------------------------------------------- } #------------------------------------------------------------------- # loop over conf file data, to confirm no losses #------------------------------------------------------------------- return( $update_conf ); } #----------------------------------------------------------- # device is currently seen #----------------------------------------------------------- sub current_device { my $prodcodeRef = shift; my $confUidPathsRef = shift; my $confPathDtlsRef = shift; my $uidPathsRef = shift; my $pathDtlsRef = shift; my $uidRef = shift; my $update_conf = 0; # flag to indicate whether conf file requires update my $func = "current_device"; printdbg( \$func, "uid ${$uidRef} (${$prodcodeRef}) disk is currently seen" ); #------------------------------------------------------- # sort the paths record for current device (uid) # count the paths of current device #------------------------------------------------------- @{${$uidPathsRef}{${$uidRef}}{paths}} = ( sort { $a<=>$b || length $a <=> length $b || $a cmp $b } ( @{${$uidPathsRef}{${$uidRef}}{paths}} ) ); #------------------------------------------------------- # has the uid been seen before? # yes, this uid comes from the conf file # no, this uid is new - check its paths #------------------------------------------------------- if ( defined( @{${$confUidPathsRef}{${$uidRef}}{paths}} ) && scalar( @{${$confUidPathsRef}{${$uidRef}}{paths}} ) ) { printdbg( \$func, "uid ${$uidRef} disk was seen before:" ); printdbg( \$func, "--> Paths (prev): @{${$confUidPathsRef}{${$uidRef}}{paths}}" ); printdbg( \$func, "--> Paths (curr): @{${$uidPathsRef}{${$uidRef}}{paths}}" ); if ( "@{${$confUidPathsRef}{${$uidRef}}{paths}}" eq "@{${$uidPathsRef}{${$uidRef}}{paths}}" ) { printdbg( \$func, "uid ${$uidRef} paths matched" ); } else { printdbg( \$func, "uid ${$uidRef} paths changed" ); } $update_conf += compare_bef_now_dev_paths( $prodcodeRef, $confUidPathsRef, $confPathDtlsRef, $uidPathsRef, $pathDtlsRef, $uidRef ); } else { printdbg( \$func, "uid ${$uidRef} not seen before - Check path statuses" ); for my $dev ( @{${$uidPathsRef}{${$uidRef}}{paths}} ) { check_dev_path_states( $pathDtlsRef, $uidRef, \$dev ); $update_conf ++; } } #------------------------------------------------------- # has the uid been seen before? #------------------------------------------------------- return( $update_conf ); } #----------------------------------------------------------------------- # compare the paths and their statuses of a given device, before and now #----------------------------------------------------------------------- sub compare_bef_now_dev_paths { my $prodcodeRef = shift; my $confUidPathsRef = shift; my $confPathDtlsRef = shift; my $uidPathsRef = shift; my $pathDtlsRef = shift; my $uidRef = shift; my $func = "compare_bef_now_dev_paths"; my $update_conf = 0; # flag to indicate whether conf file requires update my $path_reduce = 0; # flag to indicate whether device paths have reduced my $bef_path_cnt = scalar( @{${$confUidPathsRef}{${$uidRef}}{paths}} ); my $now_path_cnt = scalar( @{${$uidPathsRef}{${$uidRef}}{paths}} ); if ( $bef_path_cnt == $now_path_cnt ) { printdbg( \$func, "uid ${$uidRef} path count unchanged: $bef_path_cnt" ); } elsif ( $bef_path_cnt < $now_path_cnt ) { printdbg( \$func, "uid ${$uidRef} path count increased: $bef_path_cnt < $now_path_cnt" ); } else { printdbg( \$func, "uid ${$uidRef} path count decreased: $bef_path_cnt > $now_path_cnt" ); $path_reduce = 1; } #------------------------------------------------------------------- # loop over the device paths seen before and currently ... # ${$pathDtlsRef}{$dev}{uid} = # ${$pathDtlsRef}{$dev}{hcil} = # ${$pathDtlsRef}{$dev}{dm_st} = # ${$pathDtlsRef}{$dev}{dev_st} = # ${$pathDtlsRef}{$dev}{chk_st} = #------------------------------------------------------------------- my %already_processed = (); foreach my $dev ( sort { $a<=>$b || length $a <=> length $b || $a cmp $b } ( @{${$confUidPathsRef}{${$uidRef}}{paths}}, @{${$uidPathsRef}{${$uidRef}}{paths}} ) ) { #--------------------------------------------------------------- # avoid repetition of devs common to both before and current #--------------------------------------------------------------- if ( defined( $already_processed{$dev} ) ) { ; # nothing to do } else { $already_processed{$dev} = 1; printdbg( \$func, "-"x79 ); #----------------------------------------------------------- # was this device/uid path recorded in the conf file? # yes, this was also an old path - compare dtls with current # no, this is a new path, not seen before #----------------------------------------------------------- if ( defined( ${$confPathDtlsRef}{$dev}{uid} ) && ${$confPathDtlsRef}{$dev}{uid} eq ${$uidRef} ) { $update_conf += compare_dev_path_dtls( $confPathDtlsRef, $pathDtlsRef, $uidRef, \$dev ); } else { printdbg( \$func, "New ${$uidRef} path $dev - Path not seen before ..." ); #------------------------------------------------------- # is the prodcode for this current uid of interest? # yes, need to check paths and recover any that are bad # no, this uid is not required for recovery - ignore #------------------------------------------------------- if ( ${$uidPathsRef}{${$uidRef}}{prod} eq ${$prodcodeRef} ) { printdbg( \$func, "New ${$uidRef} path $dev - Is ${$prodcodeRef} ..." ); check_dev_path_states( $pathDtlsRef, $uidRef, \$dev ); $update_conf ++; } else { printdbg( \$func, "New ${$uidRef} path $dev - " . "${$uidPathsRef}{${$uidRef}}{prod} NOT " . "${$prodcodeRef} - Ignore" ); } #------------------------------------------------------- # is the prodcode for this current uid of interest? #------------------------------------------------------- } #----------------------------------------------------------- # was this device/uid path recorded in the conf file? #----------------------------------------------------------- } #--------------------------------------------------------------- # avoid repetition of devs common to both before and current #--------------------------------------------------------------- } return( $update_conf ); } #----------------------------------------------------------------------- # compare the paths, before and now #----------------------------------------------------------------------- sub compare_dev_path_dtls { my $confPathDtlsRef = shift; my $pathDtlsRef = shift; my $uidRef = shift; my $devRef = shift; my $func = "compare_dev_path_dtls"; my $update_conf = 0; #------------------------------------------------------------------- # is this path to the device still being seen, and is same uid? # yes, same path as seen before - check the hcil and status # no, no longer seen - lost #------------------------------------------------------------------- if ( defined( %{${$pathDtlsRef}{${$devRef}}} ) && scalar( keys( %{${$pathDtlsRef}{${$devRef}}} ) ) && ${$pathDtlsRef}{${$devRef}}{uid} eq ${$uidRef} ) { printdbg( \$func, "Old ${$uidRef} path ${$devRef} - Path still seen" ); #--------------------------------------------------------------- # compare the path dtls to identify any changes # ${$pathDtlsRef}{$dev}{uid} = # ${$pathDtlsRef}{$dev}{hcil} = # ${$pathDtlsRef}{$dev}{dm_st} = # ${$pathDtlsRef}{$dev}{dev_st} = # ${$pathDtlsRef}{$dev}{chk_st} = # if hcil has changed, we just need to remember to write it to conf # if dm_st is not 'active', recover it #--------------------------------------------------------------- if ( ${${$confPathDtlsRef}{${$devRef}}}{hcil} eq ${${$pathDtlsRef}{${$devRef}}}{hcil} ) { printdbg( \$func, "o-- Path ${$devRef} hcil unchanged: " . "'${${$pathDtlsRef}{${$devRef}}}{hcil}'" ); } else { #----------------------------------------------------------- # B : One or more hcils have changed: #----------------------------------------------------------- printdbg( \$func, "X-- Path ${$devRef} hcil changed : " . "'${${$confPathDtlsRef}{${$devRef}}}{hcil}' " . "=> '${${$confPathDtlsRef}{${$devRef}}}{hcil}'" ); $update_conf = 1; } check_dev_path_states( $pathDtlsRef, $uidRef, $devRef ); } else { printdbg( \$func, "Old ${$uidRef} path ${$devRef} - No longer seen" ); #--------------------------------------------------------------- # C : One or more paths are missing: #--------------------------------------------------------------- printdbg( \$func, "?-- Path ${$devRef} " . "hcil ${$confPathDtlsRef}{${$devRef}}{hcil} " . "missing - Recover ..." ); recover_device( $uidRef, \${$confPathDtlsRef}{${$devRef}}{hcil} ); $update_conf = 1; } return( $update_conf ); } #----------------------------------------------------------------------- # check the states of device paths and attempt recovery, where needed #----------------------------------------------------------------------- sub check_dev_path_states { my $pathDtlsRef = shift; my $uidRef = shift; my $devRef = shift; my $func = "check_dev_path_states"; #------------------------------------------------------------------- # is path 'active'? # yes, great, nothing to do # no, okay, need to attempt recovery #------------------------------------------------------------------- if ( ${${$pathDtlsRef}{${$devRef}}}{dm_st} eq "active" ) { printdbg( \$func, "+-- Path ${$devRef} dm_st is good : 'active'" ); } else { #--------------------------------------------------------------- # A : One or more statuses (dm_st) are not "active": # set path online 'echo running > ...' # create rescan event #--------------------------------------------------------------- printdbg( \$func, "+-- Path ${$devRef} dm_st is bad : " . "'${${$pathDtlsRef}{${$devRef}}}{dm_st}'" ); set_path_online( $devRef ); rescan_path( $devRef ); } return( 0 ); } #----------------------------------------------------------------------- # abort the check for some file access reason #----------------------------------------------------------------------- sub abort_check { my $fileRef = shift; my $access = shift; my $func = "abort_check"; my $msg = "ABORT: Failed '$access' operation on '${$fileRef}'"; printdbg( \$func, "$msg" ); syslogwrite( "$msg" ); die( "$0 ABORTED" ); } #----------------------------------------------------------------------- # write to a file ... #----------------------------------------------------------------------- sub sysfswrite { my $fileRef = shift; my $s = shift; my $func = "sysfswrite"; if ( -e ${$fileRef} ) { ; } else { syslogwrite( " WARN: File '${$fileRef}' doesn not exist ahead of write " . "(system may have deleted it)" ); } open( OFILE, ">${$fileRef}" ) or abort_check( $fileRef, "open output" ); printf OFILE ( "$s\n" ); close( OFILE ) or abort_check( $fileRef, "close output" ); syslogwrite( " INFO: Written '$s' to ${$fileRef}" ); foreach my $line ( `ls -ltr ${$fileRef}` ) { chomp( $line ); printdbg( \$func, "$line" ); } return( 0 ); } #----------------------------------------------------------------------- # write given string to log file (including timestamp and PID) #----------------------------------------------------------------------- sub syslogwrite { my $s = shift; my $func = "syslogwrite"; open( LFILE, ">>$logfile" ) or abort_check( \$logfile, "open append" ); my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) = localtime( time ); my $msg = sprintf ( "%4d-%02d-%02d %02d:%02d:%02d %6d $s", $year+1900, $mon+1, $mday, $hour, $min, $sec, $$ ); printf LFILE ( $msg . "\n" ); printdbg( \$func, $msg ); close( LFILE ) or abort_check( \$logfile, "close append" ); return( 0 ); } #----------------------------------------------------------------------- # path recovery of a given device path and hcil #----------------------------------------------------------------------- sub recover_device { my $uidRef = shift; my $hcilRef = shift; my $func = "recover_device"; printdbg( \$func, "Looking to recover ${$uidRef} ${$hcilRef}" ); #------------------------------------------------------------------- # bring the device back using the following method: # echo 'c i l' > /sys/class/scsi_host/hostX/scan # eg # If the following path were lost: # 4:0:2:1 sdd 8:48 active ready running # h:c:i:l # It could be brought back with: # echo '0 2 1' > /sys/class/scsi_host/host4/scan #------------------------------------------------------------------- my ( $host, $NULL, $ctlr, $lun ) = split( /[,:.\s]/, ${$hcilRef} ); my $file = "/sys/class/scsi_host/host" . $host . "/scan"; my $strg = "$NULL $ctlr $lun"; printdbg( \$func, "Generating rescan for hcil, '${$hcilRef}' in '$file', using '$strg'" ); sysfswrite( \$file, $strg ); return( 0 ); } #----------------------------------------------------------------------- # set dev/path to "online" #----------------------------------------------------------------------- sub set_path_online { my $devRef = shift; my $func = "set_path_online"; my $rc = 0; my $path = "/sys/block/${$devRef}/device"; if ( -d $path ) { my $file = $path . "/state"; if ( -w $file ) { printdbg( \$func, "Setting 'online' ('running') state for " . "'${$devRef}' in '$file'" ); sysfswrite( \$file, "running" ); } else { $file = $path . "/online"; if ( -w $file ) { printdbg( \$func, "Setting 'online' ('1') state for " . "'${$devRef}' in '$file'" ); sysfswrite( \$file, "1" ); } else { syslogwrite( " WARN: File '$file' read-only - " . "Can't set 'online' state for '${$devRef}'" ); $rc = 2; } } } else { syslogwrite( " WARN: Directory '$path' nonexistent - " . "Can't set 'online' state for '${$devRef}'" ); $rc = 1; } return( $rc ); } #----------------------------------------------------------------------- # rescan the given path #----------------------------------------------------------------------- sub rescan_path { my $devRef = shift; my $func = "rescan_path"; my $rc = 0; my $file = "/sys/block/${$devRef}/device"; if ( -d $file ) { $file .= "/rescan"; printdbg( \$func, "Initiating device rescan for '${$devRef}' in '$file'" ); sysfswrite( \$file, "1" ); } else { syslogwrite( " WARN: Directory '$file' nonexistent - " . "Can't initiate rescan for '${$devRef}'" ); $rc = 1; } return( $rc ); } #----------------------------------------------------------------------- # create "uevent" #----------------------------------------------------------------------- sub generate_uevent { my $devRef = shift; my $func = "generate_uevent"; my $rc = 0; my $file = "/sys/block/${$devRef}"; if ( -d $file ) { $file .= "/uevent"; printdbg( \$func, "Generated uevent for '${$devRef}' in '$file'" ); sysfswrite( \$file, "add" ); } else { syslogwrite( " WARN: Directory '$file' nonexistent - " . "Can't generate uevent for '${$devRef}'" ); $rc = 1; } return( $rc ); } #----------------------------------------------------------------------- # write the conf file #----------------------------------------------------------------------- sub write_conf_file { my $prodcodeRef = shift; my $uidPathsRef = shift; my $pathDtlsRef = shift; my $update_conf = shift; my $func = "write_conf_file"; printdbg( ); #------------------------------------------------------------------- # check whether we need to write a new conf file #------------------------------------------------------------------- if ( -s $confile && $update_conf == 0 ) { printdbg( \$func, "No need to write conf file - " . "Situation unchanged ($update_conf updates)" ); } else { printdbg( \$func, "Write conf file - File nonexistent OR >0 updates: $update_conf" ); syslogwrite( " NOTE: (Re)Writing device path list to conf file, '$confile'" ); open( CFILE, ">$confile" ) or abort_check( \$confile, "open output" ); #--------------------------------------------------------------- # loop over the actual devices seen ... #--------------------------------------------------------------- foreach my $uid ( sort( keys( %{$uidPathsRef} ) ) ) { #----------------------------------------------------------- # is this the required prodcode? # yes, write details to the conf file # no, ignore this device's details #----------------------------------------------------------- if ( ${$uidPathsRef}{$uid}{prod} eq ${$prodcodeRef} ) { printdbg( ); printdbg( \$func, "uid=$uid (${$uidPathsRef}{$uid}{vend}," . "${$uidPathsRef}{$uid}{prod})" ); #------------------------------------------------------- # loop over the actual device paths seen ... # ... and write a conf file line for each #------------------------------------------------------- foreach my $dev ( sort { $a<=>$b || length $a <=> length $b || $a cmp $b } ( @{${$uidPathsRef}{$uid}{paths}} ) ) { printdbg( \$func, "$uid $dev ${$pathDtlsRef}{$dev}{hcil} = " . "${$pathDtlsRef}{$dev}{dm_st}" ); print CFILE ( "$uid,${$pathDtlsRef}{$dev}{hcil},$dev\n" ); } printdbg( \$func, sprintf "$func: %-33s path count %d", $uid, scalar( @{${$uidPathsRef}{$uid}{paths}} ) ); } else { #printdbg( \$func, "uid=$uid " . # "(${$uidPathsRef}{$uid}{vend}," . # ${$uidPathsRef}{$uid}{prod}) - IGNORED" ); } } #--------------------------------------------------------------- # close the conf file #--------------------------------------------------------------- close( CFILE ) or abort_check( \$confile, "close output" ); } return( 0 ); } #----------------------------------------------------------------------- # output line of debug, if debug active #----------------------------------------------------------------------- sub printdbg { my $funcRef = shift; my $string = shift; if ( $debug ) { print( "DBG: " ); if ( defined( $funcRef ) ) { printf( "%-28s: %s\n", ${$funcRef} . "()", $string ); } else { print( "\n" ); } } return( 0 ); } #======================================================================= # # check whether following options have been specified: # '-p ': eg '-p 2145', for SVC device path recovery # '-r' : refresh conf file, following expected changes # '-d' : debug - display debug throughout operation # '-h' / else : show usage and help # #======================================================================= my $prodcode = "2145"; my $refresh = 0; while ( scalar( @ARGV ) ) { if ( $ARGV[0] eq "-d" ) { $debug++; print( "DBG: Setting debug to $debug\n" ); } elsif ( $ARGV[0] eq "-r" ) { $refresh = 1; } elsif ( $ARGV[0] eq "-p" ) { shift; if ( defi6yyned( $ARGV[0] ) && length( $ARGV[0] ) ) { $prodcode = $ARGV[0]; } else { print( "ERROR: Product code expected after '-p' parameter, eg '-p 2145'\n\n" ); exit 1; } } else { print( "USAGE: svc_health_check.pl [-p ] (specified prod dev paths)\n" . " [-r] (refresh conf file)\n" . " [-d] (show debug output)\n" . " [-h] (HELP!)\n\n" ); print( " SVC health check - Attempt recovery of 'offline' SVC paths\n\n" ); print( "WHERE: -p Product code - Product code for path recovery [OPTIONAL]\n" . " Eg '-p 2145' (SVC)\n" . " Default='2145' (SVC)\n\n" ); print( " -r Refresh - Refresh device path list [OPTIONAL]\n" . " Default=Use conf file\n\n" ); print( " -d Debug - Show debug throughout process [OPTIONAL]\n" . " Default=No debug output\n\n" ); print( " -h HELP! - Show this help text\n\n" ); print( "NOTES: No options required - Defaults to 2145 (SVC)\n\n" ); print( " : If an svc_health_check conf file exists, the contents will be used to\n" . " identify the SVC device paths to check - Otherwise, the SVC device\n" . " paths will be identified from the system and a new conf file created\n" . " from the findings\n" . " Config file name: '/etc/svc_health_check.conf.'\n\n" ); print( " : The script should first be run at a time when all SVC device paths are\n" . " present and 'online' - Same applies when using '-r' option\n\n" ); print( " : A log file is appended with script start, activity and end events\n" . " Log file name: '/var/log/svc_health_check.log'\n\n" ); print( "Licensed Materials - Property of IBM\n\n" . "2145-VC6, 2076-VM1\n\n" . "Copyright IBM Corp. 2011, All Rights Reserved.\n\n" . "US Government Users Restricted Rights - Use, duplication or\n" . "disclosure restricted by GSA ADP Schedule Contract with\n" . "IBM Corp.\n\n" ); exit 1; } shift } if ( defined( $prodcode ) and length( $prodcode ) ) { $confile .= ".$prodcode"; if ( $refresh && -w $confile ) { system( "rm $confile" ); } main( \$prodcode ); } else { print( "ERROR: Product code required, eg '-p 2145'\n\n" ); exit 1; } #eof