#!/bin/bash -p
# restartcollector v1.5 2013-08-10, Finn Magnusson, finn.magnusson@ericsson.com
# Type "restartcollector" for help.

###########################################################################################
oldestPMD=30              #PMD files older than xx days will not be fetched. Set this value to 0 to fetch all PMD files regardless of timestamp.
maxNrOfProcessMobatch=60  #mobatch running in "light" mode ==> less memory usage
mobatchTimeout=60         #timeout in minutes for mobatch. Needs to be enough since fetching of logs can take time.
includeExtra=1            #include information fields for Message and State
###########################################################################################


moshelldir=`dirname "$0"`
if [[ $moshelldir != /* ]] ; then moshelldir=`pwd`/$moshelldir ; fi
vobsinstallation=0
unamea=$(uname -a)
gawkext=""
gawklib=""
if [[ $unamea = [Ll][iI][nN][uU][xX]*x86_64* ]] ; then gawklib="lin64"   ; if [[ $vobsinstallation = 1 ]] ; then gawkext=".lin64" ; fi
elif [[ $unamea = [Ll][iI][nN][uU][xX]* ]]      ; then gawklib="linux"   ; if [[ $vobsinstallation = 1 ]] ; then gawkext=".linux" ; fi
elif [[ $unamea = SunOS*sparc* ]]               ; then gawklib="solaris" ; if [[ $vobsinstallation = 1 ]] ; then gawkext=".solaris" ; fi
elif [[ $unamea = SunOS* ]]                     ; then gawklib="sol86"   ; if [[ $vobsinstallation = 1 ]] ; then gawkext=".sol86" ; fi
else gawklib="cygwin" ; gawkext=".exe"
fi
gawkprog="gawk${gawkext}" 
gawk="$moshelldir/$gawkprog"
#special case where moshell has been installed on a linux 32 bit machine but should be run on linux 64 sharing the same file system
if [[ $unamea = [Ll][iI][nN][uU][xX]*x86_64* && $vobsinstallation != 1 && `file "$moshelldir/gawk"` = *32-bit* ]] ; then gawklib="linux" ; fi
filefunc="$moshelldir/commonjars/lib/${gawklib}/filefuncs"
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}$moshelldir/commonjars/lib/${gawklib}
export LANG=C
export LC_ALL=C

moshell="$moshelldir/moshell"
funcs="$moshelldir/funcs.awk"
extern="$moshelldir/extern.awk"
defLogDir=$moshelldir/commonjars/defLogDir.sh
moshellrc=$($defLogDir)
logdir="$moshelldir/logfiles/logs_restartcollector"
temp_logdir=$($defLogDir restartcollector)
if [[ -w $temp_logdir && -d $temp_logdir ]] ; then logdir=$temp_logdir ; fi
tempknownhostfile="/tmp/knownhost$$"
globmoshellrc=$moshelldir/jarxml/moshellrc

trap 'rm -r "$tempknownhostfile" 2>/dev/null' 0 1 2 3 15 

function print_usage()
{
cat <<EOF

Purpose:  To collect restart data from a number of Cello nodes in the network (RNC/RBS/etc.).
  
Syntax: The tool must be run in two steps: 

  1) restartcollector [-l <logdir>][-s][-v <moshellvars>][-b <processes>][-t <timeout>]  <sitefile>
This is to fetch the data for running the reports in step 2.
        * Without the "-s" option: fetches system/avli/upgrade log, llog , te log, and pmd files from all nodes specified in the sitefile.
        * With the "-s" option:    Fetches system/avli/upgrade log only. 
Using the "-s" makes the data collection faster since less data is fetched, however, this will only work for the reports 2b and 2c. For the report 2a, the full data must be fetched (ie without the "-s" option). 
The pmd files are stored in the directory ~/moshell_logfiles/logs_restartcollector/NodeLogs, the other logs are stored in ~/moshell_logfiles/logs_restartcollector/SysLogs and .../AvLogs.
The logs and pmd files are parsed/decoded and the restart data is stored in a cache directory (.../DailyRestars) which will be used by the report generation process in step 2a/2b/2c.
Optional parameters:
 -l <logdir> : <logdir> will be used as cache directory instead of the default ~/moshell_logfiles/logs_restartcollector
 -v <uservars> : to specify different moshell uservariables, eg: restartcollector -v ip_database=./myipadatabase ./mysitefile
 -b <processes> : specify the maximum number of processes that mobatch will run in parallel.
 -t <timeout>   : specify the timeout after which mobatch will kill hanging moshell sessions.

  2) restartcollector -a|-u|-r <TRfile> [-d <startDate>] [-f <endDate>] [-m <minusDays>] [-p <plusDays>] [-n <nodeFilter>]  [-l <logdir>] 
Generates a report based on the logs fetched in step 1.
Three kinds of reports can be generated:
       * restart report. Option: -r <TRfile>. Shows restart information and restart type (manual, spontaneouse) with mapping to TR for crashes.
       * availability report. Option: -a.  Shows cello downtime and application downtime.
       * upgrade report. Option: -u. Shows install and upgrade statistics.
Optional parameters:
 -d <startDate> : show all data from the start date. E.g: "-d 2004-03-20" 
 -m <minusDays> : show all data from <minusDays> days ago. E.g. "-m 7" (from 7 days ago).
 -f <endDate>   : show all data up to the end date. E.g: "-f 2004-05-20"
 -p <plusDays>  : show all data up to <plusDays> days onwards, eg: "-p 5"
 -n <nodeFilter>: only show data from nodes matching the filter. Put quotes around the filter if it contains 
regular expression metacharacters. Example:  -n "^(H3|3)"
 -l <logdir>    : logs will be parsed from <logdir>, instead of the default ~/moshell_logfiles/logs_restartcollector
Examples: 
         >> restartcollector -r TRfile -d 2004-09-20 -p 7   ==> restart report from the 2004-09-20 to the 2004-09-27
         >> restartcollector -r TRfile -m 7 -p 0            ==> restart report from one day only, 7 days ago
         >> restartcollector -r TRfile -m 7                 ==> restart report from 7 days ago until today
         >> restartcollector -r TRfile -d 2004-03-05 -f 2004-05-05   ==> restart report from the 2004-03-05 to the 2004-05-05
         >> restartcollector -a -m 30                       ==> availability report from 30 days ago up to now
         >> restartcollector -u -d 2005-05-20               ==> upgrade report from 2005-05-20 up to now
	
	* 2a) Restart data report. Option: -r <TRfile>
The report contains all the restarts (ordered and spontaneous) together with any related TRs/CSRs.
A number of statistics are provided at the end of the report.
The report is in csv format for easy import into excel. Just copy-paste the output to a text file.
Then open the text file with excel and select "Delimited", then "Semicolon" in the dialog box.
The TRfile can be empty if the TRs are not yet known.
Once the reports have been run a few times, entries will be added in the TRfile.
The TRfile should then follow the format below (last 3 fields are optional):
TR/CSR;Process;ErrorCode;Actor/ExtraData;Srr0;BoardType  
Example: 
   UABtr18236;Ose_main;0x00147823
   WRNab79006;RncLmUePT;0x070000113;drhIurCcRhSrncC;0C204F94
   SMS18943;RncLmUePT;0x070000113;drhIurCcRhSrncC;;ModuleMp

  * Note1: In the TR file, it is possible to have several TR/CSR pointing to the same process/errorcode combination.
A separate line must be made for each TR/CSR as in the example above.
  * Note2: all nodes specified in the sitefile should be defined in the ipdatabase together with their telnet password.
See example of the ipdatabase file in moshell/examples/mobatch_files
  * Note3: in order to get the RNC actor information the pmd hooks should be activated on each module.
Following command should be run, refer to PLM info:
"lh mod ueh pmdchange active ; ueh pmdchange save ; cell pmdchange active ; cell pmdchange save"
Check current status with commands: "lh mod ueh pmdstatus ; cell pmdstatus"
  * Note4: the SW level shown in the report is the SW level at the time the pmd file was fetched, which is not necessarily the same as the SW level at the time the crash occurred.

	* 2b) Availability report (system downtime). Option: -a
Generates a report presenting all the nodes downtime with average downtime per node restart and average node downtime per day.
  * cello downtime is the time elapsed between the row "*** Cls_evlogd_proc restart ***" and the row "Node operational." in syslog. 
  * application downtime is the time elapsed between the row "*** Cls_evlogd_proc restart ***" in syslog and the row "RNC Node Restart Completed" (RNC) or "Cell .* enabled" (RBS) in avlog. 
  
	* 2c) Upgrade report. Option: -u
Generates a report presenting upgrade success and failure info for the network.
Note1: in the report's upgrade part, there are two times: UpgradeTime and ConfirmTime. "ConfirmTime" is the time where the upgrade was idle, waiting for the operator or OSS to confirm the upgrade. "UpgradeTime" is the total upgrade time minus "ConfirmTime". So the total upgrade time from start ("action upgrade") to finish ("final cv made and set startable") can be calculated by adding "UpgradeTime" to "ConfirmTime".
Note2: in the upgrade report, in the field NrOfNodeRestarts, the number in brackets indicates if some of the restarts were manual and how many. This happens for instance when the upgrade rolled back after the restart and the operator had to manually set the SU CV and restart.
Note3: it is recommended to increase the property trace.file.size in the /c/tmp/Trace_01.prop to a greater value (eg. 1000) so that the upgrade log will contain information going further back in time (with the default value 100, there is only enough space for the information relating to the latest upgrade).

EOF
}

function check_file_validity()
{
	if ! test -f $1 ; then echo "File not found: $1 . Exiting..." ; exit 1 ; fi
}
function restartcollector()
{
$gawk -v moshelldir="$moshelldir" -v moshell="$moshell" -v moshellrc="$moshellrc" -v sitefile=$sitefile -v logdir=$logdir -v dailyDir=$dailyDir -v dailyNodesDir=$dailyNodesDir -v generate_report=$generate_report -v trfile=$trfile -v includeExtra=$includeExtra -v sysLogDir=$sysLogDir -v avLogDir=$avLogDir -v maxprocess=$maxNrOfProcessMobatch -v startdate=$startdate -v enddate=$enddate -v plusdays=$plusdays -v mobatchTimeout=$mobatchTimeout -v nodeFilter="$nodeFilter" -v minusdays=$minusdays -v tempknownhostfile="$tempknownhostfile" -v globmoshellrc="$globmoshellrc" -v smallreport=$smallreport -v itemise_restarts=$itemise_restarts -v mobatch_switch="$mobatch_switch" -v vobsinstallation="$vobsinstallation" -v oldestPMD="$oldestPMD" -v userVariables="$userVariables" -f $funcs -f $extern -l "$filefunc" --source '
BEGIN{
	loadFilefuncs()
	parse_user_variables(moshell)	
	parse_user_variables(globmoshellrc)	
	parse_user_variables(moshellrc)	
	parse_commandlineOptions(userVariables)
	parse_common_variables()
	DefineGlobalVars()
	globalNodeDir=sprintf("%s/NodeLogs",logdir)
	fileOps(sprintf("mkdir -p %s",dailyDir))
	fileOps(sprintf("mkdir -p %s",sysLogDir))
	fileOps(sprintf("mkdir -p %s",avLogDir))
	fileOps(sprintf("mkdir -p %s",dailyNodesDir))
	if (generate_report == 0)
	{
		syslogcollection()
		pmdcollection()
	}
	else if (generate_report == 1) make_report()
	else if (generate_report == 2) show_downtime()
	else if (generate_report == 3) upgrade_report()
}
'
}


generate_report=0
start_date=0
itemise_restarts=1   #always itemise
while getopts "r:d:l:f:p:n:m:ausiv:b:t:" options
do
case $options in
v) userVariables="$OPTARG"
   mobatch_switch=",$OPTARG" 
   ;;
r) generate_report=1
   trfile=$OPTARG
   if [[ ! -f $trfile ]] ; then echo "File $trfile not found. Exiting..." ; exit 1 ; fi
   ;;
a) generate_report=2
   ;;
u) generate_report=3
   ;;
d) startdate=$OPTARG
   ;;
f) enddate=$OPTARG
   ;;
i) itemise_restarts=1
   ;;   
p) plusdays=$OPTARG
   ;;
l) logdir=$OPTARG
   if [[ ! -d $logdir ]] ; then echo "Log directory $logdir does not exist. Exiting..." ; exit 1 ; fi
   ;;
n) nodeFilter=$OPTARG
   ;;
m) minusdays=$OPTARG
   ;;
s) smallreport=1
   ;;
b) maxNrOfProcessMobatch=$OPTARG
   ;;
t) mobatchTimeout=$OPTARG
   ;;
esac
done
shift $(($OPTIND - 1))

if [[ $generate_report -eq 0 && $# -eq 1 ]] ; then
    if test -f "$1" ; then sitefile="$1"
    else echo "File $1 not found. Exiting...." ; exit 1
    fi
elif [[ $generate_report -eq 1 && $# -eq 0 ]] ; then :
elif [[ $generate_report -eq 2 && $# -eq 0 ]] ; then :
elif [[ $generate_report -eq 3 && $# -eq 0 ]] ; then :
else print_usage ; exit 1
fi

dailyDir=$logdir/DailyRestarts
sysLogDir=$logdir/SysLogs
avLogDir=$logdir/AvLogs
dailyNodesDir=$logdir/DailyNodes
case $(uname) in
 *[cC][yY][gG]*) logdir=$(cygpath -m $logdir) ;;
esac

restartcollector
		
# P3:
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:333 ERROR:SPM #0 called OSE kernel error handler
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:334 ERROR:======================================
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:335 ERROR:process id:      0x0
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:336 ERROR:process name:    No process name available
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:337 ERROR:user called:     0
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:338 ERROR:error code:      0x1000024
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:339 ERROR:extra parameter: 0x48dff0
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:340 ERROR:fatal:           0
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:341 ERROR:subcode:         0x100
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:342 ERROR:main error code: 0x24
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:343 ERROR:msr reg.:        0x3930
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:344 ERROR:srr0 reg.:       0x4df170
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:345 ERROR:srr1 reg.:       0x1930
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:346 ERROR:dar reg.:        0x209d90
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:347 ERROR:dsisr reg.:      0x42000000
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:446 ERROR:The error was generated by the SP Firmware code
#   0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:735 ERROR:SPM Satellite Loader: Restart ordered from command line interface.

#P2.1
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:692 ERROR:SP #1 called OSE kernel error handlerESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:693 ERROR:======================================ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:694 ERROR:process id:      0x0ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:695 ERROR:process name:    No process name availableESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:696 ERROR:user called:     1ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:697 ERROR:error code:      0x81000012ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:698 ERROR:extra parameter: 0x1ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:699 ERROR:fatal:           1ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:700 ERROR:subcode:         0x100ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:701 ERROR:main error code: 0x12ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:702 ERROR:msr reg.:        0x9930ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:703 ERROR:srr0 reg.:       0x4378f0ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:704 ERROR:srr1 reg.:       0x9930ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:705 ERROR:dar reg.:        0x0ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:706 ERROR:dsisr reg.:      0x0ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.380] spp_loader sp_errormsg.cc:819 ERROR:The error was generated by the SP Firmware codeESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.380] spp_loader sp_errormsg.cc:1031 ERROR:A SW based watchdog triggered SP restart.ESC[0m
#   0016: ESC[31m[2004-05-11 14:13:38.380] spp_loader sp_errormsg.cc:1032 ERROR:The extra parameter indicates which watchdog.ESC[0m

#   0204: [2004-04-18 16:37:02.892] spp_loader spm.cc:890 INFO:spm #1 state change: LNH_UP_STATE -> CRASHED_STATE
#   0204: [2004-04-18 16:37:02.892] spp_loader spm.cc:890 INFO:spm #1 state change: CRASHED_STATE -> LOADED_STATE
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:692 ERROR:SP #1 called OSE kernel error handler
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:693 ERROR:======================================
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:694 ERROR:process id:      0x10036
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:695 ERROR:process name:    DedicatedChMainPT
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:696 ERROR:user called:     1
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:697 ERROR:error code:      0x48
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:698 ERROR:extra parameter: 0x690740
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:699 ERROR:fatal:           0
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:700 ERROR:subcode:         0x0
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:701 ERROR:main error code: 0x48
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:702 ERROR:msr reg.:        0x9930
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:703 ERROR:srr0 reg.:       0x4378f0
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:704 ERROR:srr1 reg.:       0x9930
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:705 ERROR:dar reg.:        0x0
#   0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:706 ERROR:dsisr reg.:      0x0
#   0204: [2004-04-18 16:37:02.896] spp_superviser_ro sp_superviser_ro.c:248 INFO:Connection to SPM-SP 1 lost
#   0204: [2004-04-18 16:37:10.332] spp_loader sp_errormsg.cc:637 INFO:OSE PMD written to file /c/pmd/sp/00019_020400_spm1.pmd
#   0204: [2004-04-18 16:37:10.332] spp_loader sp_errormsg.cc:1139 ERROR:Error code unknown to SPB FW:
#   0204: [2004-04-18 16:37:10.332] spp_loader sp_errormsg.cc:1141 ERROR:  Check if subcode/main error code can be found
#   0204: [2004-04-18 16:37:10.332] spp_loader sp_errormsg.cc:1142 ERROR:  in the files ose_errors.txt and ose_err.h.
#   0204: [2004-04-18 16:37:10.332] spp_loader sp_errormsg.cc:1143 ERROR:  These can respectively be found under /vobs/cello/cs/ose/powerpc/doc
#   0204: [2004-04-18 16:37:10.332] spp_loader sp_errormsg.cc:1144 ERROR:  and /vobs/cello/cs/ose/powerpc/krn-750/include
#   0204: [2004-04-18 16:37:10.332] spp_loader sp_errormsg.cc:401 INFO:Sending alarm #418 to SPM-FRO
#
#
#   ----------------------------------------------------------------------------
#    6.    Time           :  04-02-12, 05:35:25
#          Error code     :  0x00580313 (Reported via the OSE error API)
#          Process        :  RncLmUePT
#          Queued signals :  0
#          Owned signals  :  574
#          OS Run Mode    :  BACKUP
#          OSE Component  :  MMS
#
#          Error Description  :       [OSE_MMS_EUNEXPECTED_EXCEPTION]
#          An unexpected exception occured.
#
#          [0x03] : Data access exception
#
#          CPU Register Dump:
#          oldSP: 00000000   oldLR: 00EC6094
#
#          r0  :  0C2BF164   r8  :  00000000   r16 :  00000000   r24 :  00000000
#          r1  :  1641E6E8   r9  :  00000000   r17 :  00000000   r25 :  16FB0E38
#          r2  :  0C4ECEC0   r10 :  00000000   r18 :  00000000   r26 :  0C620010
#          r3  :  18B75030   r11 :  00000000   r19 :  00000000   r27 :  00000000
#          r4  :  0C2D8ADC   r12 :  1AFB4638   r20 :  00000000   r28 :  18B691E0
#          r5  :  0C2D8ADC   r13 :  0C639178   r21 :  00000000   r29 :  FFFFFFFF
#          r6  :  00000000   r14 :  00000000   r22 :  00000000   r30 :  0C6310D0
#          r7  :  0E16F738   r15 :  00000000   r23 :  00000000   r31 :  18B75030
#
#          ctr :  0C2D7748   xer :  20000000   cr  :  44000804   lr  :  0C2BF178
#          srr0:  0C2BF18C   srr1:  0000F030   dar :  00000030   vec :  00000300
#
#   ----------------------------------------------------------------------------
#    7.    Time           :  04-02-20, 08:14:35
#          Error code     :  0x00580313 (Reported via the OSE error API)
#          Process        :  RncLmUePT
#          Queued signals :  0
#          Owned signals  :  576
#          OS Run Mode    :  BACKUP
#          OSE Component  :  MMS
#
#          Error Description  :       [OSE_MMS_EUNEXPECTED_EXCEPTION]
#          An unexpected exception occured.
#
#          [0x03] : Data access exception
#
#          CPU Register Dump:
#          oldSP: 00000000   oldLR: 00EC6094
#
#          r0  :  0C204F70   r8  :  0000F030   r16 :  00000000   r24 :  002D0000
#          r1  :  1641E528   r9  :  000202D8   r17 :  00001030   r25 :  00000000
#          r2  :  0C4ECEC0   r10 :  00000026   r18 :  00000001   r26 :  1641E5B8
#          r3  :  1AF93D10   r11 :  00000000   r19 :  00000000   r27 :  14D49920
#          r4  :  0088C91C   r12 :  80000001   r20 :  00000000   r28 :  1B538BB8
#          r5  :  0000B030   r13 :  0C639178   r21 :  00000000   r29 :  00000000
#          r6  :  00003000   r14 :  0027B800   r22 :  000000BF   r30 :  1B538BB8
#          r7  :  0C474B50   r15 :  00000011   r23 :  00010003   r31 :  1AF93D10
#
#          ctr :  00E69D38   xer :  00000000   cr  :  44000204   lr  :  0C204F70
#          srr0:  0C204F94   srr1:  0000F030   dar :  8000000D   vec :  00000300
#   - BLOCK 06 ----------------------------------------------------
#   date   = 2004-02-20  08:14:35 192639
#   [GENERAL INFORMATION]
#   Processor restarted
#   Processor link handler name:
#   Error code: 00580313
#   Origin: OSE Component
#   Process: RncLmUePT
#   Time: 04  02  20  08  14  35
#   Process signal queue length: 0
#   Process owned signals: 576
#   Other extra info:
#   00000000  00 00 00 00 00 EC 60 94 0C 20 4F 70 16 41 E5 28 "......`.. Op.A.("
#   00000010  0C 4E CE C0 1A F9 3D 10 00 88 C9 1C 00 00 B0 30 ".N....=........0"
#   00000020  00 00 30 00 0C 47 4B 50 00 00 F0 30 00 02 02 D8 "..0..GKP...0...."
#   00000030  00 00 00 26 00 00 00 00 80 00 00 01 0C 63 91 78 "...&.........c.x"
#   00000040  00 27 B8 00 00 00 00 11 00 00 00 00 00 00 10 30 ".'.............0"
#   00000050  00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 "................"
#   00000060  00 00 00 BF 00 01 00 03 00 2D 00 00 00 00 00 00 ".........-......"
#   00000070  16 41 E5 B8 14 D4 99 20 1B 53 8B B8 00 00 00 00 ".A..... .S......"
#   00000080  1B 53 8B B8 1A F9 3D 10 00 E6 9D 38 00 00 00 00 ".S....=....8...."
#   00000090  44 00 02 04 0C 20 4F 70 0C 20 4F 94 00 00 F0 30 "D.... Op. O....0"
#   000000A0  80 00 00 0D 00 00 03 00                         "........"
#
#   - BLOCK 07 ----------------------------------------------------
#   date   = 2004-02-20  08:14:35 194074
#   [LOAD MODULE INFORMATION]
#
#   Load module id: CXC1321447_R7JK01
#   No module information available. Program loaded from Flash.


#else if (/^Time: /)
#{
#	if (NF==7)
#	{
#		date=sprintf("20%s-%s-%s",$2,$3,$4)
#		time=sprintf("%s:%s:%s",$5,$6,$7)
#	}
#	else if (NF==3)
#	{
#		date=$2
#		time=$3
#		if (date ~ /^0) date=sprintf("20%s",date)
#		else if (date ~ /^70/) date=sprintf("19%s",date)
#	}
#}


#SPM RESTARTS:
#DO find moshell_logfiles* -type f | xargs grep " ERROR:SPM"
#
#CPP5
#2006-03-30;06:21:57;RNC;tch35;-REFRESH RESTART;002218;;20060330_062201_642_002218_0x00000019.pmd;Spp_aalMainProc;;0x00000012;0x1C000000;;CXP9012014_R7E02;
#
#0022: [2006-03-30 06:21:57.696] spp_loader sp_errormsg.cc:424 ERROR:SPM #2 called OSE kernel error handler
#0022: [2006-03-30 06:21:57.696] spp_loader sp_errormsg.cc:425 ERROR:======================================
#0022: [2006-03-30 06:21:57.696] spp_loader sp_errormsg.cc:426 ERROR:user called:     0
#0022: [2006-03-30 06:21:57.696] spp_loader sp_errormsg.cc:427 ERROR:error code:      0x12
#0022: [2006-03-30 06:21:57.696] spp_loader sp_errormsg.cc:428 ERROR:extra parameter: 0x1c000000
#0022: [2006-03-30 06:21:57.696] spp_loader sp_errormsg.cc:429 ERROR:msr reg.:        0x3930
#0022: [2006-03-30 06:21:57.696] spp_loader sp_errormsg.cc:430 ERROR:srr0 reg.:       0x49aebc
#0022: [2006-03-30 06:21:57.696] spp_loader sp_errormsg.cc:431 ERROR:srr1 reg.:       0x3930
#0022: [2006-03-30 06:21:57.696] spp_loader sp_errormsg.cc:432 ERROR:dar reg.:        0x18b000
#0022: [2006-03-30 06:21:57.696] spp_loader sp_errormsg.cc:433 ERROR:dsisr reg.:      0x42000000
#0022: [2006-03-30 06:21:57.704] spp_loader_mgr spp_loader_mgr.c:2116 INFO:SPM #2 (00.22.18) : Reported alarm code (1), general problem
#0022: [2006-03-30 06:21:57.704] spp_loader_mgr spp_loader_mgr.c:1810 INFO:SPM #2 (00.22.18) : STOPPED
#0022: [2006-03-30 06:21:57.704] spp_loader_mgr spp_loader_mgr.c:1829 INFO:SPM #2 (00.22.18) : Restart (WARM)
#0022: [2006-03-30 06:21:58.088] Sys_Spas_hwdrv_proc spicfunc.c:3432 INFO:SAI Egress Discard. cells (port-queue): (18-2)
#0022: [2006-03-30 06:21:59.092] Sys_Spas_hwdrv_proc spicfunc.c:3432 INFO:SAI Egress Discard. cells (port-queue): (18-2)
#0022: [2006-03-30 06:22:01.840] spp_lcim_19 spp_lcim.c:630 INFO:SPM #2 (00.22.18) : Ose pmd written to file /c/pmd/642/002218/0x00000019.pmd
#0022: [2006-03-30 06:22:05.672] spp_loader_mgr spp_loader_mgr.c:2057 INFO:SPM #2 (00.22.18) : STARTED
#
#
#CPP4
#0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:333 ERROR:SPM #0 called OSE kernel error handler
#0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:336 ERROR:process name:    No process name available
#0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:338 ERROR:error code:      0x1000024
#0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:339 ERROR:extra parameter: 0x48dff0
#0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:344 ERROR:srr0 reg.:       0x4df170
#0019: [2004-12-11 02:06:19.336] spp_loader sp_errormsg.cc:735 ERROR:SPM Satellite Loader: Restart ordered from command line interface.
#
#0016: [2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:692 ERROR:SP #1 called OSE kernel error handler
#0016: [2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:695 ERROR:process name:    No process name available
#0016: [2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:697 ERROR:error code:      0x81000012
#0016: [2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:698 ERROR:extra parameter: 0x1
#0016: [2004-05-11 14:13:38.376] spp_loader sp_errormsg.cc:703 ERROR:srr0 reg.:       0x4378f0
#0016: [2004-05-11 14:13:38.380] spp_loader sp_errormsg.cc:1031 ERROR:A SW based watchdog triggered SP restart.
#
#0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:692 ERROR:SP #1 called OSE kernel error handler
#0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:695 ERROR:process name:    DedicatedChMainPT
#0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:697 ERROR:error code:      0x48
#0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:698 ERROR:extra parameter: 0x690740
#0204: [2004-04-18 16:37:02.892] spp_loader sp_errormsg.cc:703 ERROR:srr0 reg.:       0x4378f0
#0204: [2004-04-18 16:37:10.332] spp_loader sp_errormsg.cc:637 INFO:OSE PMD written to file /c/pmd/sp/00019_020400_spm1.pmd
