#!/usr/bin/perl
#----------------------------------------------------------------------------------------------
# pmExtract: Extract performance management counters from rop files
#
# usage: pmExtract <mo_filter> <counter_filter> [<file1> [<file2...] ]
#
# 	<mo_filter> = Regular expression to select which managed objects will be extracted
#	<counter_filter> = Regular expression to select which counters will be extracted
#	<file1>... = rop files to read. If none specified then std input will be read.
#----------------------------------------------------------------------------------------------
#
# Copyright (c) 2005 Ericsson Australia Pty Ltd
#
# This program may be used and/or copied only 
# with the prior written permission of Ericsson 
# Australia or in accordance with the terms and 
# conditions stipulated in the contract or agreement 
# under which the program has been supplied.
#
#----------------------------------------------------------------------------------------------
#
# v 1.0  26/05/2003 M. Harris    Initial Version
# v 1.1  11/05/2004 M. Harris    Mods to cope with Cello counter files
# v 1.2  14/05/2004 M. Harris    Add debug prints
# v 1.3  01/06/2004 M. Harris    Add detection & output of subnetwork
# v 1.4  08/06/2004 M. Harris    Add handling of local / utc time
# v 1.5  13/07/2004 M. Harris    Add optimisations stolen from Dave Markham's version
# v 1.6  26/09/2004 M. Harris    Change to case insensitive matching, add -help option
# v 1.7  08/11/2004 M. Harris    Fix times so they fetch from <cbt>, remove use of 'END' reserved word
# v 1.8  11/11/2004 M. Harris    Fix so scanning restarts at </mdc>
# v 1.9  13/09/2005 M. Harris    Handle <md> blocks with no counter instances
# v 1.10 22/09/2005 M. Harris    Add -tz option
# v 1.11 23/01/2006 M. Harris    Split matching of <moid> to allow $ or ^ to be used in mo regex
# v 1.12 22/08/2006 M. Harris    Change -tz to floating point (thanks epaerez)
# v 1.13 18/10/2007 M. Aldrin	 Corrected handling of multiple lines between <r> and </r>
# v 1.14 14/11/2007 M. Harris	 Slight efficiency improvement for above correction
# v 1.15 15/11/2007 F. Magnusson Slight change in above correction to not include linefeed on multiple line counter
# v 1.16 17/03/2008 M. Harris    Cater for duplicated counters (input from M. Aldrin & F. Magnusson)
# v 1.17 19/03/2008 F. Magnusson Print every warning only once
# v 1.18 09/06/2008 F. Magnusson Changed method of removing duplicated counters, TR WRNae08645
# v 1.19 30/01/2009 F. Magnusson Add a new counter DummyMo::pmRopCounter used to count the number of rops, needed for certain formulas
# v 1.20 04/03/2009 F. Magnusson Discarding of counters marked with the suspected faulty tag (<sf>TRUE</sf>).
# v 1.21 05/05/2009 M. Aldrin	 New option -f to highlight the counters marked with the suspected faulty tag.
# v 1.22 03/09/2010 F. Magnusson New option -k to keep counters marked with the suspected faulty tag (<sf>TRUE</sf>)
# v 1.23 30/09/2010 F. Magnusson New option -w to print warnings to a file. Used by pmrw.
# v 1.24 10/10/2010 F. Magnusson Added detection and tagging (when using option -f) of empty and negative counters
# v 1.25 31/10/2010 F. Magnusson Option -w only prints warnings to file, not to screen.
# v 1.26 13/02/2011 F. Magnusson Duplicate counters are not printed anymore, only how many. Use pmxs to show all duplicate counters.
# v 1.27 01/03/2011 F. Magnusson New option -l to print warnings to log and to screen.
# v 1.28 03/10/2011 F. Magnusson Support 3gsim ROP files where MO name does not have LDN
# v 1.29 11/09/2012 F. Magnusson New option -me to include the MeContext in the MO name
# v 1.30 10/04/2013 F. Magnusson Remove spaces in the LDN around the comma and equal sign (some TSP nodes like CSCF have this)
# v 1.31 10/04/2013 F. Magnusson Replace "&amp;" with "&" and ";" with "." in MO name
# v 1.32 10/04/2013 F. Magnusson Allow negative filter on the MO filter eg: pmExtract !<mofilter> 
# v 1.33 22/04/2013 F. Magnusson DummyMo::pmRopCounter was always being printed due to bug in previous pmExtract version 1.32
# v 1.34 07/02/2016 F. Magnusson Alias option "--sf" same as "-f" 
#----------------------------------------------------------------------------------------------

use strict;
use Getopt::Long;
use Time::Local;

my $version = "v1.34, 2016-02-07";

sub printUsage {

	if (@_) {print STDERR "@_\n";};
	die <<_EOTXT_;
pmExtract: Extract performance management counters from rop files

$version

usage: pmExtract [options] <mo_filter> <counter_filter> [<file1> [<file2...] ]

 	<mo_filter> = Regular expression to select which managed objects will be extracted
	<counter_filter> = Regular expression to select which counters will be extracted
	<file1>... = rop files to read. If none specified then std input will be read.

Options:

    -u, --utc
           Output time as UTC - otherwise time will be converted to local time
           (make sure environment variable TZ is set)
	   
    -tz <timezone>
           Specify timezone other than the one in which this machine is located
	   eg. -tz +10 means 10 hours ahead of GMT
	   
    -s
           Include subnetwork in managed object name
           
    -me   
           Include MeContext in managed object name
           
    -f, --sf      
           Highlight the counters marked with the suspected faulty tag.
           Without this option, the suspected faulty counters are discarded.

    -k      
           Keep the counters marked with the suspected faulty tag but do not highlight them.
           Without this option, the suspected faulty counters are discarded.

    -w <file>      
           Print warnings and list of faulty counters to a file, not to screen.
    
    -l <file>
           Print warnings to a file and to screen.
           
_EOTXT_

}
	
my ($includeSubNetwork, $utc_time, $tz, $help, $sfprint, $sfkeep, $sfdiscard, $warnfile, $warnlog, $includeMeContext);

GetOptions(
	"s|subnetwork" => \$includeSubNetwork,
	"me"           => \$includeMeContext,
	"u|utc"        => \$utc_time,
	"tz=f"         => \$tz,
	"f|sf"         => \$sfprint,
	"k"            => \$sfkeep,
	"w=s"          => \$warnfile,
	"l=s"          => \$warnlog,
	"help"         => \$help
) or printUsage;

if ($warnfile) { open (WARNFILE, ">>$warnfile") or die "unable to open $warnfile"; }
if ($warnlog) { open (WARNLOG, ">>$warnlog") or die "unable to open $warnlog"; }
if (!($sfprint || $sfkeep) ) { $sfdiscard=1; }

printUsage if ($help);
die "-tz and -utc cannot be specified together\n" if ($tz and $utc_time);

my $negmofilter = 0;
my $mo_filter=shift @ARGV or printUsage("Missing MO Filter");
my $counter_filter=shift @ARGV or printUsage("Missing Counter Filter");
if ($mo_filter =~ /^!/) { 
	$mo_filter =~ s/^!//; 
	$negmofilter = 1; 
}

my $counter_regexp=qr/$counter_filter/i;
my $motagmatch = qr/<moid>(.*)<\/moid>/;
my $momatch = qr/$mo_filter/i;
my $counternametemplate = qr/<mt>(.+)<\/mt>/;
my $countervaltemplate = qr/<r>([^<\n]*)/;
my ($currprefix, $currindex, $wantedarraypos, $name, $timestamp);
my $subnetwork = '';
my $mecontext = '';
my (@wantedctrnames, @wantedctrindices);
my ($yy,$mo,$dd,$hh,$mi,$ss,$mobj);
my ($line,$fline,$sncount, $counter, $mocounter, $moldn, $dupsize);
my (%table, %dupcounters, %duptable);
my (@buffer);
my ($bcount, $sftrue, $i, $empty, $negative, $output, $sfcount);
my $sftag = "";
my $sfwarn;
my $warnoutput;


READINGHEADER:
	%duptable = ();  #this table must be emptied at each ROP file
	$sncount++;      #this counter increases at each ROP file
	if ($sncount == 2){
		%table = ();  #this table was only used at the first rop file parsing, it can now be cleared
		$dupsize = keys(%dupcounters); #the number of duplicated counters found in the first ROP file
	}
	while (<>) {
		if ($includeSubNetwork && /<sn>(.*)<\/sn>/) {
			$subnetwork = $1.',';
		} elsif ($includeMeContext && /<sn>.*MeContext=([^, ]+)[ \t]*<\/sn>/) {
			$mecontext = "Me=".$1.',';
	  	} elsif (/<md>/) {
	    	goto IDLE;
	  	} elsif (($yy,$mo,$dd,$hh,$mi,$ss) = /<cbt>(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)/) {

		  	if (! $utc_time) {
				if ($tz) {
					($ss,$mi,$hh,$dd,$mo,$yy) = gmtime(timegm($ss,$mi,$hh,$dd,$mo-1,$yy-1900)+($tz*3600));
				} else {
					($ss,$mi,$hh,$dd,$mo,$yy) = localtime(timegm($ss,$mi,$hh,$dd,$mo-1,$yy-1900));
				}
			  	$yy+=1900;
			  	$mo++;
		  	}
			$timestamp = sprintf("%04d-%02d-%02d %02d:%02d:%02d", ($yy,$mo,$dd,$hh,$mi,$ss));
			
	  	}
	  		  	
	}
	goto ENDFILE;

IDLE:
	while (<>) {
		if (/<mi>/) {
	    		$currindex = 0;
	    		@wantedctrnames = ();
	    		@wantedctrindices = ();
	    		goto LOOKINGFORCOUNTERNAMES;
	  	}
		if (/<\/mdc>/) {
			if (($negmofilter==0 && "ManagedElement=1,DummyMo=1" =~ $momatch && "pmRopCounter" =~ $counter_regexp)||
			    ($negmofilter==1 && "ManagedElement=1,DummyMo=1" !~ $momatch && "pmRopCounter" =~ $counter_regexp)) { 
				if ($includeMeContext) { print $timestamp.';'.$mecontext."DummyMo=1".';'."pmRopCounter".';'."1\n";}
				else { print $timestamp.';'.$subnetwork."ManagedElement=1,DummyMo=1".';'."pmRopCounter".';'."1\n";}
			}
			goto READINGHEADER ;
		}

	}
	goto ENDFILE;

LOOKINGFORCOUNTERNAMES:
	while (<>) {
	 	if ($_ =~ $counternametemplate) {
			$name = $1;
			if ($name =~ $counter_regexp) {
				push (@wantedctrnames, $name);
				push (@wantedctrindices, $currindex);
			}
			$currindex++;
	  	} elsif (/<mv>/) {
			if (@wantedctrnames>0) {goto LOOKINGFORMONAMES;}  #only bother matching MOs if any counters have matched
			else {goto IDLE;}
	  	} elsif (/<\/mi>/) {
			goto IDLE;
		}
	}
	goto ENDFILE;

LOOKINGFORMONAMES:
	while (<>) {
		if ($_ =~ $motagmatch) {
			my $mobj=$1;
			$mobj =~ s/\s*,\s*/,/g;
			$mobj =~ s/\s*=\s*/=/g;
			$mobj =~ s/&amp;/&/g;
			$mobj =~ s/;/./g;
			if ($mobj !~ '=') { $mobj = 'Mo=' . $mobj;}
			if ($mobj !~ 'ManagedElement=1') { $mobj = 'ManagedElement=1,' . $mobj;}
			if ($includeMeContext) { 
				$mobj =~ s/ManagedElement=1,//; 
				if ($mobj =~ /^[^,]+,/) { $mobj =~ s/^[^,]+,//;}
			}
			if (($negmofilter == 0 && $mobj =~ $momatch)||($negmofilter == 1 && $mobj !~ $momatch)) {
				if ($includeMeContext) { $currprefix = $timestamp.';'.$mecontext.$mobj; }
				else { $currprefix = $timestamp.';'.$subnetwork.$mobj; }
				$moldn = $mobj;
				$currindex = 0;
				$wantedarraypos = 0;
				goto LOOKINGFORCOUNTERVALS;
			}
	  	} elsif (/<\/mi>/) {
			goto IDLE;
		} elsif (/<sf>TRUE/) { 
		 		$sftrue = 1;
				if ($warnfile) {
			  		for ($i = 0; $i < $bcount ; $i++) { 
			  			print WARNFILE "$buffer[$i] (SuspectedFaulty)\n";
			  			$sfcount++;
			  		}
			  	}	
		 		if ($sfdiscard){ @buffer = (); $bcount = 0; }
		 		if ($sfprint) { $sftag = " (SuspectedFaulty)"; }
		 		
	  	} elsif (/<\/mv>/) {
	  		for ($i = 0; $i < $bcount ; $i++) { 
	  			$output = $buffer[$i];
	  			if ($sftag eq '') { 
	  				if ($output =~ /;(,[^;]+|)$/) { 
	  					$empty++;
	  					if ($sfprint) { $output .= " (EmptyCounter)"; }
	  					elsif ($warnfile) { print WARNFILE "$output (EmptyCounter)\n" ;}
	  				}
	  				elsif ($output =~ /;-[0-9]+$/) {
	  					$negative++;
	  					if ($sfprint) { $output .= " (NegativeCounter)"; }
	  					elsif ($warnfile) { print WARNFILE "$output (NegativeCounter)\n" ;}
	  				}
	  			}
	  			else { 
	  				$sfcount++;
	  				if ($sfprint) { $output = $buffer[$i].$sftag; }
	  				elsif ($warnfile) { print WARNFILE "$output$sftag\n" ;}
	  			}
	  			print "$output\n";
	  			#print $buffer[$i].$sftag."\n" ;
	  		}
	  		@buffer = (); $bcount = 0; 
	  		$sftag = "";
	  	}
	  	
	}
	goto ENDFILE;

LOOKINGFORCOUNTERVALS:
	if ($sncount > 1 && $dupsize == 0) {
		while (<>) { 
			if ($_ =~ $countervaltemplate) {
			    	if ($currindex == $wantedctrindices[$wantedarraypos]) {
			      		$buffer[$bcount++] = $currprefix.";$wantedctrnames[$wantedarraypos++];$1";
				      	if ($wantedarraypos >= @wantedctrindices) {  #if there are no counters left to extract for this MO...
				       		goto LOOKINGFORMONAMES;
				      	}
				}
			    	$currindex++;
			} elsif (/<sf>TRUE/) { 
		 		$sftrue = 1;
				if ($warnfile) {
			  		for ($i = 0; $i < $bcount ; $i++) { 
			  			print WARNFILE "$buffer[$i] (SuspectedFaulty)\n";
			  			$sfcount++;
			  		}
			  	}	
				if ($sfdiscard){ @buffer = (); $bcount = 0; }
		 		if ($sfprint) { $sftag = " (SuspectedFaulty)"; }
			} elsif (/<\/mv>/) {
				for ($i = 0; $i < $bcount ; $i++) { 
		  			$output = $buffer[$i];
		  			if ($sftag eq '') { 
		  				if ($output =~ /;(,[^;]+|)$/) { 
		  					$empty++;
		  					if ($sfprint) { $output .= " (EmptyCounter)"; }
		  					elsif ($warnfile) { print WARNFILE "$output (EmptyCounter)\n" ;}
		  				}
		  				elsif ($output =~ /;-[0-9]+$/) {
		  					$negative++;
		  					if ($sfprint) { $output .= " (NegativeCounter)"; }
		  					elsif ($warnfile) { print WARNFILE "$output (NegativeCounter)\n" ;}
		  				}
		  			}
		  			else { 
		  				$sfcount++;
		  				if ($sfprint) { $output = $buffer[$i].$sftag; }
		  				elsif ($warnfile) { print WARNFILE "$output$sftag\n" ;}
		  			}
		  			print "$output\n";
		  			#print $buffer[$i].$sftag."\n" ;
				}
				@buffer = (); $bcount = 0; 
				$sftag = "";
				goto LOOKINGFORMONAMES;
			}
		} 		
	} elsif ($sncount > 1 && $dupsize > 0) {
		while (<>) { 
			if ($_ =~ $countervaltemplate) {
		    	if ($currindex == $wantedctrindices[$wantedarraypos]) {
		    		$counter  = $wantedctrnames[$wantedarraypos++];
		    		$mocounter= $moldn.";$counter";
		    		$line  = $currprefix.";$counter";
		    		$fline = $line.";$1";
		    		if (! defined $dupcounters{$mocounter}) { $buffer[$bcount++] = $fline; }
		    		elsif (! defined $duptable{$line}) { 
		    				$buffer[$bcount++] = $fline; 
		    				$duptable{$line} = 1;
		    		}
				if ($wantedarraypos >= @wantedctrindices) {  #if there are no counters left to extract for this MO...
			       		goto LOOKINGFORMONAMES;
			      	}
			    }
		    	$currindex++;
		    	} elsif (/<sf>TRUE/) { 
		 		$sftrue = 1;
				if ($warnfile) {
			  		for ($i = 0; $i < $bcount ; $i++) { 
			  			print WARNFILE "$buffer[$i] (SuspectedFaulty)\n";
			  			$sfcount++;
			  		}
			  	}	
	    	 		if ($sfdiscard){ @buffer = (); $bcount = 0; }
		 		if ($sfprint) { $sftag = " (SuspectedFaulty)";}
			} elsif (/<\/mv>/) {
				for ($i = 0; $i < $bcount ; $i++) { 
		  			$output = $buffer[$i];
		  			if ($sftag eq '') { 
		  				if ($output =~ /;(,[^;]+|)$/) { 
		  					$empty++;
		  					if ($sfprint) { $output .= " (EmptyCounter)"; }
		  					elsif ($warnfile) { print WARNFILE "$output (EmptyCounter)\n" ;}
		  				}
		  				elsif ($output =~ /;-[0-9]+$/) {
		  					$negative++;
		  					if ($sfprint) { $output .= " (NegativeCounter)"; }
		  					elsif ($warnfile) { print WARNFILE "$output (NegativeCounter)\n" ;}
		  				}
		  			}
		  			else { 
		  				$sfcount++;
		  				if ($sfprint) { $output = $buffer[$i].$sftag; }
		  				elsif ($warnfile) { print WARNFILE "$output$sftag\n" ;}
		  			}
		  			print "$output\n";
		  			#print $buffer[$i].$sftag."\n" ;
				}
				@buffer = (); $bcount = 0; 
				$sftag = "";
				goto LOOKINGFORMONAMES;
			}
		}
	} elsif ($sncount == 1){
		while (<>) { 
			if ($_ =~ $countervaltemplate) {
		    	if ($currindex == $wantedctrindices[$wantedarraypos]) {
		    		$counter  = $wantedctrnames[$wantedarraypos++];
		    		$mocounter= $moldn.";$counter";
		    		$line  = $currprefix.";$counter";
		    		$fline = $line.";$1";
	    			if (defined $table{$line}) { $dupcounters{$mocounter} = 1; }
	    			else { 
	    				$buffer[$bcount++] = $fline ; 
	    				$table{$line} = 1; 
	    			}
				if ($wantedarraypos >= @wantedctrindices) {  #if there are no counters left to extract for this MO...
			       		goto LOOKINGFORMONAMES;
			      	}
			    }
		    	$currindex++;
		    	} elsif (/<sf>TRUE/) { 
		 		$sftrue = 1;
				if ($warnfile) {
			  		for ($i = 0; $i < $bcount ; $i++) { 
			  			print WARNFILE "$buffer[$i] (SuspectedFaulty)\n";
			  			$sfcount++;
			  		}
			  	}	
	    	 		if ($sfdiscard){ @buffer = (); $bcount = 0; }
		 		if ($sfprint) { $sftag = " (SuspectedFaulty)";}
		  	} elsif (/<\/mv>/) {
		  		for ($i = 0; $i < $bcount ; $i++) { 
		  			$output = $buffer[$i];
		  			if ($sftag eq '') { 
		  				if ($output =~ /;(,[^;]+|)$/) { 
		  					$empty++;
		  					if ($sfprint) { $output .= " (EmptyCounter)"; }
		  					elsif ($warnfile) { print WARNFILE "$output (EmptyCounter)\n" ;}
		  				}
		  				elsif ($output =~ /;-[0-9]+$/) {
		  					$negative++;
		  					if ($sfprint) { $output .= " (NegativeCounter)"; }
		  					elsif ($warnfile) { print WARNFILE "$output (NegativeCounter)\n" ;}
		  				}
		  			}
		  			else { 
		  				$sfcount++;
		  				if ($sfprint) { $output = $buffer[$i].$sftag; }
		  				elsif ($warnfile) { print WARNFILE "$output$sftag\n" ;}
		  			}
		  			print "$output\n";
		  			#print $buffer[$i].$sftag."\n" ;
		  		}
		  		@buffer = (); $bcount = 0; 
		  		$sftag = "";
		    		goto LOOKINGFORMONAMES;
		  	}
		}
	}

ENDFILE:
	if ($dupsize > 0) 
	{
		$warnoutput = "\nINFO: There are $dupsize counters that appear several times in the ROP files, duplicated instances have been discarded.\n";
		if ($warnfile)   { print WARNFILE $warnoutput;}
		elsif ($warnlog) { print WARNLOG  $warnoutput; warn $warnoutput; }
		else             { warn $warnoutput;}
	 	for my $element (sort keys %dupcounters) 
	 	{ 
	 		$warnoutput = "           - $element\n";
	 		if ($warnfile) {print WARNFILE $warnoutput;}
	 		elsif ($sfprint) { warn $warnoutput ; }
	 	}  
	 	$warnoutput = "It is recommended to remove duplicate counter definitions from the PM scanners.\n";
	 	if (!($sfprint)) { $warnoutput .= "To print the duplicated counters, run command \"pmxs\".\n"; }
		if ($warnfile)   { print WARNFILE $warnoutput ;}
		elsif ($warnlog) { print WARNLOG  $warnoutput ; warn $warnoutput; }
		else             { warn $warnoutput ; }
	}
	if ($sftrue > 0)
	{
		$sfwarn = "\nINFO: The ROP files contain $sfcount suspected faulty counter values.\n";
		if ($sfprint) { $sfwarn .= " They have been marked with the \"SuspectedFaulty\" label.\n"; }
		elsif ($sfdiscard) { $sfwarn .= "They have been discarded but can be kept with pmr/pmx option \"k\" (pmrk/pmxk) or highlighted with pmx option \"s\" (pmxs)\n"; }
		elsif ($sfkeep) { $sfwarn .= "To print the suspected faulty counters, run command \"pmxs\".\n" ; }
		if ($warnfile)   { print WARNFILE $sfwarn ;}
		elsif ($warnlog) { print WARNLOG  $sfwarn ; warn $sfwarn; }
		else             { warn $sfwarn;}
	}
	if ($empty > 0)
	{
		$sfwarn = "\nINFO: The ROP files contain $empty empty counter values.\n";
		if ($sfprint) { $sfwarn .= " They have been marked with the \"EmptyCounter\" label.\n"; }
		else { $sfwarn .= "To print the empty counters, run command \"pmxs\".\n" ; }
		if ($warnfile)   { print WARNFILE $sfwarn ;}
		elsif ($warnlog) { print WARNLOG $sfwarn ; warn $sfwarn; }
		else             { warn $sfwarn;}
	}
	if ($negative > 0)
	{
		$sfwarn = "\nINFO: The ROP files contain $negative negative counter values.\n";
		if ($sfprint) { $sfwarn .= " They have been marked with the \"NegativeCounter\" label.\n"; }
		else { $sfwarn .= "To print the negative counters, run command \"pmxs\".\n" ; }
		if ($warnfile)   { print WARNFILE $sfwarn ;}
		elsif ($warnlog) { print WARNLOG $sfwarn; warn $sfwarn; }
		else             { warn $sfwarn;}
	}
	if ($warnfile)   { close(WARNFILE); }
	elsif ($warnlog) { close(WARNLOG);  if ($dupsize > 0 || $sftrue > 0 || $empty > 0 || $negative > 0) { warn "\n";} }
	elsif ($dupsize > 0 || $sftrue > 0 || $empty > 0 || $negative > 0) {warn "\n";}
	exit;
