#!/usr/bin/perl
#----------------------------------------------------------------------------------------------
#
# pmXtab: Cross-tabulate PM counters
#
# Copyright (c) 2005 Ericsson Australia Pty Ltd
#
# This program may be used and/or copied only 
# with the prior written permission of Ericsson 
# Australia or in accordance with the terms and 
# conditions stipulated in the contract or agreement 
# under which the program has been supplied.
#----------------------------------------------------------------------------------------------
# v 1.0  03/08/2004 M. Harris    Initial Version, based on pmAggregate
# v 1.1  12/08/2004 M. Harris	Fix formatting for text output, add formulas
# v 1.2  30/08/2004 M. Harris    Correct error in formula calculation routine
#                                Add usage, add aggregation by related MO
#                                Fix formatting of floating point numbers
# v 1.3  22/11/2004 M. Harris    Add precision option
#                                Force formulas to evaluate in same order as file
#                                Change date/time column heading
# v 1.4  22/11/2004 M. Harris    Change behaivour so unmatched rows are discarded rather than die
# v 1.5  24/02/2005 M. Harris    Intelligent date formatting - if all on same day then remove date
#                                Format column width to be at least as wide as precision of numbers
#                                Sort Objects numerically if they are all numbers
# v 1.6  28/02/2005 M. Harris    Add default for precision
#                                Fix formatting errors; minimum width of Object column
#                                Right justify text and left justify numbers
#                                Use format_data for headings as well as data rows
#                                Add -pipe option
# v 1.7  28/02/2005 M. Harris    Fix case where -d option results in null times
# v 1.8  19/03/2005 M. Harris    Add aggregate sum functions
#                                Remove 'our' for backward compatibility with perl 5.0 (D. Smith)
#                                Enhance formatting for cols=time over multiple days
# v 1.9  23/03/2005 M. Harris    Add -a option to aggregate all time periods together
# v 1.10 06/04/2005 M. Harris    Only call tput if trunc used
#                                Right-align dates if they are column headings
#                                Change handling of -pipe; bidirectional pipe now used
#                                to ensure data ordering on STDOUT
# v 1.11 17/04/2005 M. Harris    Only try to reduce timestamps if not using csv output fmt
# v 1.12 21/06/2005 M. Harris    Add -filter, -sort and -top options
# v 1.13 12/12/2005 M. Harris    Add facility for multiple -r,-rdef options
#                                Add handling of array counter types
#                                Change all warning prints to 'warn' calls
#                                Only print warnings if formula error is not 'div by zero'
#                                Print warning & discard data of relation-map not matched
# v 1.14 16/03/2006 M. Harris    Change output formatting of integers from signed to unsigned
#                                Only output counters if they exist for a particular object
#                                Output headers for -a reports giving min & max time
# v 1.15 12/07/2006 M. Harris    Fix comment handling in formula files
# v 1.16 22/8/2006  M. Harris    Change formatting for integers to %ld (thanks esmidav)
# v 1.17 15/1/2007  M. Harris    Fix bug where last day is not displayed if -cols time
#                                is used on multiple day's worth of data.
# v 1.18 23/02/2007 T. Husz, F. Magnusson  Handling of non-pm attributes in ROP files
# v 1.19 23/03/2007 M. Aldrin	 Added an id tag to every row in the html table with the same name as cluster
# v 1.20 10/01/2008 F. Magnusson Remove duplicate warnings for unmatched rules in relation file.
# v 1.21 08/07/2008 M. Harris	 Added new subroutines for formula calculations (WeightedPercentile)
# v 1.22 05/09/2008 M. Harris	 Added new subroutines for formula calculations (MaxNonZeroIndex, MaxWeightedNonZeroIndex, MinNonZeroIndex, MinWeightedNonZeroIndex, MaximumWeightedValue)
# v 1.23 21/01/2009 M. Harris    Fix handling of large integer values, TR WRNae36124
# v 1.24 02/07/2009 M. Harris    New option -append used with mapfile (-rdef) to append the MO specified in option "-r" instead of substituting it
# v 1.25 05/09/2010 F. Magnusson Fix bug where empty counter values were printed as 0. 
# v 1.26 25/09/2010 F. Magnusson Changes in csv format printout, needed for pmrw. Remove duplicate warnings when fail to process a PDF formula.
# v 1.27 30/11/2010 F. Magnusson Modification of bug fix 1.25: only show empty counters when no aggregation. If -a/-d/-h is used then allow aggregation ==> empty counter values will be combined with non-empty counter values. Otherwise the whole aggregated value would have been empty.
# v 1.28 13/02/2011 F. Magnusson Fix bug introduced in 1.26 where formulas containing string attributes (e.g. userLabel) were being set to 0
#                                Missing semicolon separator between PDF counter values in csv format
# v 1.29 24/05/2012 M. Goransson Fix printout format of very small integers e.g 5.77290852864583e-05
# v 1.30 04/07/2013 F. Magnusson Subroutines Min, Max, Sum were not working
# v 1.31 16/09/2013 M. Harris    New subroutine decompressArray to decompress a compressed PDF counter, eg: De_pmPdcpVolDlDrbLastTTIQci = decompressArray(pmPdcpVolDlDrbLastTTIQci) , then PdcpVolDlDrbLastTTIQci_9 = @{De_pmPdcpVolDlDrbLastTTIQci}[9]
# v 1.32 23/11/2013 F. Magnusson Reduced verbosity of PDF formula exceptions
# v 1.33 03/01/2014 F. Magnusson Modification of bug fix 1.27: empty counters should be aggregated also in the case of MO aggregation at ManagedElement level eg "-m ManagedElement="
# v 1.34 16/02/2014 F. Magnusson Aggregate to object "Unknown" when object not found in ref file
# v 1.35 05/06/2014 F. Magnusson Aggregation on ManagedElement level was not working when some PDF counters contained empty values (,,,,,,,,,,)
# v 1.36 15/08/2014 F. Magnusson Allow aggregation even for counter names that don't start with "pm", as long as they contain a integer value
# v 1.37 28/09/2017 Z. He        Bug fix for aggregation of Qci counters
# v 1.38 27/10/2017 Z. He        Bug fix for aggregation of Qci counters with different array length between ROPs
# v 1.39 29/04/2018 H. Delpech   Add support for aggregation of compressed PDF counters
# v 1.40 29/04/2018 F. Magnusson Fixed column alignment for PDF counters
# v 1.41 01/05/2018 F. Magnusson Fixed aggregation for PDF counters when some ROPs contain non-array values
#----------------------------------------------------------------------------------------------

use Getopt::Long;
use IPC::Open2;
use POSIX;
use List::Util qw/ min max sum /;

my $version = "v 1.35 2014-06-05";
sub printUsage ($);
printUsage "At least one option must be specified!" if $#ARGV==-1;

# Globals
my %data, %ctr_map, %ts_map, %mo_map, %formulas, %warnlist, %warncntlist, %warncmplist;
my ($daily, $hourly, $mofilter, $cols, $fmt, @formulas_txt, $cntr_filter, $relmap_accum, $reldef_filename, $me_agg);
my @maxcol;
my ($min_time, $max_time, $csv_min_time, $csv_max_time);

# Defaults
$precision = "6.2";
$fmt = 'txt';
$cols= 'ctr';

GetOptions(
	"d"		 => \$daily,
	"h"		 => \$hourly,
	"a"      => \$all_time,
	"append" => \$relmap_accum,
	"m=s"    => \$mofilter,
	"cols=s" => \$cols,
	"fmt=s"  => \$fmt,
	"f=s"    => \@formulas_txt,
	"i=s"    => \$include_filter,
	"fdef=s" => \$formula_file,
	"rdef=s" => \@reldef_file,
	"r=s"    => \@relfilter,
	"x=s"    => \$cntr_filter,
	"p=s"    => \$precision,
	"trunc"  => \$truncate_lines,
	"help"   => \$help,
	"sort=s" => \$sort_key,
	"top=s"	 => \$top,
	"filter=s" => \$filter_exp,
	"pipe=s" => \$pipe_output
) or printUsage("Invalid Options");

my $tty_columns = `tput cols` if $truncate_lines;

if ($mofilter =~ /ManagedElement=/) { $me_agg=1 ; }

#----------------------------------------------------------------------------------------------
# Subroutines
#----------------------------------------------------------------------------------------------
# Print usage
#----------------------------------------------------------------------------------------------

sub decompressArray {
      my ($in) = @_;
      my @out;
      for (my $i=1; $i<@$in; $i+=2) {
            $out[$in->[$i]] = $in->[$i+1];
      }
      return \@out;
}

sub printUsage ($) {
	my $msg=shift;
	
	if ($msg) {print STDERR "\n\033[1;31mpmXtab: $msg\033[0m\n";};
	print STDERR <<_EOTXT_;

pmXtab: Cross-tabulate PM counters
$version

usage: pmXtab -cols (ctr|mo|time) -fmt (csv|txt|html|htmltab) [-p <precision>]
              [-trunc] [-h] [-d] [-m <mo-filter>] [-f formula ...] 
	      [-fdef <formulafile>] [-rdef <relationfile>] [-append]
	      [-r <relation_filter>] [-i <counter_filter>] [-x <output_filter>]
	      [-sort <ctr/formula>] [-top <n>] [-pipe <shell_cmd>]
_EOTXT_
	die "\n" if ($msg);
}

sub printHelp {
	
	printUsage("");
	print STDERR <<_EOTXT_;
	
Aggregation by Time:
--------------------
-h : Aggregate hourly
-d : Aggregate Daily
-a : Aggregate all time periods into a single bucket

Aggregation by MO Filter
------------------------
-m <filter> : Aggregate by a filtered MO expression
              This expression must be a perl RE which has one grouped expression
              This expression will be used as the aggregation
              Example: UtranCell=(.+)$ to aggregate on cell level
                       MeContext=(.+), to aggregate on NE level

Aggregation by related MO
-------------------------
-rdef <file> : File that specifies the MOs related to this one
               (eg. relationships between cells & iublinks, for instance)

               The file should be in the format:
               <base-mo>:<related-mo> [<related-mo>...]

-r <filter>  : A filter to select the related MO to aggregate on

The way this function works is that first, the -m filter is used to
extract the MO that will be used to match against a relation string.
Then the -r filter is used to select the relation that will be used.

Example: Suppose the relation file contains the following

UtranCell=12345:IubLink=rbs6554 RncModule=1
UtranCell=12346:IubLink=rbs6554 RncModule=1
UtranCell=12347:IubLink=rbs6554 RncModule=1
UtranCell=12348:IubLink=rbs6123 RncModule=1

If you wanted to aggregate cell counters to the RBS level then you would use
-m "(UtranCell=\\w+)" -r "IubLink=(\\w+)"
Or to the module level:
-m "(UtranCell=\\w+)" -r "RncModule=(\\w+)"

If you just want to specify the IubLink or RncModule next to the UtranCell (instead 
of replacing it), use the -append option. Example:
-m "(UtranCell=\\w+)" -r "(RncModule=\\w+)" -append

Formulas
--------
-f <formula>
	Add a formula based on counter values that are being fed to pmXtab
	A formula should consist of a name, then an equals sign, then an expression
	The expression may include counter names, constants, operators
	and functions (all operators & functions allowed in perl may
	be used, these are similar to those in C).

	Example:
	RabEstSuccRate = 100 * pmTotNoRrcConnectReqSuccess / pmTotNoRrcConnectReq
	
	Multiple formulas may be added by specifying -f option more than once.

-i <pattern>
	Include formulas from a formula definition file if they match <pattern>. The 
	formula definition file to be used is specified via option -fdef. Note that
	if this option is not specified then ALL formulas in the definition file
	will be included

-fdef <file>
	Specify a config file that contains formulas to be used.

Filtering and Sorting
---------------------
-pipe <unix cmd>
	Specify a shell command. Output (excluding header and footer) will be piped
	through this command. Applies only to csv and txt output formats. Useful to
	(for example) sort output : eg. -pipe "sort -n +2" will sort by the first
	data column
	
-sort <counter>
	Specify a sort field. The sort field should be one of the counters
	or formula results. Within each time aggregate, the MOs will be
	output sorted according to the given field. If the last character
	is a '-' then sort is descending, otherwise it is ascending.

-top <number>
	Specify a maximum number of rows to output. When used with -sort this 
	allows for 'top 10' type reports.

Output Formatting
-----------------
-x <pattern>
	Specify a filter to be used to filter counters before output. If specified
	all available counters (including the results of formulas) will be filtered
	using this expression before being output. Can be used to remove counters
	while leaving calculated values.
	       
-cols <ctr|mo|time>
	Specify column headings
	ctr : Counter names will be column headings
	mo  : Managed Objects will be column headings
	      (use with -m option)
	time: Times will be column headings

-fmt <txt|csv|html|htmltab>
	Specify output format
	txt	: Text only (default)
	csv	: Comma separated
	html	: HTML
	htmltab : HTML tables only (use for embedding into a web page)
	
-p <x.y>
	Specify precision for non-integer data
	x=total number of digits
	y=number of decimal places

-trunc
	Specify if lines should be truncated if they would wrap on the
	terminal (only for text format)	       

More information on pmTools can be found at http://newtran01.au.ao.ericsson.se/utranwiki/pmTools

_EOTXT_
	exit;
}

#----------------------------------------------------------------------------------------------
# Calculate column widths (only used for text based report)
#----------------------------------------------------------------------------------------------
sub col_widths {
	
	my $ind1,$ind2,$ind3,$i;
	my $max_ts=0;
	
	# Start with the column headings
	# Note that subroutine should be called with @_ = headings for columns
	$i=0;
	$maxcol[$i++]=($df_numchars > length($_)) ? $df_numchars : length($_) foreach (@_);
	
	$max_ts = ($max_ts < length($_)) ? length($_) : $max_ts foreach (keys %ts_map);
	$max_ctr = ($max_ctr < length($_)) ? length($_) : $max_ctr foreach (keys %ctr_map);
	$max_mo = 6; # Min column size for MO column must be able to contain the word 'object'
	$max_mo = ($max_mo < length($_)) ? length($_) : $max_mo foreach (keys %mo_map);
	
	if ($cols eq 'ctr') {
		$maxcol[0]=$max_ts;
		$maxcol[1]=$max_mo;
	} elsif ($cols eq 'mo') {
		$maxcol[0]=$max_ts;
		$maxcol[1]=$max_ctr;
	} elsif ($cols eq 'time') {
		$maxcol[0]=$max_mo;
		$maxcol[1]=$max_ctr;
	}					 
	
	# Return is in global @maxcol
	#print "Col Width $_\n" foreach (@maxcol);

}

#----------------------------------------------------------------------------------------------
# Format data, used in text output only
#----------------------------------------------------------------------------------------------
sub format_data {

	my ($d,$w) = @_;
	my $joinedarray;

	# Fixme: this is very crude. Columns, etc will not line up!
	if (ref $d eq 'ARRAY') {
		$joinedarray = join(",", @{$d});
		return sprintf("%*s",$w,$joinedarray);
	}

	if ($d =~ /^[\+\-]?[\d\.]+(e\-\d+)?$/) {
	
		# Decimal Integer
		return sprintf "%${w}.0f",($d) if ($d ==int($d));
	
		# otherwise must be a floating Point Number
		return sprintf "%${w}${df_numdec}f",($d);
		
	}
	
	# Right Align times if in columns
	return sprintf "%${w}s",($d) if ($cols eq 'time' and $d =~ /^\d\d:\d\d$/);

	# Special Case
	return sprintf "%${w}s",($d) if ($d eq "N/A");
	
	# Otherwise assume a string
	return sprintf "%-${w}s",($d)
		
}
#----------------------------------------------------------------------------------------------
# Output a heading row
#----------------------------------------------------------------------------------------------
sub out_headrow {
	
	my $extra_headingline=shift;
	
	if ($fmt eq 'html') {
		print <<_EOF_;
<html>
	<head>
	<title>pmTools - pmXtab Output</title>
	<style TYPE="text/css">
        	<!--
			table {
				border-collapse: collapse;
				border: 1px solid black;
				padding: 0;
			}
			
			th {
				border: 1px solid black;
				font: 10pt Tahoma,Arial,Helvetica;
				font-weight: bold;
				background-color: #9999CC;
				margin: 0;
			}
			
			td {
				border: 1px solid black;
				font: 10pt Tahoma,Arial,Helvetica;
				color: black;
				background-color: #CCCCCC;
				padding: 2;
				margin: 0;
				text-align: center;
			}
			-->
			</style>
	</head>
	<body>
_EOF_
	} 
	my $wline;
	my $cmpline;
	my $cntline;
	foreach $wline (keys %warncntlist) { $cntline .= "$wline " }
	foreach $wline (keys %warncmplist) { $cmpline .= "$wline " }
	foreach $wline (keys %warnlist) 
	{ 
		if ($wline =~ /Cannot add array to non-array/) {$wline .= "$cntline\n";}
		if ($wline =~ /Cannot add compressed array to non-array/) {$wline .= "$cmpline\n";}
		warn $wline ;
	}
	if ($fmt eq 'txt') {
		col_widths(@_);
		print "\n$extra_headingline";
		my $i=0;
		my $hdr="";
		$hdr = join(" ",map(format_data($_, $maxcol[$i++]), @_));
		$hdr = substr($hdr, 0, $tty_columns) if $truncate_lines;
		print "$hdr\n";
	} elsif ($fmt eq 'csv') {
		print "$extra_headingline";
		print "$_; " foreach (@_);
		print "\n";
	} elsif ($fmt eq 'html' or $fmt eq 'htmltab') {
		print "$extra_headingline<br><table>\n<tr>";
		$_ && print "<th>$_</th>" foreach (@_);
		print "</tr>\n";
	}
}
#----------------------------------------------------------------------------------------------
# Output a data row
#----------------------------------------------------------------------------------------------
sub out_row {
	
	if ($fmt eq 'txt') {
		my $i=0;
		my $line="";
		$line = join(" ",map(format_data($_, $maxcol[$i++]), @_));
		$line = substr($line, 0, $tty_columns) if $truncate_lines;
		print $outputfh "$line\n";
	} elsif ($fmt eq 'csv') {
		my $line="";
		foreach $d (@_) {
			if (ref $d eq 'ARRAY') {
				$line .= join(',',@{$d});
				$line .= "; ";
			} else {
				if ($d =~ /^[\+\-]?[\d\.]+(e\-\d+)?$/) { 
					if ($d ==int($d)) { $line .= sprintf "%${w}.0f; ",($d); }
					else {$line .= sprintf "%${w}${df_numdec}f; ",($d); }
				}
				else { $line .= "$d; "; }
			}
		}
		print $outputfh "$line\n";
	} elsif ($fmt eq 'html' or $fmt eq 'htmltab') {
		# if -a was specified then no date column
		shift if ($all_time);
		print "<tr id=@_[0]>";
		print "<td>".format_data($_)."</td>" foreach (@_);
		print "</tr>\n";
	}
	
	# If we are piping through a filter: see if there is anything here to read
	# NB. FILTER_IN is set to nonblocking at this time
	if ($pipe_output) {
		my $l=<FILTER_IN>;
		print $l if $l;
	}	 
	
}
#----------------------------------------------------------------------------------------------
# Output footer
#----------------------------------------------------------------------------------------------
sub out_footer {
	
	my $l="";
	my $finished=0;
	
	# Now we need to see if there is any more data left in the filter pipe
	# Note that for tools like sort, they read all data then output only
	# after their input is closed..
	# First reset nonblocking mode of FH, then close filter pipe output FH,
	# then read all remaining lines in the filter pipe input FH
	if ($pipe_output) {
		fcntl(\*FILTER_IN, F_SETFL(), 0);
		close FILTER_OUT;
		while (<FILTER_IN>) {
			print $_;
		}
		close FILTER_IN;
		waitpid $pipe_pid,0;
	}
	
	if ($fmt eq 'htmltab') {
		print "</table>\n";
	} elsif ($fmt eq 'html') {
		print "</table>\n";
		print "</body></html>\n";	
	} elsif ($fmt eq 'txt') {
		print "\n";
	}
}
#----------------------------------------------------------------------------------------------
# Reduce Time Formats
#----------------------------------------------------------------------------------------------
sub reduce_time_format {
	
	my $day="";
	my $ts,$hm,$day;
	
	# If using csv format then we don't want to do it
	# (assume some downstream tool will be reading, or maybe
	# import to excel)
	if ($fmt eq 'csv') {
		if ($all_time) {
			return (0, "","Start Time: $min_time End Time: $max_time Granularity: whole period.\n");
		}
		elsif ($hourly) {
			return (0, "Date/Time","Start Time: $csv_min_time End Time: $csv_max_time Granularity: hour.\n") ;
		}
		elsif ($daily) {
			return (0, "Date/Time","Start Time: $csv_min_time End Time: $csv_max_time Granularity: day.\n") ;
		}
		else {
			return (0, "Date/Time","Start Time: $csv_min_time End Time: $csv_max_time Granularity: rop.\n") ;
		}
	}
	
	# If time to be aggregated into a bucket
	if ($all_time) {
		return (0, "","Start Time: $min_time End Time: $max_time\n");
	}
	
	# If daily aggregation then only date is given
	if ($daily) {
		return (0, "Date","");
	}
	
	# Determine if all samples in the same day
	foreach $ts (sort keys %data) {
		if (! $day) {
			$day=substr($ts,0,10);
		} else {
			return (0, "Date       Time","") if ($day ne substr($ts,0,10));
		}
	}
	
	# If we reach this point then the date is the same for all entries
	undef %ts_map;
	foreach $ts (sort keys %data) {
		$hm = substr($ts,11);
		$data{$hm}=$data{$ts};
		$sum_time{$hm}=$sum_time{$ts};
		$ts_map{$hm}=1;
		delete $data{$ts};
		delete $sum_time{$ts};
	}
	return(1, "Time", "Date: $day\n");

}
#----------------------------------------------------------------------------------------------
# Parse Formulas
#----------------------------------------------------------------------------------------------
# We don't really parse the formulas. We simply convert them into a perl-like syntax so that
# we can 'eval' them later. The following steps are taken:
# 1. Separate formula name (bit before =) from formula itself
# 2. Replace all instances of counterName in the formula to the form $ctrs->{counterName}
# 3. Replace all instances of sum_counterName with $sum{counterName}. Same with sum_mo and sum_time
# 4. Stash each formula in the hash %formulas
sub parse_formulas {
	
	my $f, $ctr, $name;

	foreach $f (@formulas_txt) {
		
		printUsage "Formula must start with 'name='" if (! ($f =~ /^(@*\w+)\s*=\s*(.+)$/));
		$name = $1;
		$f = $2;
				
		# Each counter referenced in the formula must be replaced by
		# $ctrs->{name}
		# Also, sum_xxxxx must be replaced with $sum{name}
		foreach $ctr (keys %ctr_map) {
			$f =~ s/\b$ctr\b/\$ctrs->{$ctr}/g;
			$f =~ s/\bsum_$ctr\b/\$sum{$ctr}/g;
			$f =~ s/\bsum_mo_$ctr\b/\$tsum_mo->{$ctr}/g;
			$f =~ s/\bsum_time_$ctr\b/\$tsum_time->{$ctr}/g;
		}
		
		$formulas{$name}=$f;
		push @formula_list, $name;
		$ctr_map{$name}=1;
		
		#print "Compiled Formula: $name = $f\n";
		
	}
	
	if ($filter_exp) {
		foreach $ctr (keys %ctr_map) {
			$filter_exp =~ s/\b$ctr\b/\$data{\$ts}{\$mo}{$ctr}/g;
		}
		#print "Compiled Filter_exp: $filter_exp\n";
	}
	
}

#----------------------------------------------------------------------------------------------
# Execute Formulas
#----------------------------------------------------------------------------------------------
sub exec_formulas {

	my $ts, $mo, $ctr, $fname,$r;
	my $ctrs, $tsum_mo, $tsum_time;
	
	foreach $ts (keys %data) {
		$tsum_time = \%{$sum_time{$ts}};
		foreach $mo (keys %{$data{$ts}}) {
			$tsum_mo = \%{$sum_mo{$mo}};
			$ctrs = \%{$data{$ts}{$mo}};
			foreach $fname (@formula_list) {
				
				# Formula names that start with @ indicate that
				# we want an array as a result
				if ($fname =~ /^@/) {
					@{$data{$ts}{$mo}{$fname}} = eval $formulas{$fname};
					if ($@ and $@ !~ /division by zero/) { $warnlist{"Error evaluating formula $fname"} = 1; } # = $formulas{$fname}:\n$@\n
				} else {
					$r = eval $formulas{$fname};
					if ($@ and $@ !~ /division by zero/) { $warnlist{"Error evaluating formula $fname"} = 1; } # = $formulas{$fname}:\n$@\n
					$data{$ts}{$mo}{$fname} = $@ ? "N/A":$r;
				}
			}
			
		}
	
	}
		
}
#----------------------------------------------------------------------------------------------
# Sum of array elements : op1=op1+op2
# Sum of compressed array elements : op1=op1+op2 handling of compressed arrays like 2,1,18,5,602
#----------------------------------------------------------------------------------------------
sub array_add {
	my $op1=shift;
	my $op2=shift;
	my $op3=shift;
	
	my $i;
	my $j;
	my $k;
	my $n=0;
	my $p=0;
	my $z=0;

	# Sum of array elements : op1=op1+op2
	if (!(exists $CompressedCounters{$op3})){
		if ((defined $$op1)&&(!(ref $$op1 eq 'ARRAY'))) { undef $$op1 ; }
		if (defined $$op1) {
			if (ref $$op1 eq 'ARRAY'|| ref $op2 eq 'ARRAY') {
				if (ref $$op1 eq 'ARRAY') { $n=$#{$$op1} ; }
				if (ref $op2 eq 'ARRAY') { $p=$#{$op2} ; }
				$z=$n;
				if ($p > $n) { $z=$p ;}
				for ($i=0; $i<=$z; $i++) {
					$$op1->[$i]+=$op2->[$i];
				}
			} else {
				# Trying to add an array to a non-array: abort!
				$warnlist{"Cannot add array to non-array: "} = 1;
				$warncntlist{"$op3"} = 1;
			}
		} else {
			if (ref $op2 eq 'ARRAY') {
				@{$$op1}=@{$op2};
			} else {
				#OP1 is created with a zero value of OP2
				$$op1->[0]=0;
			}
		}
		return
	}
	#Sum of compressed array elements 
	if (defined $$op1) {
		if ((ref $$op1 eq 'ARRAY') && (ref $op2 eq 'ARRAY')) {
			#print "Array Case\n";
			#check if QCI index are same before adding up
			#in case two ROP counters with different length like 3,1,1,5,20,9,22 and 2,5,22,9,18
			if ($$op1->[0] != 0) {
				for ($i=2; $i<=$#{$op2}; $i+=2) {
					for ($j=2;$j<=$#{$$op1}; $j+=2) { 
						if ( $$op1->[$j-1] == $op2->[$i-1] ) {
						#Insert OP1/OP2 QCI value matching
							$$op1->[$j]+=$op2->[$i];
							$j=$#{$$op1}+1;
						}
						elsif ( $$op1->[$j-1] > $op2->[$i-1] ) {
						#OP2 QCI value to be inserted in OP1
							for ($k=$#{$$op1}; $k>=$j; $k-=2) {
								$$op1->[$k+1]=$$op1->[$k-1];
								$$op1->[$k+2]=$$op1->[$k];
							}						
							$$op1->[$j]=$op2->[$i];
							$$op1->[$j-1]=$op2->[$i-1];
							$$op1->[0]++;
							$j=$#{$$op1}+1;
						}
						elsif (( $$op1->[$j-1] < $op2->[$i-1] ) && ( [$j] == $#{$$op1})) {
							#Insert OP2 QCI at the end OP1 QCI 
							$$op1->[$#{$$op1}+1]=$op2->[$i-1];
							$$op1->[$#{$$op1}+2]=$op2->[$i];
							$$op1->[0]++;
							$j=$#{$$op1}+1;
						}
					}
				}
			} else {
				# If OP1 had zero value only, it copies non zero OP2
				@{$$op1}=@{$op2};
			}
		} else {
			#print "Not Array Case\n";
			# Trying to add an array to a non-array: abort!
			$warnlist{"Cannot add compressed array to non-array: "} = 1;
			$warncmplist{"$op3"} = 1;
			return;
		}
	} else {
		if (ref $op2 eq 'ARRAY') {
			@{$$op1}=@{$op2};
		} else {
			#OP1 is created with a zero value of OP2
			$$op1->[0]=0;
		}
	}
}

#----------------------------------------------------------------------------------------------
# Formula support functions
#----------------------------------------------------------------------------------------------
# Average(\@array) : return average of the array elements
sub Average {
	my $d=shift; # Ref to array to be averaged
	my $sum=0;
	
	die "Non-array supplied to Average()" unless ref $d eq 'ARRAY'; 
	$sum+=$_ foreach (@{$d});
	return scalar @{$d} ? $sum/(scalar @{$d}) : 0;
}

# Count(\@array) : return number of elements in array
sub Count {
	my $d=shift;
	return scalar @{$d};
}

# WeightedAverage(\@counterarray, \@weightingarray) : each element of counterarray
# is multiplied by the corresponding element of the weighting array, then total
# is divided by the sum of the elements in counterarray
sub WeightedAverage {
	my $d=shift;
	my $w=shift;
	my $i;
	my $wsum=0;
	my $count=0;
	
	die "Non-Array supplied as first parameter to WeightedAverage()" unless ref $d eq 'ARRAY';
	die "Non-Array supplied as second parameter to WeightedAverage()" unless ref $w eq 'ARRAY';
	
	for ($i=0; $i<=$#{$d}; $i++) {
		$count+=$d->[$i];
		$wsum+=($d->[$i] * $w->[$i]);
	}
	
	return $count ? $wsum / $count : 0;
}

##-----------------------------------------------------------------------------
# log10($x)
# Calculate the base 10 log of $x. Useful for mathematics involving dB.
# - $x		Input value
# RETURNS: $result
# - $result	Base 10 log of $x
#------------------------------------------------------------------------------
sub log10 {
	# using basic algebraic principle: log_n(x) = log_e(x) / log_e(n)
	my $x=shift;
	if (($x+0) == 0) { return "N/A"; }
	return log($x)/2.30258509299405; # denom is log(10). 
}
##-----------------------------------------------------------------------------
# Sum($array)
# Return sum of elements in array
# - $array	Ref to array
# RETURNS: $sum
# - $sum	Sum of elements
#------------------------------------------------------------------------------
sub Sum {
	return sum (@{$_[0]});
}
##-----------------------------------------------------------------------------
# Max($array)
# Return max of elements in array
# - $array Ref to array
# RETURNS: $max
# - $max Maximum Array Value
#------------------------------------------------------------------------------
sub Max {
	return max(@{$_[0]});
}
##-----------------------------------------------------------------------------
# Min($array)
# Return min of elements in array
# - $array Ref to array
# RETURNS: $min
# - $min Minimum Array Value
#------------------------------------------------------------------------------
sub Min {
	return min(@{$_[0]});
}
##-----------------------------------------------------------------------------
# Percentile($percentile, $array)
# Return the supplied percentile of the given array.
# - $percentile  Percentile to calculate
# - $array       Ref to array
# RETURNS: $weightedavg
# - $perc        Weighted Average of $array
#------------------------------------------------------------------------------
sub Percentile {
        my ($percentile,$arr) = @_;

        die "Non-Array supplied as first parameter to Percentile()" unless ref $arr eq 'ARRAY';

        #$srv->logDebug(5,"Input array is as follows: @{$arr}") if DEBUG5;
        # copy array so we can modify it
        my @parray = @$arr;

        $percentile /= 100;

        my ($fold, $ifold);
        @parray = sort {$a <=> $b} @parray;

        $fold = @parray*$percentile;
        $ifold = int(@parray*$percentile);

        if (($fold - $ifold) > .001) {
                return $parray[$ifold];
        } else {
                return ($parray[$ifold] + $parray[$ifold-1]) / 2;
        }
}
##-----------------------------------------------------------------------------
# WeightedPercentile($percentile, $array, $weightingarray)
# Return weighted percentile. Each element of the array is multiplied by the
# corresponding element of the weighting array, the resulting array is then
# used to calculate the given percentile.
# - $percentile           The percentile to calculate
# - $array                Ref to array
# - $weightingarray       Ref to array of weightings
# RETURNS: $weightedpercentile
# - $weightedpercentile   Weighted percentile of $array
#------------------------------------------------------------------------------
sub WeightedPercentile {
        my $p=shift;
        my $d=shift;
        my $w=shift;

        my $count;

        $p /= 100;

        die "Non-Array supplied as first parameter to WeightedPercentile()" unless ref $d eq 'ARRAY';
        die "Non-Array supplied as second parameter to WeightedPercentile()" unless ref $w eq 'ARRAY';
        die "Counter Array and Weighting array of different sizes in WeightedPercentile()" unless $#{$d} == $#{$w};

        # Find the count
        for (my $i=0; $i<=$#{$d}; $i++) {
                $count+=$d->[$i];
        }

        #$srv->logDebug(5,"Number of data points: $count") if DEBUG5;

        # Calculate the value where the count/percentile lies.  Since we are not zero based (as normal arrays)
        # we need to add 1 to this calculation
        my ($fold, $ifold);
        $fold = $count*$p + 1;
        $ifold = int($count*$p) + 1;

        #$srv->logDebug(5,"ifold: $ifold, fold: $fold") if DEBUG5;

        # So we want to figure out what weighting value is
        # at the count value at the given percentile
        $count = 0;
        my $prevWeighting;
        for (my $i=0; $i<=$#{$d}; $i++) {
                $count+=$d->[$i];

                # We're at the right place now
                if($count >= $ifold) {
                        # We don't need to average two weighting values,
                        # so just return the current weighting value
                        if (($fold - $ifold) > .001) {
                                return $w->[$i];
                        } else {
                                # This is the count point for this weighting value
                                my $startCount = $count - $d->[$i] + 1;

                                # If we start after the fold point, we need to take
                                # an average - otherwise we are in the range of this
                                # weight.
                                if($startCount > $ifold-1) {
                                        return (($w->[$i] + $prevWeighting) / 2);
                                } else {
                                        return $w->[$i];
                                }
                        }
                }

                # Store for next loop
                $prevWeighting = $w->[$i];
        }
}
##-----------------------------------------------------------------------------
# MaxNonZeroIndex($array)
# Return the highest index with a non-zero value
# - $array Ref to array
# RETURNS: $max_index
# - $max_index The highest index with non-zero value.
#------------------------------------------------------------------------------
sub MaxNonZeroIndex {
  my $d=shift;

  die "Non-Array supplied as first parameter to MaxNonZeroIndex()" unless ref $d eq 'ARRAY';

  my $max_index;
  my $i = 0;
  for(;$i<$#{$d};$i++) {
    $max_index = $i if($d->[$i] != 0);
  }

  return $max_index;
}

##-----------------------------------------------------------------------------
# MaxWeightedNonZeroIndex($array,$weightingarray)
# Return the weighted value of the highest index with a non-zero value in counter array
# - $array Ref to array
# RETURNS: $max_index_value
# - $max_index_value The value of the weighting array at the highest index with non-zero value.
#------------------------------------------------------------------------------
sub MaxWeightedNonZeroIndex {
        my $d=shift;
        my $w=shift;

        die "Non-Array supplied as first parameter to MaxWeightedNonZeroIndex()" unless ref $d eq 'ARRAY';
        die "Non-Array supplied as second parameter to MaxWeightedNonZeroIndex()" unless ref $w eq 'ARRAY';
  die "Counter Array and Weighting array of different sizes in MaxWeightedNonZeroIndex()" unless $#{$d} == $#{$w};

        my $max_index;
        my $i = 0;
        for(;$i<$#{$d};$i++) {
                $max_index = $i if($d->[$i] != 0);
        }

        return $w->[$max_index];
}

##-----------------------------------------------------------------------------
# MinNonZeroIndex($array)
# Return the lowest index with a non-zero value
# - $array Ref to array
# RETURNS: $min_index
# - $min_index The highest index with non-zero value.
#------------------------------------------------------------------------------
sub MinNonZeroIndex {
        my $d=shift;

        die "Non-Array supplied as first parameter to MinNonZeroIndex()" unless ref $d eq 'ARRAY';

        for(my $i=0;$i<$#{$d};$i++) {
                return $i if($d->[$i] != 0);
        }
}

##-----------------------------------------------------------------------------
# MinWeightedNonZeroIndex($array,$weightingarray)
# Return the weighted value of the lowest index with a non-zero value in counter array
# - $array Ref to array
# RETURNS: $min_index_value
# - $min_index_value The value of the weighting array at the lowest index with non-zero value.
#------------------------------------------------------------------------------
sub MinWeightedNonZeroIndex {
        my $d=shift;
        my $w=shift;

        die "Non-Array supplied as first parameter to MinWeightedNonZeroIndex()" unless ref $d eq 'ARRAY';
        die "Non-Array supplied as second parameter to MinWeightedNonZeroIndex()" unless ref $w eq 'ARRAY';
        die "Counter Array and Weighting array of different sizes in MinWeightedNonZeroIndex()" unless $#{$d} == $#{$w};

        for(my $i=0;$i<$#{$d};$i++) {
                return $w->[$i] if($d->[$i] != 0);
        }
}

##-----------------------------------------------------------------------------
# MaximumWeightedValue($array, $weightingarray)
# Return the value within the weighted array which occurs the most in the counter array
# - $array              Ref to array
# - $weightingarray     Ref to array of weightings
# RETURNS: $maxWeightingValue
# - $maxWeightingValue        Max Weighting value
#------------------------------------------------------------------------------
sub MaximumWeightedValue {
        my $d=shift;
        my $w=shift;

        die "Non-Array supplied as first parameter to MaximumWeightedValue()" unless ref $d eq 'ARRAY';
        die "Non-Array supplied as second parameter to MaximumWeightedValue()" unless ref $w eq 'ARRAY';
  die "Counter Array and Weighting array of different sizes in MaximumWeightedValue()" unless $#{$d} == $#{$w};

  my ($max,$max_pos,$i);
  for($i = 0;$i<$#{$d};$i++) {
    if(!defined $max || $d->[$i] > $max) {
      $max = $d->[$i];
      $max_pos = $i;
    }
  }

  return $w->[$max_pos];
}


#----------------------------------------------------------------------------------------------
# MAIN
#----------------------------------------------------------------------------------------------

# Validate parameters
printHelp() if ($help);
die("-h and -d are mutually exclusive\n") if ($hourly && $daily);
die("Invalid Column Specification\n") if !($cols =~ /^(ctr|mo|time)$/);
die("Invalid Format Specification\n") if !($fmt =~ /^(txt|csv|html|htmltab)$/);
die("-rdef must be specified if -r is used\n") if (@relfilter and !@reldef_file);
die("Number of -rdef and -r options must match\n") if ($#relfilter != $#reldef_file);
die("-m must be specified if -r is used\n") if (@relfilter and ! $mofilter);
die("-pipe only valid for txt or csv output formats\n") if ($pipe_output and $fmt !~ /^(txt|csv)$/);
die("-sort can only be used if -cols=ctr\n") if ($sort_key and $cols ne 'ctr');
die("-filter can only be used if -cols=ctr\n") if ($filter_exp and $cols ne 'ctr');

# Precompile filters
my $cntr_filter_re = qr/$cntr_filter/;
my $include_filter_re = qr/$include_filter/;
my $mofilter_re = qr/$mofilter/;
my $i;
my @relfilter_re;
for ($i=0; $i<=$#relfilter; $i++) {
	my $re=$relfilter[$i];
	$relfilter_re[$i] = qr/$re/;
}

# Precision of data fields
if ($precision =~ /(\d+)(\.\d+)/) {
	$df_numchars=$1;
	$df_numdec=$2;
} else {
	die "Precision must be in form x.y\n";
}

# If user wants to pipe output..
if ($pipe_output) {
	$pipe_pid=open2(\*FILTER_IN,\*FILTER_OUT,"$pipe_output");
	$outputfh = \*FILTER_OUT;
	# Set FILTER_IN to non-blocking, so we don't have any deadlocks
	fcntl(\*FILTER_IN, F_SETFL(), O_NONBLOCK());
} else {
	$outputfh = \*STDOUT;
}


# Read predefined formulas from file, if supplied
if ($formula_file) {
	open FFILE, "<$formula_file" or die "Could not open $formula_file";
	while (<FFILE>) {
		chomp;
		s/#.*$//;          # Remove Comments
		next if (/^\s*$/); # Skip blank lines
		push @formulas_txt,$_ if ( (! $include_filter) || $_ =~ $include_filter_re);
	}
	close FFILE;
}

# Read Mo Relations from File if supplied
if (scalar @reldef_file) {
	for ($i=0; $i<=$#reldef_file; $i++) {
		my ($mo, $rmo);
		open RFILE, "<$reldef_file[$i]" or die "Could not open $reldef_file[$i]";
		while (<RFILE>) {
			next if (/^\s*$/); # Skip blank lines
			($mo, $rmo) = split ':';
			$relmo[$i]{$mo}=$rmo;
		}
		close RFILE;
	}
}

# Creation of hash table containing Compressed counters
our %CompressedCounters;
my @a = split(' ','pmActiveDrbDlSumQci pmActiveUeDlSumQci pmBranchDeltaSinrDistr0 pmBranchDeltaSinrDistr1 pmBranchDeltaSinrDistr2 pmBranchDeltaSinrDistr3 pmBranchDeltaSinrDistr4 pmBranchDeltaSinrDistr5 pmBranchDeltaSinrDistr6 pmCaWaitGoodCov pmDrbThpTimeDlQci pmErabEstabAttAddedGbrArp pmErabEstabAttAddedHoOngoingArp pmErabEstabAttAddedHoOngoingQci pmErabEstabAttAddedQci pmErabEstabAttInitGbrArp pmErabEstabAttInitQci pmErabEstabAttRejAdmMsrAddedArp pmErabEstabAttRejAdmMsrInitArp pmErabEstabFailUeAdmLicRejArp pmErabEstabSuccAddedGbrArp pmErabEstabSuccAddedQci pmErabEstabSuccInitGbrArp pmErabEstabSuccInitQci pmErabLevRopStartQci pmErabModAttQci pmErabModSuccQci pmErabQciLevSum pmErabQciMax pmErabRelAbnormalEnbActArp pmErabRelAbnormalEnbActCdtAutoPnrQci pmErabRelAbnormalEnbActCdtAutoQci pmErabRelAbnormalEnbActCdtQci pmErabRelAbnormalEnbActGbrArp pmErabRelAbnormalEnbActHoQci pmErabRelAbnormalEnbActHprQci pmErabRelAbnormalEnbActPeQci pmErabRelAbnormalEnbActQci pmErabRelAbnormalEnbActTnFailQci pmErabRelAbnormalEnbActUeLostQci pmErabRelAbnormalEnbArp pmErabRelAbnormalEnbCdtAutoPnrQci pmErabRelAbnormalEnbCdtAutoQci pmErabRelAbnormalEnbCdtQci pmErabRelAbnormalEnbGbrArp pmErabRelAbnormalEnbHoQci pmErabRelAbnormalEnbHprQci pmErabRelAbnormalEnbPeQci pmErabRelAbnormalEnbQci pmErabRelAbnormalEnbTnFailQci pmErabRelAbnormalEnbUeLostQci pmErabRelAbnormalMmeActArp pmErabRelAbnormalMmeActGbrArp pmErabRelAbnormalMmeActQci pmErabRelAbnormalMmeArp pmErabRelAbnormalMmeGbrArp pmErabRelAbnormalMmeQci pmErabRelAllEnbQci pmErabRelAllMmeQci pmErabRelDlInactGapQci pmErabRelMmeActQci pmErabRelMmeArp pmErabRelMmeGbrArp pmErabRelMmeQci pmErabRelNormalEnbActArp pmErabRelNormalEnbActQci pmErabRelNormalEnbArp pmErabRelNormalEnbGbrArp pmErabRelNormalEnbQci pmErabRelNormalEnbUserInactivityQci pmErabRelUlInactGapQci pmHoExeOutAttQci pmHoExeOutSuccQci pmInhibitImeisvFeature pmLaaSelectedChannel pmMacDrbThpTimeDlQci pmMacDrbThpVolDlQci pmMacLatTimeDlDrxSyncQci pmMacLatTimeDlNoDrxNoSyncQci pmMacLatTimeDlNoDrxSyncQci pmMacLatTimeDlQci pmMacTimeDlDrxSyncSampQci pmMacTimeDlNoDrxNoSyncSampQci pmMacTimeDlNoDrxSyncSampQci pmMacTimeDlSampQci pmModImeisvParamSet pmPdcchCceUsedDlQci pmPdcchCceUsedUlQci pmPdcpLatPktTransDlQci pmPdcpLatTimeDlQci pmPdcpPktDiscDlAqmQci pmPdcpPktDiscDlHoQci pmPdcpPktDiscDlPelrQci pmPdcpPktDiscDlPelrUuQci pmPdcpPktLostUlMissingPdus2Qci pmPdcpPktLostUlQci pmPdcpPktLostUlRohcFail2Qci pmPdcpPktReceivedDlQci pmPdcpPktReceivedUlQci pmPdcpPktTransDlQci pmPdcpVolDlCmpHdrQci pmPdcpVolDlDrbFiltQci pmPdcpVolDlDrbLastTTIQci pmPdcpVolDlDrbQci pmPdcpVolDlDrbTransQci pmPdcpVolDlHdrQci pmPdcpVolUlCmpHdrQci pmPdcpVolUlDrbQci pmPdcpVolUlHdrQci pmPrbUsedDlDtchFirstTransQci pmPrbUsedMimoLayersDlDistr pmPrbUsedMimoLayersUlDistr pmPrbUsedUlDtchQci pmRlcDelayPktTransDlQci pmRlcDelayTimeDlQci pmRrcConnReestAttQci pmRrcConnReestSuccQci pmServiceTimeDrbQci pmSessionTimeDrbQci pmUeCtxtFetchSuccErabActQci pmZtemporary131Compr pmZtemporary132Compr pmZtemporary133Compr pmZtemporary134Compr pmZtemporary135Compr pmZtemporary158Compr pmZtemporary159Compr pmZtemporary16Qci pmZtemporary17Qci pmZtemporary18Qci pmZtemporary224Compr pmZtemporary225Compr pmZtemporary226Compr pmZtemporary230Compr pmZtemporary231Compr pmZtemporary232Compr pmZtemporary233Compr pmZtemporary59 pmZtemporary80 pmZtemporary81 pmZtemporary82 pmZtemporary83',-1);
foreach $i ($[ .. $#a) { $CompressedCounters{$a[$i]} = 1; }

# Table containing the list of PDF counters
my %isPdf;

# Parse input
COUNTERDATA: while (<>) {
	
	($ts,$mo,$counter,$value) = split ';';
	
	# Skip any lines that don't look like valid counters
	next if (! ($ts =~ /(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)/));
	
	($yy,$mm,$dd,$hh,$mi,$ss) = ($1,$2,$3,$4,$5,$6);
	
	# Figure out MO Aggregation
	if ($mofilter) {
		
		$mo =~ $mofilter_re or next; # Skip any rows not matched
                $mo_agg=$1;

		if (scalar @relfilter) {
		        for ($i=0; $i<=$#reldef_file; $i++) {
				#print ".. $i $mo_agg $relmo[$i]{$mo_agg}\n";
				if ($relmo[$i]{$mo_agg} =~ $relfilter_re[$i]) {
					if ($relmap_accum) {
					        $mo_agg=sprintf("%s (%s)",$mo_agg,$1);
					        $mo_agg =~ s/RncModule=/Mod=/;
					} else {
					        $mo_agg=$1;
					}
				} else {
					#warn "Warning: Object $mo_agg is not matched by rule $relfilter[$i], file $reldef_file[$i]\n";
					$reldef_filename = $reldef_file[$i];
					$reldef_filename =~ s/^.*\///;
					$warnlist{"Warning: Object $mo_agg is not matched by rule $relfilter[$i], file $reldef_filename\n"} = 1;
					if ($relmap_accum) {
					        $mo_agg=sprintf("%s (%s)",$mo_agg,"Unknown");
					        $mo_agg =~ s/RncModule=/Mod=/;
					} else {
					        $mo_agg="Unknown";
					}
					#next COUNTERDATA;
				}
			}
		}
		
	} else {
		$mo_agg = $mo;
	}
	
	# Define time level aggregation
	if ($hourly) {
		$tu="$yy-$mm-$dd $hh:00";
	} elsif ($daily) {
		$tu="$yy-$mm-$dd";
	} elsif ($all_time) {
		$tu="";
		$min_time = $ts if ($ts lt $min_time or (! $min_time));
		$max_time = $ts if ($ts gt $max_time or (! $max_time));
	} else {
		$tu="$yy-$mm-$dd $hh:$mi";
	}
	if ($fmt eq 'csv') {
		$csv_min_time = $ts if ($ts lt $csv_min_time or (! $csv_min_time));
		$csv_max_time = $ts if ($ts gt $csv_max_time or (! $csv_max_time));
	}
	
	# Maps are used so that we can step through all possible times, mo's and counters
	$ctr_map{$counter}=1;
	$mo_map{$mo_agg}=1;
	$ts_map{$tu}=1;
	
	# Check if we have an array value (consists of values separated by commas)
	# If it is an array, then the hash contains an array reference instead of
	# a scalar
	# Do not aggregate if the value contains non numeric characters (eg moRef, string, etc) or if the value is empty and we are not aggregating on time basis or ManagedElement level
	# Allow aggregation even if the counter name does not start with pm (used to be: || $counter !~ /^[pP]m/ )
	if ($value =~ /[A-Z]|[a-z]/ || ($value =~ /^[ \t]*$/ and not $all_time and not $hourly and not $daily and not $me_agg) ) {
		chomp $value;
		#replace the traffic descriptor with its peak cell rate
		if ($value =~ /,AtmTrafficDescriptor=[^P]+P([0-9]+).*$/) { $value = $1; }
		$data{$tu}{$mo_agg}{$counter}=$value;
		$sum{$counter}=$value;
		$sum_time{$tu}{$counter}=$value;
		$sum_mo{$mo_agg}{$counter}=$value;
	}
	elsif ($value =~ /,/ || (exists $isPdf{$counter}) || (exists $CompressedCounters{$counter})) {
		chomp $value;
		my @values=split ',',$value;
		$isPdf{$counter}=1;
		array_add(\$data{$tu}{$mo_agg}{$counter}, \@values, $counter);
		array_add(\$sum{$counter},\@values, $counter);
		array_add(\$sum_time{$tu}{$counter},\@values, $counter);
		array_add(\$sum_mo{$mo_agg}{$counter},\@values, $counter);
	} else {
		$data{$tu}{$mo_agg}{$counter}+=$value;
		$sum{$counter}+=$value;
		$sum_time{$tu}{$counter}+=$value;
		$sum_mo{$mo_agg}{$counter}+=$value;
	}
	
}

# If possible, reduce times. If all samples in the same day date is removed.
my ($one_day_only, $date_hdr, $extra_headline) = reduce_time_format;

# Parse Formulas
parse_formulas;

# Execute formulas
exec_formulas;

# Filter the output counters
if ($cntr_filter) {
	foreach $ctr (keys %ctr_map) {
		delete $ctr_map{$ctr} if ($ctr !~ $cntr_filter_re);
	}
}

# Check if MOs should be sorted numerically or not
my $numMo=1;
foreach $mo (sort keys %mo_map) {
	$numMo=0 if ($mo !~ /^[0-9.]+$/);
}

# MO List can be sorted Numerically, Alphabetically or by one counter value
# Here we create the subroutine that will be used to sort MOs
if ($sort_key =~ /^(\w+)([\-\+])*$/) {
	if ($2 eq '-') {
		#descending
		$sort_key1=$1;
		$sort_function = sub { 
			$data{$ts}{$b}{$sort_key1} <=> $data{$ts}{$a}{$sort_key1}
			or
			$numMo ? $a <=> $b : $a cmp $b;
		};
	} else {
		# ascending
		$sort_key1=$1;
		$sort_function = sub { 
			$data{$ts}{$a}{$sort_key1} <=> $data{$ts}{$b}{$sort_key1}
			or
			$numMo ? $a <=> $b : $a cmp $b;
		};
	}
} elsif ($numMo) {
	$sort_function = sub { $a <=> $b; };
} else {
	$sort_function = sub { $a cmp $b; };
}

# Output data
my $num_rows=1;
if ($cols eq 'ctr') {
	out_headrow($extra_headline, $date_hdr,"Object",sort keys %ctr_map);
	foreach $ts (sort keys %ts_map) {
		foreach $mo (sort $sort_function keys %mo_map) {
			next if ($filter_exp and not eval($filter_exp));
			@row = ($ts,$mo);
			foreach $ctr (sort keys %ctr_map) {
				push @row,$data{$ts}{$mo}{$ctr}; 
			}
			out_row(@row);
			last if ($top and $num_rows++ >= $top);
		}
	}
} elsif ($cols eq 'mo') {
	my @mo_list=(sort $sort_function keys %mo_map);
	out_headrow($extra_headline, $date_hdr,"Counter",@mo_list);
	foreach $ts (sort keys %ts_map) {
		foreach $ctr (sort keys %ctr_map) {
			@row = ($ts,$ctr);
			my $colcnt=0;
			foreach $mo (@mo_list) {
				# Only try to output if this MO instance
				# actually has this counter, otherwise
				# output a blank column
				if (exists $data{$ts}{$mo}{$ctr}) {
					push @row,$data{$ts}{$mo}{$ctr};
					$colcnt++;
				} else {
					push @row,undef;
				}
			}
			# Only need to output this row if there was some 
			# columns!
			if ($colcnt) {
				out_row(@row);
				last if ($top and $num_rows++ >= $top);
			}
		}
	}
} elsif ($cols eq 'time') {
	if ($fmt eq 'txt' and not $one_day_only and not $daily and not $all_time) {
		# In the case of a text output, we want to break it into days
		# to minimise wrapping
		my @times, $day, $time, $current_day="";
		my @timestamps = (sort keys %ts_map);
		foreach $ts (@timestamps) {
			($day,$time) = ($ts =~ /(\d\d\d\d-\d\d-\d\d) (\d\d:\d\d)/);
			$current_day = $day if (! $current_day);
			if (($day eq $current_day) and ($ts ne $timestamps[$#days])) {
				push @times, $time;
			} else {
				out_headrow("Date: $current_day\n", "Object","Counter",@times);
				foreach $mo (sort $sort_function keys %mo_map) {
					foreach $ctr (sort keys %ctr_map) {
						@row = ($mo,$ctr);
						my $colcnt=0;
						foreach $ts (@times) {
							if (exists $data{"$current_day $ts"}{$mo}{$ctr}) {
								push @row,$data{"$current_day $ts"}{$mo}{$ctr};
								$colcnt++;
							} else {
								push @row, undef;
							}
						}
						if ($colcnt) {
							out_row(@row);
							last if ($top and $num_rows++ >= $top);
						}
					}
				}
				$current_day=$day;
				@times=();
				push @times, $time;
			}
		}
		if (@times) {
			out_headrow("Date: $current_day\n", "Object","Counter",@times);
			foreach $mo (sort $sort_function keys %mo_map) {
				foreach $ctr (sort keys %ctr_map) {
					@row = ($mo,$ctr);
					my $colcnt=0;
					foreach $ts (@times) {
						if (exists $data{"$current_day $ts"}{$mo}{$ctr}) {
							push @row,$data{"$current_day $ts"}{$mo}{$ctr};
							$colcnt++;
						} else {
							push @row, undef;
						}
					}
					if ($colcnt) {
						out_row(@row);
						last if ($top and $num_rows++ >= $top);
					}
				}
			}
		}
	} else {
		out_headrow($extra_headline, "Object","Counter",sort keys %ts_map);
		foreach $mo (sort $sort_function keys %mo_map) {
			foreach $ctr (sort keys %ctr_map) {
				@row = ($mo,$ctr);
				my $colcnt=0;
				foreach $ts (sort keys %ts_map) {
					if (exists $data{$ts}{$mo}{$ctr}) {
						push @row,$data{$ts}{$mo}{$ctr};
						$colcnt++;
					} else {
						push @row,undef;
					}
				}
				if ($colcnt) {
					out_row(@row);
					last if ($top and $num_rows++ >= $top);
				}
			}
		}
	}
}

out_footer;
