#!/usr/bin/perl
#----------------------------------------------------------------------------------------------
#
# pmXtab: Cross-tabulate PM counters
#
# Copyright (c) 2005-2024 Ericsson Australia Pty Ltd
#
# This program may be used and/or copied only 
# with the prior written permission of Ericsson 
# Australia or in accordance with the terms and 
# conditions stipulated in the contract or agreement 
# under which the program has been supplied.
#----------------------------------------------------------------------------------------------
# v 1.0  03/08/2004 M. Harris    Initial Version, based on pmAggregate
# v 1.1  12/08/2004 M. Harris	Fix formatting for text output, add formulas
# v 1.2  30/08/2004 M. Harris    Correct error in formula calculation routine
#                                Add usage, add aggregation by related MO
#                                Fix formatting of floating point numbers
# v 1.3  22/11/2004 M. Harris    Add precision option
#                                Force formulas to evaluate in same order as file
#                                Change date/time column heading
# v 1.4  22/11/2004 M. Harris    Change behaivour so unmatched rows are discarded rather than die
# v 1.5  24/02/2005 M. Harris    Intelligent date formatting - if all on same day then remove date
#                                Format column width to be at least as wide as precision of numbers
#                                Sort Objects numerically if they are all numbers
# v 1.6  28/02/2005 M. Harris    Add default for precision
#                                Fix formatting errors; minimum width of Object column
#                                Right justify text and left justify numbers
#                                Use format_data for headings as well as data rows
#                                Add -pipe option
# v 1.7  28/02/2005 M. Harris    Fix case where -d option results in null times
# v 1.8  19/03/2005 M. Harris    Add aggregate sum functions
#                                Remove 'our' for backward compatibility with perl 5.0 (D. Smith)
#                                Enhance formatting for cols=time over multiple days
# v 1.9  23/03/2005 M. Harris    Add -a option to aggregate all time periods together
# v 1.10 06/04/2005 M. Harris    Only call tput if trunc used
#                                Right-align dates if they are column headings
#                                Change handling of -pipe; bidirectional pipe now used
#                                to ensure data ordering on STDOUT
# v 1.11 17/04/2005 M. Harris    Only try to reduce timestamps if not using csv output fmt
# v 1.12 21/06/2005 M. Harris    Add -filter, -sort and -top options
# v 1.13 12/12/2005 M. Harris    Add facility for multiple -r,-rdef options
#                                Add handling of array counter types
#                                Change all warning prints to 'warn' calls
#                                Only print warnings if formula error is not 'div by zero'
#                                Print warning & discard data of relation-map not matched
# v 1.14 16/03/2006 M. Harris    Change output formatting of integers from signed to unsigned
#                                Only output counters if they exist for a particular object
#                                Output headers for -a reports giving min & max time
# v 1.15 12/07/2006 M. Harris    Fix comment handling in formula files
# v 1.16 22/8/2006  M. Harris    Change formatting for integers to %ld (thanks esmidav)
# v 1.17 15/1/2007  M. Harris    Fix bug where last day is not displayed if -cols time
#                                is used on multiple day's worth of data.
# v 1.18 23/02/2007 T. Husz, F. Magnusson  Handling of non-pm attributes in ROP files
# v 1.19 23/03/2007 M. Aldrin	 Added an id tag to every row in the html table with the same name as cluster
# v 1.20 10/01/2008 F. Magnusson Remove duplicate warnings for unmatched rules in relation file.
# v 1.21 08/07/2008 M. Harris	 Added new subroutines for formula calculations (WeightedPercentile)
# v 1.22 05/09/2008 M. Harris	 Added new subroutines for formula calculations (MaxNonZeroIndex, MaxWeightedNonZeroIndex, MinNonZeroIndex, MinWeightedNonZeroIndex, MaximumWeightedValue)
# v 1.23 21/01/2009 M. Harris    Fix handling of large integer values, TR WRNae36124
# v 1.24 02/07/2009 M. Harris    New option -append used with mapfile (-rdef) to append the MO specified in option "-r" instead of substituting it
# v 1.25 05/09/2010 F. Magnusson Fix bug where empty counter values were printed as 0. 
# v 1.26 25/09/2010 F. Magnusson Changes in csv format printout, needed for pmrw. Remove duplicate warnings when fail to process a PDF formula.
# v 1.27 30/11/2010 F. Magnusson Modification of bug fix 1.25: only show empty counters when no aggregation. If -a/-d/-h is used then allow aggregation ==> empty counter values will be combined with non-empty counter values. Otherwise the whole aggregated value would have been empty.
# v 1.28 13/02/2011 F. Magnusson Fix bug introduced in 1.26 where formulas containing string attributes (e.g. userLabel) were being set to 0
#                                Missing semicolon separator between PDF counter values in csv format
# v 1.29 24/05/2012 M. Goransson Fix printout format of very small integers e.g 5.77290852864583e-05
# v 1.30 04/07/2013 F. Magnusson Subroutines Min, Max, Sum were not working
# v 1.31 16/09/2013 M. Harris    New subroutine decompressArray to decompress a compressed PDF counter, eg: De_pmPdcpVolDlDrbLastTTIQci = decompressArray(pmPdcpVolDlDrbLastTTIQci) , then PdcpVolDlDrbLastTTIQci_9 = @{De_pmPdcpVolDlDrbLastTTIQci}[9]
# v 1.32 23/11/2013 F. Magnusson Reduced verbosity of PDF formula exceptions
# v 1.33 03/01/2014 F. Magnusson Modification of bug fix 1.27: empty counters should be aggregated also in the case of MO aggregation at ManagedElement level eg "-m ManagedElement="
# v 1.34 16/02/2014 F. Magnusson Aggregate to object "Unknown" when object not found in ref file
# v 1.35 05/06/2014 F. Magnusson Aggregation on ManagedElement level was not working when some PDF counters contained empty values (,,,,,,,,,,)
# v 1.36 15/08/2014 F. Magnusson Allow aggregation even for counter names that don't start with "pm", as long as they contain a integer value
# v 1.37 28/09/2017 Z. He        Bug fix for aggregation of Qci counters
# v 1.38 27/10/2017 Z. He        Bug fix for aggregation of Qci counters with different array length between ROPs
# v 1.39 29/04/2018 H. Delpech   Add support for aggregation of compressed PDF counters
# v 1.40 29/04/2018 F. Magnusson Fixed column alignment for PDF counters
# v 1.41 01/05/2018 F. Magnusson Fixed aggregation for PDF counters when some ROPs contain non-array values
# v 1.42 05/08/2018 F. Magnusson In pmxy the header shall say "KPI" instead of "Counter". 
# v 1.43 30/08/2018 F. Magnusson Updated the list of compressed counters.
# v 1.44 02/12/2018 F. Magnusson Updated the list of compressed counters.
# v 1.45 10/12/2018 N. Eriksson  Fixed bug last rop sometimes printed on a different line than the others
# v 1.45 12/12/2018 F. Magnusson Added new function "exponent", eg to square a value use: exponent(value,2)
# v 1.46 13/03/2019 P. Ek        Cleanup of local/global variable declarations
# v 1.47 01/04/2019 P. Ek        Add option -q print all rop results are printed on a single row, even over different dates.
# v 1.48 01/05/2019 P. Ek        Fixed bug in option -q
# v 1.49 02/07/2019 F. Magnusson Updated list of compressed counters
# v 1.50 09/09/2019 F. Magnusson Added support functions ArrayAdd, DivideArrays, MultiplyArrays, ArrayMultiply, PdfNormal, SumArrayElements, SumArrays, WeightedStdDev
# v 1.51 12/09/2019 F. Magnusson Updated list of compressed counters
# v 1.52 08/12/2019 F. Magnusson Updated list of compressed counters
# v 1.53 10/02/2020 F. Magnusson Updated list of compressed counters
# v 1.54 01/04/2020 F. Magnusson Updated list of compressed counters
# v 1.55 03/07/2020 D. Vos       Bug fix when printing compressed counters aggregated on node level
# v 1.56 01/09/2020 F. Magnusson Added option -u to show the seconds
# v 1.57 07/09/2020 F. Magnusson Updated list of compressed counters
# v 1.58 20/12/2020 F. Magnusson Updated list of compressed counters
# v 1.59 16/03/2021 F. Magnusson Updated list of compressed counters
# v 1.60 05/06/2021 F. Magnusson Updated list of compressed counters
# v.1.61 09/06/2021 W. Axbrink   Added support for custom compressed counter settings with file insertion
# v.1.62 02/09/2021 P. Ek        Added "pval=1" option to filter formula on MO.
# v.1.63 07/09/2021 P. Ek        Added "pval=2" option to remove the MOC: part from the KPI name
# v.1.64 12/09/2021 F. Magnusson Added options "-xf" and "-if" to enter -x and -i from file
# v.1.65 16/05/2022 F. Magnusson Updates to the functions WeightedAverage, Percentile, WeightedPercentile and added new functions PadArray and MapArray
# v.1.66 28/06/2022 I. Einberg   Added "-avg" (average) option.
# v.1.67 04/07/2022 I. Einberg   Added support for variable NR_ROP in the formulas
# v.1.68 29/04/2023 F. Magnusson The Average function now ignores -2 values, needed for pmxet (parsing array of values from pget)
# v.1.69 30/05/2023 F. Magnusson The Min function now ignores -2 values, needed for pmxet (parsing array of values from pget)
# v.1.70 30/07/2023 F. Magnusson The Min function was reporting zero or empty when the array contained an empty value
# v 1.71 26/10/2023 P.Ek         Missing data was ignored when the -q option was used, leading to data being shifted for irregular data sets.
#----------------------------------------------------------------------------------------------

#use strict;
#use warnings;

use Getopt::Long;
use IPC::Open2;
use POSIX;
use List::Util qw/ min max sum /;

my $version = "v 1.71 2023-10-26";
sub printUsage ($);
printUsage "At least one option must be specified!" if $#ARGV==-1;

# Globals
my (%data, %ctr_map, %ts_map, %mo_map, %formulas, %warnlist, %warncntlist, %warncmplist, %mo_ts);
my ($daily, $hourly, $mofilter, $cols, $fmt, @formulas_txt, $cntr_filter, $relmap_accum, $reldef_filename, $me_agg, $pmxy, $q, $secondly, @mom_file, $pval, $avg);
my @maxcol;
my ($min_time, $max_time, $csv_min_time, $csv_max_time, $xfile, $ifile);
my ($all_time, $include_filter, $formula_file, @reldef_file, @relfilter, $truncate_lines, $help, $sort_key, $top, $filter_exp, $pipe_output, $max_ctr, $max_mo);
my ($df_numchars, $df_numdec, $outputfh, $pipe_pid, %sum_time, @formula_list, %CompressedCounters, @relmo, %sum_mo, %sum, %formula_pval_filter);

# Defaults
my $precision = "6.2";
$fmt = 'txt';
$cols= 'ctr';
$pval= 0;

GetOptions(
	"d"		 => \$daily,
	"h"		 => \$hourly,
	"a"      => \$all_time,
	"u"      => \$secondly,
	"append" => \$relmap_accum,
	"m=s"    => \$mofilter,
	"cols=s" => \$cols,
	"fmt=s"  => \$fmt,
	"f=s"    => \@formulas_txt,
	"i=s"    => \$include_filter,
	"fdef=s" => \$formula_file,
	"rdef=s" => \@reldef_file,
	"r=s"    => \@relfilter,
	"x=s"    => \$cntr_filter,
	"p=s"    => \$precision,
	"trunc"  => \$truncate_lines,
	"help"   => \$help,
	"sort=s" => \$sort_key,
	"top=s"	 => \$top,
	"filter=s" => \$filter_exp,
	"y"      => \$pmxy,
	"pipe=s" => \$pipe_output,
	"q"      => \$q,
	"mom=s"	 => \@mom_file,
	"pval=i" => \$pval,
	"xf=s"   => \$xfile,
	"if=s"   => \$ifile,
	"avg" 	 => \$avg
) or printUsage("Invalid Options");

my $tty_columns = `tput cols` if $truncate_lines;

# Also aggregate if the "avg" option i selected.
$all_time = 1 if $avg;

if ($mofilter =~ /ManagedElement=/) { $me_agg=1 ; }
my $Counter="Counter";
if ($pmxy) { $Counter="KPI" ; }

#----------------------------------------------------------------------------------------------
# Subroutines
#----------------------------------------------------------------------------------------------
# Print usage
#----------------------------------------------------------------------------------------------

sub decompressArray {
      my ($in) = @_;
      my @out;
      for (my $i=1; $i<@$in; $i+=2) {
            $out[$in->[$i]] = $in->[$i+1];
      }
      return \@out;
}

sub exponent {
	my $val = shift ;
	my $exp = shift ; 
	return $val ** $exp ;
}

sub printUsage ($) {
	my $msg=shift;
	
	if ($msg) {print STDERR "\n\033[1;31mpmXtab: $msg\033[0m\n";};
	print STDERR <<_EOTXT_;

pmXtab: Cross-tabulate PM counters
$version

usage: pmXtab -cols (ctr|mo|time) -fmt (csv|txt|html|htmltab) [-p <precision>]
              [-trunc] [-q] [-h] [-d] [-m <mo-filter>] [-f formula ...] 
	      [-fdef <formulafile>] [-rdef <relationfile>] [-append]
	      [-r <relation_filter>] [-i <counter_filter>] [-x <output_filter>]
	      [-sort <ctr/formula>] [-top <n>] [-pipe <shell_cmd>]
	      [-mom <file>|<dir>] [-pval <0|1|2>]
_EOTXT_
	die "\n" if ($msg);
}

sub printHelp {
	
	printUsage("");
	print STDERR <<_EOTXT_;
	
Aggregation by Time:
--------------------
-h : Aggregate hourly
-d : Aggregate Daily
-a : Aggregate all time periods into a single bucket

Aggregation by MO Filter
------------------------
-m <filter> : Aggregate by a filtered MO expression
              This expression must be a perl RE which has one grouped expression
              This expression will be used as the aggregation
              Example: UtranCell=(.+)$ : to aggregate on cell level
                       MeContext=(.+), : to aggregate on NE level

Aggregation by related MO
-------------------------
-rdef <file> : File that specifies the MOs related to this one
               (eg. relationships between cells & iublinks, for instance)

               The file should be in the format:
               <base-mo>:<related-mo> [<related-mo>...]

-r <filter>  : A filter to select the related MO to aggregate on

The way this function works is that first, the -m filter is used to
extract the MO that will be used to match against a relation string.
Then the -r filter is used to select the relation that will be used.

Example: Suppose the relation file contains the following

UtranCell=12345:IubLink=rbs6554 RncModule=1
UtranCell=12346:IubLink=rbs6554 RncModule=1
UtranCell=12347:IubLink=rbs6554 RncModule=1
UtranCell=12348:IubLink=rbs6123 RncModule=1

If you wanted to aggregate cell counters to the RBS level then you would use
-m "(UtranCell=\\w+)" -r "IubLink=(\\w+)"
Or to the module level:
-m "(UtranCell=\\w+)" -r "RncModule=(\\w+)"

If you just want to specify the IubLink or RncModule next to the UtranCell (instead 
of replacing it), use the -append option. Example:
-m "(UtranCell=\\w+)" -r "(RncModule=\\w+)" -append

Formulas
--------
-f <formula>
	Add a formula based on counter values that are being fed to pmXtab
	A formula should consist of a name, then an equals sign, then an expression
	The expression may include counter names, constants, operators
	and functions (all operators & functions allowed in perl may
	be used, these are similar to those in C).

	Example:
	RabEstSuccRate = 100 * pmTotNoRrcConnectReqSuccess / pmTotNoRrcConnectReq
	
	Multiple formulas may be added by specifying -f option more than once.

-i <pattern>
	Include formulas from a formula definition file if they match <pattern>. The 
	formula definition file to be used is specified via option -fdef. Note that
	if this option is not specified then ALL formulas in the definition file
	will be included

-fdef <file>
	Specify a config file that contains formulas to be used.

Filtering and Sorting
---------------------
-pipe <unix cmd>
	Specify a shell command. Output (excluding header and footer) will be piped
	through this command. Applies only to csv and txt output formats. Useful to
	(for example) sort output : eg. -pipe "sort -n +2" will sort by the first
	data column
	
-sort <counter>
	Specify a sort field. The sort field should be one of the counters
	or formula results. Within each time aggregate, the MOs will be
	output sorted according to the given field. If the last character
	is a '-' then sort is descending, otherwise it is ascending.

-top <number>
	Specify a maximum number of rows to output. When used with -sort this 
	allows for 'top 10' type reports.

-mom <file>|<dir>
	Specify a config file that contains which compressed counters to be used.

-pval <0|1|2> 
	0 or unspecified: legacy behaviour
	1 : it will assume that MO class prefix is given in the formula file and will filter out KPIs belonging to MO instances where the KPI does not apply
	2 : same as 1 but the MO class prefix will be removed from the KPI names

Output Formatting
-----------------
-x <pattern>
	Specify a filter to be used to filter counters before output. If specified
	all available counters (including the results of formulas) will be filtered
	using this expression before being output. Can be used to remove counters
	while leaving calculated values.
	       
-cols <ctr|mo|time>
	Specify column headings
	ctr : Counter names will be column headings
	mo  : Managed Objects will be column headings
	      (use with -m option)
	time: Times will be column headings

-fmt <txt|csv|html|htmltab>
	Specify output format
	txt	: Text only (default)
	csv	: Comma separated
	html	: HTML
	htmltab : HTML tables only (use for embedding into a web page)
	
-p <x.y>
	Specify precision for non-integer data
	x=total number of digits
	y=number of decimal places

-trunc
	Specify if lines should be truncated if they would wrap on the
	terminal (only for text format)	       

-q
	Print all rop periods as a single row of columns.

More information on pmTools can be found at http://newtran01.au.ao.ericsson.se/utranwiki/pmTools

_EOTXT_
	exit;
}

#----------------------------------------------------------------------------------------------
# Calculate column widths (only used for text based report)
#----------------------------------------------------------------------------------------------
sub col_widths {
	
	my ($ind1,$ind2,$ind3,$i);
	my $max_ts=0;
	
	# Start with the column headings
	# Note that subroutine should be called with @_ = headings for columns
	$i=0;
	$maxcol[$i++]=($df_numchars > length($_)) ? $df_numchars : length($_) foreach (@_);
	
	$max_ts = ($max_ts < length($_)) ? length($_) : $max_ts foreach (keys %ts_map);
	$max_ctr = ($max_ctr < length($_)) ? length($_) : $max_ctr foreach (keys %ctr_map);
	$max_mo = 6; # Min column size for MO column must be able to contain the word 'object'
	$max_mo = ($max_mo < length($_)) ? length($_) : $max_mo foreach (keys %mo_map);
	
	if ($cols eq 'ctr') {
		$maxcol[0]=$max_ts;
		$maxcol[1]=$max_mo;
	} elsif ($cols eq 'mo') {
		$maxcol[0]=$max_ts;
		$maxcol[1]=$max_ctr;
	} elsif ($cols eq 'time') {
		$maxcol[0]=$max_mo;
		$maxcol[1]=$max_ctr;
	}					 
	
	# Return is in global @maxcol
	#print "Col Width $_\n" foreach (@maxcol);

}

#----------------------------------------------------------------------------------------------
# Format data, used in text output only
#----------------------------------------------------------------------------------------------
sub format_data {

	my ($d,$w) = @_;
	my $joinedarray;

	# Fixme: this is very crude. Columns, etc will not line up!
	if (ref $d eq 'ARRAY') {
		$joinedarray = join(",", @{$d});
		return sprintf("%*s",$w,$joinedarray);
	}

	if ($d =~ /^[\+\-]?[\d\.]+(e\-\d+)?$/) {
	
		# Decimal Integer
		return sprintf "%${w}.0f",($d) if ($d ==int($d));
	
		# otherwise must be a floating Point Number
		return sprintf "%${w}${df_numdec}f",($d);
		
	}
	
	# Right Align times if in columns
	return sprintf "%${w}s",($d) if ($cols eq 'time' and $d =~ /^\d\d:\d\d$/);

	# Special Case
	return sprintf "%${w}s",($d) if ($d eq "N/A");
	
	# Otherwise assume a string
	return sprintf "%-${w}s",($d)
		
}
#----------------------------------------------------------------------------------------------
# Output a heading row
#----------------------------------------------------------------------------------------------
sub out_headrow {
	
	my $extra_headingline=shift;
	
	if ($fmt eq 'html') {
		print <<_EOF_;
<html>
	<head>
	<title>pmTools - pmXtab Output</title>
	<style TYPE="text/css">
        	<!--
			table {
				border-collapse: collapse;
				border: 1px solid black;
				padding: 0;
			}
			
			th {
				border: 1px solid black;
				font: 10pt Tahoma,Arial,Helvetica;
				font-weight: bold;
				background-color: #9999CC;
				margin: 0;
			}
			
			td {
				border: 1px solid black;
				font: 10pt Tahoma,Arial,Helvetica;
				color: black;
				background-color: #CCCCCC;
				padding: 2;
				margin: 0;
				text-align: center;
			}
			-->
			</style>
	</head>
	<body>
_EOF_
	} 
	my $wline;
	my $cmpline;
	my $cntline;
	foreach $wline (keys %warncntlist) { $cntline .= "$wline " }
	foreach $wline (keys %warncmplist) { $cmpline .= "$wline " }
	foreach $wline (keys %warnlist) 
	{ 
		if ($wline =~ /Cannot add array to non-array/) {$wline .= "$cntline\n";}
		if ($wline =~ /Cannot add compressed array to non-array/) {$wline .= "$cmpline\n";}
		warn $wline ;
	}
	if ($fmt eq 'txt') {
		col_widths(@_);
		print "\n$extra_headingline";
		my $i=0;
		my $hdr="";
		$hdr = join(" ",map(format_data($_, $maxcol[$i++]), @_));
		$hdr = substr($hdr, 0, $tty_columns) if $truncate_lines;
		print "$hdr\n";
	} elsif ($fmt eq 'csv') {
		print "$extra_headingline";
		print "$_; " foreach (@_);
		print "\n";
	} elsif ($fmt eq 'html' or $fmt eq 'htmltab') {
		print "$extra_headingline<br><table>\n<tr>";
		$_ && print "<th>$_</th>" foreach (@_);
		print "</tr>\n";
	}
}
#----------------------------------------------------------------------------------------------
# Output a data row
#----------------------------------------------------------------------------------------------
sub out_row {
	
	my ($d, $w);
	if ($fmt eq 'txt') {
		my $i=0;
		my $line="";
		$line = join(" ",map(format_data($_, $maxcol[$i++]), @_));
		$line = substr($line, 0, $tty_columns) if $truncate_lines;
		print $outputfh "$line\n";
	} elsif ($fmt eq 'csv') {
		my $line="";
		foreach $d (@_) {
			if (ref $d eq 'ARRAY') {
				$line .= join(',',@{$d});
				$line .= "; ";
			} else {
				if ($d =~ /^[\+\-]?[\d\.]+(e\-\d+)?$/) { 
					if ($d ==int($d)) { $line .= sprintf "%${w}.0f; ",($d); }
					else {$line .= sprintf "%${w}${df_numdec}f; ",($d); }
				}
				else { $line .= "$d; "; }
			}
		}
		print $outputfh "$line\n";
	} elsif ($fmt eq 'html' or $fmt eq 'htmltab') {
		# if -a was specified then no date column
		shift if ($all_time);
		print "<tr id=$_[0]>";
		print "<td>".format_data($_)."</td>" foreach (@_);
		print "</tr>\n";
	}
	
	# If we are piping through a filter: see if there is anything here to read
	# NB. FILTER_IN is set to nonblocking at this time
	if ($pipe_output) {
		my $l=<FILTER_IN>;
		print $l if $l;
	}	 
	
}
#----------------------------------------------------------------------------------------------
# Output footer
#----------------------------------------------------------------------------------------------
sub out_footer {
	
	my $l="";
	my $finished=0;
	
	# Now we need to see if there is any more data left in the filter pipe
	# Note that for tools like sort, they read all data then output only
	# after their input is closed..
	# First reset nonblocking mode of FH, then close filter pipe output FH,
	# then read all remaining lines in the filter pipe input FH
	if ($pipe_output) {
		fcntl(\*FILTER_IN, F_SETFL(), 0);
		close FILTER_OUT;
		while (<FILTER_IN>) {
			print $_;
		}
		close FILTER_IN;
		waitpid $pipe_pid,0;
	}
	
	if ($fmt eq 'htmltab') {
		print "</table>\n";
	} elsif ($fmt eq 'html') {
		print "</table>\n";
		print "</body></html>\n";	
	} elsif ($fmt eq 'txt') {
		print "\n";
	}
}
#----------------------------------------------------------------------------------------------
# Reduce Time Formats
#----------------------------------------------------------------------------------------------
sub reduce_time_format {
	
	my ($ts, $hm, $day);
	$day="";
	
	# If using csv format then we don't want to do it
	# (assume some downstream tool will be reading, or maybe
	# import to excel)
	if ($fmt eq 'csv') {
		if ($all_time) {
			return (0, "","Start Time: $min_time End Time: $max_time Granularity: whole period.\n");
		}
		elsif ($hourly) {
			return (0, "Date/Time","Start Time: $csv_min_time End Time: $csv_max_time Granularity: hour.\n") ;
		}
		elsif ($daily) {
			return (0, "Date/Time","Start Time: $csv_min_time End Time: $csv_max_time Granularity: day.\n") ;
		}
		else {
			return (0, "Date/Time","Start Time: $csv_min_time End Time: $csv_max_time Granularity: rop.\n") ;
		}
	}
	
	# If time to be aggregated into a bucket
	if ($all_time) {
		return (0, "","Start Time: $min_time End Time: $max_time\n");
	}
	
	# If daily aggregation then only date is given
	if ($daily) {
		return (0, "Date","");
	}
	
	# Determine if all samples in the same day
	foreach $ts (sort keys %data) {
		if (! $day) {
			$day=substr($ts,0,10);
		} else {
			return (0, "Date       Time","") if ($day ne substr($ts,0,10));
		}
	}
	
	# If we reach this point then the date is the same for all entries
	undef %ts_map;
	foreach $ts (sort keys %data) {
		$hm = substr($ts,11);
		$data{$hm}=$data{$ts};
		$sum_time{$hm}=$sum_time{$ts};
		$ts_map{$hm}=1;
		delete $data{$ts};
		delete $sum_time{$ts};
	}
	return(1, "Time", "Date: $day\n");

}
#----------------------------------------------------------------------------------------------
# Parse Formulas
#----------------------------------------------------------------------------------------------
# We don't really parse the formulas. We simply convert them into a perl-like syntax so that
# we can 'eval' them later. The following steps are taken:
# 1. Separate formula name (bit before =) from formula itself
# 2. Replace all instances of counterName in the formula to the form $ctrs->{counterName}
# 3. Replace all instances of sum_counterName with $sum{counterName}. Same with sum_mo and sum_time
# 4. Stash each formula in the hash %formulas
sub parse_formulas {
	
	my ($f, $ctr, $name, $printname, @fparts);

	foreach $f (@formulas_txt) {
		
		printUsage "Formula must start with 'name='" if (! ($f =~ /^(@*\w+:?\w*)\s*=\s*(.+)$/));
		$name = $1;
		$f = $2;
				
		# Each counter referenced in the formula must be replaced by
		# $ctrs->{name}
		# Also, sum_xxxxx must be replaced with $sum{name}
		foreach $ctr (keys %ctr_map) {

			# If using the NR_ROP variable.
			if ($f =~ /\b$ctr\b(?=.*\bNR_ROP\b)/){
				if ($all_time){
					# Number of different ROPs for the counter's ($ctr) corresponding MO.
					my $count = 0;
					# Loop through all MOs to find the one corresponding to this counter.
					foreach my $mo (keys %mo_map){
							# Verify that $ctr is a counter of $mo 
							if (exists $sum_mo{$mo}{$ctr}){
								# Count the number of different ROPs for this MO.
								$count = scalar keys %{$mo_ts{$mo}};
								last;
							}	
						}
					
					# Replace NR_ROP with number of ROPs for the counters' corresponding MO.
					$f =~ s/NR_ROP/$count/;
				} else{
					# If not aggregating values, replace NR_ROP with 1.
					$f =~ s/NR_ROP/1/;
				}
					
			}
			$f =~ s/\b$ctr\b/\$ctrs->{$ctr}/g;
			$f =~ s/\bsum_$ctr\b/\$sum{$ctr}/g;
			$f =~ s/\bsum_mo_$ctr\b/\$tsum_mo->{$ctr}/g;
			$f =~ s/\bsum_time_$ctr\b/\$tsum_time->{$ctr}/g;
			
		}
		
		$formulas{$name}=$f;
		push @formula_list, $name;	
		$ctr_map{$name}=1;

		if ($pval == 2){
			#Store the value of counters with format MO:Formula, as Formula.
			@fparts = split(/:/, $name);	
			if ((scalar @fparts) == 2) {
				$formula_pval_filter{$name}=$fparts[1];
			}
		}
		
		#print "Compiled Formula: $name = $f\n";
		
	}
	
	if ($filter_exp) {
		foreach $ctr (keys %ctr_map) {
			$filter_exp =~ s/\b$ctr\b/\$data{\$ts}{\$mo}{$ctr}/g;
		}
		#print "Compiled Filter_exp: $filter_exp\n";
	}
	
}

#----------------------------------------------------------------------------------------------
# Execute Formulas
#----------------------------------------------------------------------------------------------
sub exec_formulas {

	my ($ts, $mo, $ctr, $fname,$r,$printname);
	my ($ctrs, $tsum_mo, $tsum_time);
	my (@fparts, @moparts, $last, @mops);
	
	foreach $ts (keys %data) {
		$tsum_time = \%{$sum_time{$ts}};
		foreach $mo (keys %{$data{$ts}}) {
			$tsum_mo = \%{$sum_mo{$mo}};
			$ctrs = \%{$data{$ts}{$mo}};
			foreach $fname (@formula_list) {
				#print "FNAME=$fname, MO=$mo\n";
				$printname=$fname;
				#If pval > 0, filter all formulas, where the format MO:Formula belongs to wrong MO.
				if ($pval > 0)
				{
					@fparts = split(/:/, $fname);
					@mops = split(/,/, $mo);
					@moparts = split(/=/, $mops[-1]);
					next unless (scalar @fparts == 2 && scalar @moparts == 2 && $fparts[0] eq $moparts[0]);
					#next if (scalar @fparts == 2 && scalar @moparts == 2 && $fparts[0] ne $moparts[0]);
					if ($pval == 2){
						#refer to the result value as Formula, instead of MO:Formula.
						$printname = @fparts[1] if (scalar @fparts == 2);
					}
				}

				# Formula names that start with @ indicate that
				# we want an array as a result
				if ($fname =~ /^@/) {
					@{$data{$ts}{$mo}{$printname}} = eval $formulas{$fname};
					if ($@ and $@ !~ /division by zero/) { $warnlist{"Error evaluating formula $fname"} = 1; } # = $formulas{$fname}:\n$@\n
				} else {
					$r = eval $formulas{$fname};

					if ($@ and $@ !~ /division by zero/) { $warnlist{"Error evaluating formula $fname"} = 1; } # = $formulas{$fname}:\n$@\n
					$data{$ts}{$mo}{$printname} = $@ ? "N/A":$r;
				}
			}
			
		}
	
	}
		
}
#----------------------------------------------------------------------------------------------
# Sum of array elements : op1=op1+op2
# Sum of compressed array elements : op1=op1+op2 handling of compressed arrays like 2,1,18,5,602
#----------------------------------------------------------------------------------------------
sub array_add {
	my $op1=shift;
	my $op2=shift;
	my $op3=shift;
	
	my $i;
	my $j;
	my $k;
	my $n=0;
	my $p=0;
	my $z=0;

	# Sum of array elements : op1=op1+op2
	if (!(exists $CompressedCounters{$op3})){
		if ((defined $$op1)&&(!(ref $$op1 eq 'ARRAY'))) { undef $$op1 ; }
		if (defined $$op1) {
			if (ref $$op1 eq 'ARRAY'|| ref $op2 eq 'ARRAY') {
				if (ref $$op1 eq 'ARRAY') { $n=$#{$$op1} ; }
				if (ref $op2 eq 'ARRAY') { $p=$#{$op2} ; }
				$z=$n;
				if ($p > $n) { $z=$p ;}
				for ($i=0; $i<=$z; $i++) {
					$$op1->[$i]+=$op2->[$i];
				}
			} else {
				# Trying to add an array to a non-array: abort!
				$warnlist{"Cannot add array to non-array: "} = 1;
				$warncntlist{"$op3"} = 1;
			}
		} else {
			if (ref $op2 eq 'ARRAY') {
				@{$$op1}=@{$op2};
			} else {
				#OP1 is created with a zero value of OP2
				$$op1->[0]=0;
			}
		}
		return
	}
	#Sum of compressed array elements 
	if (defined $$op1) {
		if ((ref $$op1 eq 'ARRAY') && (ref $op2 eq 'ARRAY')) {
			#print "Array Case\n";
			#check if QCI index are same before adding up
			#in case two ROP counters with different length like 3,1,1,5,20,9,22 and 2,5,22,9,18
			if ($$op1->[0] != 0) {
				for ($i=2; $i<=$#{$op2}; $i+=2) {
					for ($j=2;$j<=$#{$$op1}; $j+=2) { 
						if ( $$op1->[$j-1] == $op2->[$i-1] ) {
						#Insert OP1/OP2 QCI value matching
							$$op1->[$j]+=$op2->[$i];
							$j=$#{$$op1}+1;
						}
						elsif ( $$op1->[$j-1] > $op2->[$i-1] ) {
						#OP2 QCI value to be inserted in OP1
							for ($k=$#{$$op1}; $k>=$j; $k-=2) {
								$$op1->[$k+1]=$$op1->[$k-1];
								$$op1->[$k+2]=$$op1->[$k];
							}						
							$$op1->[$j]=$op2->[$i];
							$$op1->[$j-1]=$op2->[$i-1];
							$$op1->[0]++;
							$j=$#{$$op1}+1;
						}
						elsif (( $$op1->[$j-1] < $op2->[$i-1] ) && ( $j == $#{$$op1})) {
							#Insert OP2 QCI at the end OP1 QCI 
							$$op1->[$#{$$op1}+1]=$op2->[$i-1];
							$$op1->[$#{$$op1}+1]=$op2->[$i];
							$$op1->[0]++;
							$j=$#{$$op1}+1;
						}
					}
				}
			} else {
				# If OP1 had zero value only, it copies non zero OP2
				@{$$op1}=@{$op2};
			}
		} else {
			#print "Not Array Case\n";
			# Trying to add an array to a non-array: abort!
			$warnlist{"Cannot add compressed array to non-array: "} = 1;
			$warncmplist{"$op3"} = 1;
			return;
		}
	} else {
		if (ref $op2 eq 'ARRAY') {
			@{$$op1}=@{$op2};
		} else {
			#OP1 is created with a zero value of OP2
			$$op1->[0]=0;
		}
	}
}

#----------------------------------------------------------------------------------------------
# Formula support functions
#----------------------------------------------------------------------------------------------
# Average(\@array) : return average of the array elements
sub Average {
	my $d=shift; # Ref to array to be averaged
	my $sum=0;
     my $count=0;
	die "Non-array supplied to Average()" unless ref $d eq 'ARRAY'; 
	for (@{$d}) {
	   #ignore -2 values which may appear in pget when the rop is not complete, e.g. the counter EnergyMeter::pmVoltage
        if($_ eq -2){ next;}
        $sum+=$_ ;
        $count++;
     }
	return $count ? $sum/$count : 0;
}

# Count(\@array) : return number of elements in array
sub Count {
	my $d=shift;
	return scalar @{$d};
}

##-----------------------------------------------------------------------------
# WeightedAverage($array, $weightingarray, $bin)
# Return weighted average. Each element of the array is multiplied by the 
# corresponding element of the weighting array, then total is divided by the
# sum of the elements in the array
# - $array           Ref to array (these are numbers of samples)
# - $weightingarray  Ref to array of weightings (these are kbps, pct, etc)
# - $bin             If set to 'bin', then average in bin will be returned too
# RETURNS: $weightedavg
# - $weightedavg        Weighted Average of $array: if $bin => [in bins, in units]
#------------------------------------------------------------------------------
sub WeightedAverage {
        my $d = shift;
        my $w = shift;
        my $b = shift;
        
        # counters may be undef for certain ROP's, that is not a fatal error
        return undef unless defined $d;
        
        die "Non-Array supplied as first parameter to WeightedAverage()\n" unless ref $d eq 'ARRAY';
        
        if (defined $w) {
                die "Non-Array supplied as second parameter to WeightedAverage()\n" unless ref $w eq 'ARRAY';
        } else {
                $w = [];
                push @$w, $_ for 0 .. $#{$d};
        }
        
        if ($#{$d} > $#{$w}) {
                die 'Counter array ('.($#{$d}+1).') larger than weighting array ('.($#{$w}+1).") in WeightedAverage()\n";
        }
        
        if ($#{$d} < $#{$w}) {
                my $toAdd = $#{$w} - $#{$d};
                my @parray = @$d;
                push @parray, (0) x $toAdd;
                $d = \@parray;
        }
        
        my $count = 0.0;
        my $wsum  = 0.0;
        for (my $i=0; $i<=$#{$d}; $i++) {
                $count +=  $d->[$i];
                $wsum  += ($d->[$i] * $w->[$i]);
        }
        
        if (defined $b && $b eq 'bin') {
                my $bwsum = 0.0;
                for (my $i=0; $i<=$#{$d}; $i++) {
                        $bwsum += ($d->[$i] * $i);
                }
                return $count ? [$bwsum/$count, $wsum/$count] : [0.0,0.0];
        }
        
        return $count ? $wsum/$count : 0.0;
}

##-----------------------------------------------------------------------------
# log10($x)
# Calculate the base 10 log of $x. Useful for mathematics involving dB.
# - $x		Input value
# RETURNS: $result
# - $result	Base 10 log of $x
#------------------------------------------------------------------------------
sub log10 {
	# using basic algebraic principle: log_n(x) = log_e(x) / log_e(n)
	my $x=shift;
	if (($x+0) == 0) { return "N/A"; }
	return log($x)/2.30258509299405; # denom is log(10). 
}
##-----------------------------------------------------------------------------
# Sum($array)
# Return sum of elements in array
# - $array	Ref to array
# RETURNS: $sum
# - $sum	Sum of elements
#------------------------------------------------------------------------------
sub Sum {
	return sum (@{$_[0]});
}
##-----------------------------------------------------------------------------
# Max($array)
# Return max of elements in array
# - $array Ref to array
# RETURNS: $max
# - $max Maximum Array Value
#------------------------------------------------------------------------------
sub Max {
	return max(@{$_[0]});
}
##-----------------------------------------------------------------------------
# Min($array)
# Return min of elements in array
# - $array Ref to array
# RETURNS: $min
# - $min Minimum Array Value
#------------------------------------------------------------------------------
sub Min {
	#return min(@{$_[0]});
	my $d=shift;
	my @a=();
	for (@{$d}) {
		#ignore -2 values which may appear in pget when the rop is not complete, e.g. the counter EnergyMeter::pmVoltage
		if($_ eq -2){ next;}
		#ignore empty values
		if($_ eq ''){ next;}
		push (@a, $_);
	}
	return min(@a);
}
##-----------------------------------------------------------------------------
# Percentile($percentile, $array)
# Return the supplied percentile of the given array.
# - $percentile  Percentile to calculate
# - $array       Ref to array
# RETURNS: $weightedavg
# - $perc        Weighted Average of $array
#------------------------------------------------------------------------------
sub Percentile {
        my ($percentile,$arr) = @_;

        die "Non-Array supplied as first parameter to Percentile()\n" unless ref $arr eq 'ARRAY';

        #logDebug5("Input array is as follows: @{$arr}") if DEBUG5;

        # copy array so we can modify it
        my @parray = @$arr;

                # percent
        $percentile /= 100;

        # sort
        @parray = sort {$a <=> $b} @parray;

                # Calculate the n
                my $n =  $percentile * (scalar(@parray) - 1) + 1;

        # case where $n = 1
        if ($n == 1) {
                return $parray[0];
        # case where n = number of number element.
        } elsif ($n == scalar(@parray)) {
                return $parray[$n-1];
        }

                # slit n into its integer componant k and decimal component d
                my $k = int ($n);

                #decimal componant.
                my $d = $n - $k;

                # correction by -1 because the array starts at zero.
                my $kthElement = $parray[$k-1];

                return  $kthElement + $d * ($parray[$k] - $kthElement );
}

##-----------------------------------------------------------------------------
# WeightedPercentile($percentile, $array, $weightingarray, $interpolation, $start)
# Return weighted percentile. Each element of the array is multiplied by the
# corresponding element of the weighting array, the resulting array is then
# used to calculate the given percentile.
# - $percentile           The percentile to calculate
# - $array                Ref to array
# - $weightingarray       Ref to array of weightings
# - $interpolation        Interpolation (LINEAR)
# - $start                Bin to start calculation at (0 = first)
# RETURNS: $weightedpercentile
# - $weightedpercentile   Weighted percentile of $array
#------------------------------------------------------------------------------
sub WeightedPercentile {
        my $p=shift;
        my $d=shift;
        my $w=shift;
        my $int=shift;
        my $start=shift;
        
        # counters may be undef for certain ROP's, that is not a fatal error
        return undef unless defined $d;
        
        die "Non-Array supplied as first parameter to WeightedPercentile()\n"  unless ref $d eq 'ARRAY';
        die "Non-Array supplied as second parameter to WeightedPercentile()\n" unless ref $w eq 'ARRAY';

        if ($#{$d} > $#{$w}) {
                die "Counter array (".($#{$d}+1).") larger than weighting array (".($#{$w}+1).") in WeightedPercentile()\n";
        }
        if ($#{$d} < $#{$w}) {
                my $toAdd = $#{$w} - $#{$d};
                my @parray = @$d;
                push @parray, (0) x $toAdd;
                $d = \@parray;
        }
        
        if ($start) { 
                # start calculation at indicated bin
                # shorten the provided arrays
                my @parray = @$d;
                splice(@parray, 0, $start);
                $d = \@parray;
                
                my @parray2 = @$w;
                splice(@parray2, 0, $start);
                $w = \@parray2;
        }
        
        # make p a decimal
        $p /= 100;
        die "Supplied percentile is greater than 100\n" unless $p <= 1.0;
        
        # sum all the elements of the weighted array to get N
        my $N = 0;
        $N += $_ for @$d;
        my $n = $p*($N - 1) + 1;
        
        # create a array of [data, weight] pairings
        my @pairs;
        for (my $i = 0; $i < @$d; $i++) {
                push @pairs, [ $d->[$i], $w->[$i] ];
        }
        @pairs = sort {$a->[1] <=> $b->[1]} @pairs;
        
        if (! $int or $int =~ /^standard$/i) {
                # case where $n = 1 (return first weight)
                if ($n == 1) {
                        return $pairs[0]->[1];
                } # case where n = number of number element (return last weight)
                elsif ($n == $N) {
                        return $pairs[$#pairs]->[1];
                }
                
                # split n into its integer componant k and decimal component d
                my $k = int ($n);
                my $dec = $n - $k;
                
                # get v(k) and v(k+1)
                my ($vk, $vkplus1);
                
                # current counter
                my ($position, $i) = (0,0);
                while ($position < $k) {
                        # if this [i] pair does not reach k, move on to the next pair
                        if (($position + $pairs[$i]->[0]) < $k) {
                                # does no reach k, increment position and pair index
                                $position += $pairs[$i]->[0];
                                $i++;
                        } else {
                                # this pair is the right pair, get the first weight
                                if (!defined $vk) {
                                        # we first define v(k), then we do another loop setting k = k+1
                                        $vk = $pairs[$i]->[1];
                                        $k++;
                                } else {
                                        $vkplus1 = $pairs[$i]->[1];
                                        $position = $k;
                                }
                        }
                }
                
                # v(k) + d*(v(k+1) - v(k))
                return ($vk + $dec*($vkplus1 - $vk));
                
        } elsif ($int =~ /^linear$/i) {
                # calculate the integral of the array in range [0,1] for liear interpoloation
                my @wp = map { $_->[0] } @pairs;
                for (my $i = 0; $i < @wp; $i++) {
                        $wp[$i] += $wp[$i-1] if $i > 0;
                }
                
                @wp = map { $_/$N } @wp;
                
                # if we are before or on the first element, return it
                if ($p <= $wp[0]) {
                        return ( ( $pairs[0]->[1]) / ($wp[0]) * ($p));
                }
                
                # get the position of the lower bound
                my $i = 0;
                while ( ($wp[$i+1]) < $p ) {
                        $i++;
                }
                
                # we hit the value directly, return it
                if (($wp[$i+1]) == $p) {
                        return $pairs[$i+1]->[1];
                } else {
                        # it must be greater than p so p is in between wp[i] and wp[i+1] ...
                        # v = [v(k+1) - v(k)] / [p(k+1) - p(k)]  *[(p-p(k)]
                        
                        return ( ( $pairs[$i+1]->[1] - $pairs[$i]->[1] ) / ($wp[$i+1] - $wp[$i]) * ($p-$wp[$i]) + $pairs[$i]->[1]);
                }
        }
        
        return 0;
}
##-----------------------------------------------------------------------------
# MaxNonZeroIndex($array)
# Return the highest index with a non-zero value
# - $array Ref to array
# RETURNS: $max_index
# - $max_index The highest index with non-zero value.
#------------------------------------------------------------------------------
sub MaxNonZeroIndex {
  my $d=shift;

  die "Non-Array supplied as first parameter to MaxNonZeroIndex()" unless ref $d eq 'ARRAY';

  my $max_index;
  my $i = 0;
  for(;$i<$#{$d};$i++) {
    $max_index = $i if($d->[$i] != 0);
  }

  return $max_index;
}

##-----------------------------------------------------------------------------
# MaxWeightedNonZeroIndex($array,$weightingarray)
# Return the weighted value of the highest index with a non-zero value in counter array
# - $array Ref to array
# RETURNS: $max_index_value
# - $max_index_value The value of the weighting array at the highest index with non-zero value.
#------------------------------------------------------------------------------
sub MaxWeightedNonZeroIndex {
        my $d=shift;
        my $w=shift;

        die "Non-Array supplied as first parameter to MaxWeightedNonZeroIndex()" unless ref $d eq 'ARRAY';
        die "Non-Array supplied as second parameter to MaxWeightedNonZeroIndex()" unless ref $w eq 'ARRAY';
  die "Counter Array and Weighting array of different sizes in MaxWeightedNonZeroIndex()" unless $#{$d} == $#{$w};

        my $max_index;
        my $i = 0;
        for(;$i<$#{$d};$i++) {
                $max_index = $i if($d->[$i] != 0);
        }

        return $w->[$max_index];
}

##-----------------------------------------------------------------------------
# MinNonZeroIndex($array)
# Return the lowest index with a non-zero value
# - $array Ref to array
# RETURNS: $min_index
# - $min_index The highest index with non-zero value.
#------------------------------------------------------------------------------
sub MinNonZeroIndex {
        my $d=shift;

        die "Non-Array supplied as first parameter to MinNonZeroIndex()" unless ref $d eq 'ARRAY';

        for(my $i=0;$i<$#{$d};$i++) {
                return $i if($d->[$i] != 0);
        }
}

##-----------------------------------------------------------------------------
# MinWeightedNonZeroIndex($array,$weightingarray)
# Return the weighted value of the lowest index with a non-zero value in counter array
# - $array Ref to array
# RETURNS: $min_index_value
# - $min_index_value The value of the weighting array at the lowest index with non-zero value.
#------------------------------------------------------------------------------
sub MinWeightedNonZeroIndex {
        my $d=shift;
        my $w=shift;

        die "Non-Array supplied as first parameter to MinWeightedNonZeroIndex()" unless ref $d eq 'ARRAY';
        die "Non-Array supplied as second parameter to MinWeightedNonZeroIndex()" unless ref $w eq 'ARRAY';
        die "Counter Array and Weighting array of different sizes in MinWeightedNonZeroIndex()" unless $#{$d} == $#{$w};

        for(my $i=0;$i<$#{$d};$i++) {
                return $w->[$i] if($d->[$i] != 0);
        }
}

##-----------------------------------------------------------------------------
# MaximumWeightedValue($array, $weightingarray)
# Return the value within the weighted array which occurs the most in the counter array
# - $array              Ref to array
# - $weightingarray     Ref to array of weightings
# RETURNS: $maxWeightingValue
# - $maxWeightingValue        Max Weighting value
#------------------------------------------------------------------------------
sub MaximumWeightedValue {
        my $d=shift;
        my $w=shift;

        die "Non-Array supplied as first parameter to MaximumWeightedValue()" unless ref $d eq 'ARRAY';
        die "Non-Array supplied as second parameter to MaximumWeightedValue()" unless ref $w eq 'ARRAY';
  die "Counter Array and Weighting array of different sizes in MaximumWeightedValue()" unless $#{$d} == $#{$w};

  my ($max,$max_pos,$i);
  for($i = 0;$i<$#{$d};$i++) {
    if(!defined $max || $d->[$i] > $max) {
      $max = $d->[$i];
      $max_pos = $i;
    }
  }

  return $w->[$max_pos];
}

##-----------------------------------------------------------------------------
# ArrayAdd($offset,$array)
# Return an array where each element has been offset by the supplied value
# - $offset 
# - $array Ref to array
# RETURNS: $newarray
# - $newarray New array with the given offsets
#------------------------------------------------------------------------------
sub ArrayAdd {
	my $offset=shift;
	my $array=shift;
	my $i;
	my $newarray = ();
	
	die "Non-Array supplied as second parameter to ArrayAdd()\n" unless ref $array eq 'ARRAY';

	for (my $i=0; $i<=$#{$array}; $i++) {
		$newarray->[$i] = $offset + $array->[$i];
	}
	
	return $newarray;
}

##-----------------------------------------------------------------------------
# DivideArrays($numArray,$denArray)
# Return the numerator array divided by the denominator array, one element at a time
# - $numArray  Numerator PDF array
# - $denArray  Denominator PDF array
# RETURNS: $divArray
# - $divArray		Ref to divided PDF array
#------------------------------------------------------------------------------
sub DivideArrays {
	my $numArray = shift;
	my $denArray = shift;

	unless (ref $numArray eq 'ARRAY') {
		die "Non-array supplied as 1st parameter to DivideArrays()\n";
	}

	unless (ref $denArray eq 'ARRAY') {
		die "Non-array supplied as 2nd parameter to DivideArrays()\n";
	}

	unless ($#{$numArray} == $#{$denArray}) {
		die "Numerator array and Denominator array of different sizes in DivideArrays()\n";
	}

	return [ map $denArray->[$_]!=0 ? ($numArray->[$_] / $denArray->[$_]) : '!DIV0!', (0..$#{$numArray}) ];
}
##-----------------------------------------------------------------------------
# MultiplyArrays($array1[,$array2,...])
# Return an array which is the indexed multiply of each supplied array.  ie
# all elements at index 0 are summed for each array.
# - $array1 Ref to array
# - $array2 Ref to array
# ....
# RETURNS: $newarray
# - $newarray New array 
#------------------------------------------------------------------------------
sub MultiplyArrays {
	my (@arrays) = @_;
	
	die "Two or more array counters are required for MultiplyArrays()\n" unless @arrays>1;
	my $i=0;
	++$i && ref $_ ne 'ARRAY' && die "MultiplyArrays: Non-array supplied as argument $i\n" foreach @arrays;
	
	my $minsize = min(map scalar @$_, @arrays);
	die "Empty array found in MultiplyArrays()\n" if $minsize == 0;

	my @out = (1) x $minsize;
	
	foreach my $ary (@arrays) {
		$out[$_-1] *= $ary->[$_-1] foreach (1..$minsize);
	}
	
	return \@out;
}
##-----------------------------------------------------------------------------
# ArrayMultiply($coefficient,$array)
# Return an array where each element has been multiplied by the supplied coefficient
# - $coefficient 
# - $array Ref to array
# RETURNS: $newarray
# - $newarray New array with the given offsets
#------------------------------------------------------------------------------
sub ArrayMultiply {
	my $coeff=shift;
	my $array=shift;
	my $i;
	my $newarray = ();
	
	die "Non-Array supplied as second parameter to ArrayMultiply()\n" unless ref $array eq 'ARRAY';

	for (my $i=0; $i<=$#{$array}; $i++) {
		$newarray->[$i] = $coeff*$array->[$i];
	}
	
	return $newarray;
}
##-----------------------------------------------------------------------------
# PadArray($array, $range[, $value])
# Pads an array to the given size, using the given value.
# - $array      Ref to array
# - $range      Either the total number of elements the array should be, or an array
#           containing a range of elements to pad with 0's (eg. [ 5..9 ])
# - $value      Value to pad with (default to 0)
# RETURNS: $newarray
# - $newarray Padded array 
#------------------------------------------------------------------------------
sub PadArray {
        my ($array, $range, $value) = @_;
        $value //= 0;
        $range = [ 0..$range-1 ] unless ref $range;
        my @ary;
        $ary[$_] = $array->[$_]//$value foreach @$range;
        return \@ary;
}

##-----------------------------------------------------------------------------
# MapArray($sub, $array[, $array2...][, $indexes])
# Apply an operation to each element of one or more arrays, returning an array
# containing the results of each operation. 
#
# The operation is supplied as a sub reference which will be called with the 
# corresponding element from each of the input arrays, followed by the index number. 
# 
# If the arrays passed in are different sizes then the input will 
# be padded with 'undef' values.
# 
# - $sub        Subroutine which will be called for each of the sets of elements
# - @array      Ref to one or more arrays
# - $indexes    Optional CSV of indexes, such as '-3,5,7-8,10-'
# RETURNS: $result
# - $result     Output array or a scalar if a single bin has been targetted
#------------------------------------------------------------------------------
sub MapArray (&@) {
        my ($sub, @array) = @_;
        
        # make a copy as we're about to modify this array
        my @arrays = (@array);
        
        # new 'indexes' scalar option added at the end, 
        # so as not to break any existing formulas involving MapArray
        my $indexes;
        $indexes = pop @arrays unless ref $arrays[$#arrays] eq 'ARRAY';
        
        # find the number of indexes to cover
        my $numArrays = $#arrays;
        my $maxElems = max(map $#{$_}, @arrays);
        my @indexes = (0..$maxElems);
        
        # may target specific indexes, to reduce the N/A present,
        # or to focus on specific bins for engineering/business considerations
        if ($indexes) {
                my %indexes;
                foreach my $index (split /,/, $indexes) {
                        if ($index =~ /^(\d+)$/) {
                                $indexes{$1} = 1 unless (0+$1) > $maxElems;
                        } elsif ($index =~ /^\-(\d+)$/) {
                                $indexes{$_} = 1 foreach 0..$1;
                        } elsif ($index =~ /^(\d+)\-$/) {
                                $indexes{$_} = 1 foreach $1..$maxElems;
                        } elsif ($index =~ /^(\d+)\-(\d+)$/) {
                                $indexes{$_} = 1 foreach $1..min(0+$2,$maxElems);
                        } else {
                                die "Invalid MapArray index: '$index'\n"
                        }
                }
                @indexes = sort keys %indexes;
        }
        
        # cannot use: require Itk::PmData::Formula
        # cannot use: use Itk::PmData::Formula as it impacts Safe
        my @result = map {
                my $idx = $_;
                my @args = map $_->[$idx], @arrays;
                my $r;
                eval {
                        $r = &$sub(@args, $idx);
                };
                if ($@ and $@ =~ /division\sby\szero/) {
                        $r = '!DIV0!';
                } elsif ($@) {
                        $r = '!EVAL!';
                } elsif (!defined $r) {
                        $r = '!NULL!';
                }
                $r;
        } @indexes;
        
        # if a single bin has been targetted, we return a scalar
        return $result[0] if $indexes && scalar(@indexes) == 1;
        
        return \@result;
}
##-----------------------------------------------------------------------------
# PdfNormal($pdfCounter)
# Returns a normalized version of an input PDF array - such that the sum of all
# bins is 100
# - $pdfCounter		Ref to PDF Array
# RETURNS: $pdfNormal
# - $pdfNormal		Ref to normalized PDF
##-----------------------------------------------------------------------------
sub PdfNormal {
	my ($pdf) = @_;
	die "PdfNormal: Argument 1 should be an array reference\n" unless ref $pdf eq 'ARRAY';
	my $total=0; 
	$total+=$_ foreach @$pdf; 
	my @pdfNormal;
	if ($total != 0) {
		push @pdfNormal, 100 * $_ / $total foreach @$pdf;
	} else {
		push @pdfNormal, 0 foreach @$pdf;
	}
	return \@pdfNormal;
}
##-----------------------------------------------------------------------------
# SumArrayElements($array)
# Return sum of elements in array
# - $array	Ref to array
# RETURNS: $sum
# - $sum	Sum of elements
#------------------------------------------------------------------------------
sub SumArrayElements {
	die "Non-Array supplied as parameter to SumArrayElements()\n" unless ref $_[0] eq 'ARRAY';
	return sum (@{$_[0]});
}
##-----------------------------------------------------------------------------
# SumArrays($array1[,$array2,...])
# Return an array which is the indexed sum of each supplied array.  ie
# all elements at index 0 are summed for each array.
# - $array1 Ref to array
# - $array2 Ref to array
# ....
# RETURNS: $newarray
# - $newarray New array 
#------------------------------------------------------------------------------
sub SumArrays {
	my (@arrays) = @_;
	
	die "Two or more array counters are required for SumArrays()\n" unless @arrays>1;
	my $i=0;
	++$i && ref $_ ne 'ARRAY' && die "SumArrays: Non-array supplied as argument $i\n" foreach @arrays;
	
	my $maxsize = max(map scalar @$_, @arrays);
	my @out = map 0, (0..$maxsize-1);
	
	my $i=1;
	foreach my $ary (@arrays) {
		my $j=0;
		$out[$j++] += $_ foreach @$ary;
		$i++;
	}
	
	return \@out;	
}
##-----------------------------------------------------------------------------
# WeightedStdDev($array, $weightingarray, $bin)
# Return weighted std dev.
# - $array           Ref to array (these are numbers of samples)
# - $weightingarray  Ref to array of weightings (these are kbps, pct, etc)
# - $bin             If set to 'bin', then average in bin will be returned too
# RETURNS: $weightedstddev
# - $weightedstddev  Weighted Std Dev of $array: if $bin => [in bins, in units]
#------------------------------------------------------------------------------
sub WeightedStdDev {
	my $d = shift;
	my $w = shift;
	my $b = shift;
	
	# counters may be undef for certain ROP's, that is not a fatal error
	return undef unless defined $d;
	
	die "Non-Array supplied as first parameter to WeightedStdDev()\n" unless ref $d eq 'ARRAY';
	
	if (defined $w) {
		die "Non-Array supplied as second parameter to WeightedStdDev()\n" unless ref $w eq 'ARRAY';
	} else {
		$w = [];
		push @$w, $_ for 0 .. $#{$d};
	}
	
	if ($#{$d} > $#{$w}) {
		die 'Counter array ('.($#{$d}+1).') larger than weighting array ('.($#{$w}+1).") in WeightedStdDev()\n";
	}
	
	if ($#{$d} < $#{$w}) {
		my $toAdd = $#{$w} - $#{$d};
		my @parray = @$d;
		push @parray, (0) x $toAdd;
		$d = \@parray;
	}
	
	my $count = 0.0;
	my $wsum  = 0.0;
	for (my $i=0; $i<=$#{$d}; $i++) {
		$count +=  $d->[$i];
		$wsum  += ($d->[$i] * $w->[$i]);
	}
	
	my $avrg = $count ? $wsum/$count : 0.0;
	
	my $bavrg;
	if (defined $b && $b eq 'bin') {
		my $bwsum = 0.0;
		for (my $i=0; $i<=$#{$d}; $i++) {
			$bwsum += ($d->[$i] * $i);
		}
		$bavrg = $count ? $bwsum/$count : 0.0;
	}
	
	# factor due to small sample size
	$count -= 1.0;
	
	$wsum = 0.0;
	for (my $i=0; $i<=$#{$d}; $i++) {
		$wsum += ($d->[$i] * ($w->[$i]-$avrg)**2);
	}
	
	if (defined $b && $b eq 'bin') {
		my $bwsum = 0.0;
		for (my $i=0; $i<=$#{$d}; $i++) {
			$bwsum += ($d->[$i] * ($i-$bavrg)**2);
		}
		return $count ? [($bwsum/$count)**0.5,($wsum/$count)**0.5] : [0.0,0.0];
	}
	
	return $count ? ($wsum/$count)**0.5 : 0.0;
}

#----------------------------------------------------------------------------------------------
# MAIN
#----------------------------------------------------------------------------------------------

# Validate parameters
printHelp() if ($help);
die("-h and -d are mutually exclusive\n") if ($hourly && $daily);
die("Invalid Column Specification\n") if !($cols =~ /^(ctr|mo|time)$/);
die("Invalid Format Specification\n") if !($fmt =~ /^(txt|csv|html|htmltab)$/);
die("-rdef must be specified if -r is used\n") if (@relfilter and !@reldef_file);
die("Number of -rdef and -r options must match\n") if ($#relfilter != $#reldef_file);
die("-m must be specified if -r is used\n") if (@relfilter and ! $mofilter);
die("-pipe only valid for txt or csv output formats\n") if ($pipe_output and $fmt !~ /^(txt|csv)$/);
die("-sort can only be used if -cols=ctr\n") if ($sort_key and $cols ne 'ctr');
die("-filter can only be used if -cols=ctr\n") if ($filter_exp and $cols ne 'ctr');

# Read xfile and ifile, if supplied
if ($xfile) {
	open FFILE, "<$xfile" or die "Could not open $xfile";
	while (<FFILE>) {
		chomp;
		s/#.*$//;          # Remove Comments
		next if (/^\s*$/); # Skip blank lines
		$cntr_filter = $_;
	}
	close FFILE;
}
if ($ifile) {
	open FFILE, "<$ifile" or die "Could not open $ifile";
	while (<FFILE>) {
		chomp;
		s/#.*$//;          # Remove Comments
		next if (/^\s*$/); # Skip blank lines
		$include_filter = $_;
	}
	close FFILE;
}
# Precompile filters
my $cntr_filter_re = qr/$cntr_filter/;
my $include_filter_re = qr/$include_filter/;
my $mofilter_re = qr/$mofilter/;
my $i;
my @relfilter_re;

for ($i=0; $i<=$#relfilter; $i++) {
	my $re=$relfilter[$i];
	$relfilter_re[$i] = qr/$re/;
}

# Precision of data fields
if ($precision =~ /(\d+)(\.\d+)/) {
	$df_numchars=$1;
	$df_numdec=$2;
} else {
	die "Precision must be in form x.y\n";
}

# If user wants to pipe output..
if ($pipe_output) {
	$pipe_pid=open2(\*FILTER_IN,\*FILTER_OUT,"$pipe_output");
	$outputfh = \*FILTER_OUT;
	# Set FILTER_IN to non-blocking, so we don't have any deadlocks
	fcntl(\*FILTER_IN, F_SETFL(), O_NONBLOCK());
} else {
	$outputfh = \*STDOUT;
}


# Read predefined formulas from file, if supplied
if ($formula_file) {
	open FFILE, "<$formula_file" or die "Could not open $formula_file";
	while (<FFILE>) {
		chomp;
		s/#.*$//;          # Remove Comments
		next if (/^\s*$/); # Skip blank lines
		push @formulas_txt,$_ if ( (! $include_filter) || $_ =~ $include_filter_re);
	}
	close FFILE;
}

# Read Mo Relations from File if supplied
if (scalar @reldef_file) {
	for ($i=0; $i<=$#reldef_file; $i++) {
		my ($mo, $rmo);
		open RFILE, "<$reldef_file[$i]" or die "Could not open $reldef_file[$i]";
		while (<RFILE>) {
			next if (/^\s*$/); # Skip blank lines
			($mo, $rmo) = split ':';
			$relmo[$i]{$mo}=$rmo;
		}
		close RFILE;
	}
}
# Creation of hash table containing Compressed counters
# Our $CompressedCounters
# Example of custom file values:
# EUTranCellFDD;pmErabEstabAttAddedCsfbArp;compressed          	#Valid counter
# EUTranCellFDD;pmErabEstabAttAddedCsfbP1;			#Will be ignored
my @a;
if(@mom_file){
	open(RFILE, '<', @mom_file) or die "Could not open @mom_file";
	while(<RFILE>){
		if($_ =~ /compressed/){
			# Splits each line into three parts and takes the middle word
			@counter = split(';', $_);
			push @a, @counter[1];
		}
	}
	close(RFILE);
} else {
@a = split(' ','pmAasHorizontalDistr pmAasSchedPrbDlHorizontalDistr pmAasSchedPrbDlVerticalDistr pmAasVerticalDistr pmActiveDrbDlSumQci pmActiveUeDlSumQci pmBadCovMeasTimeActDistr pmBadCovMeasTimeDistr pmBranchDeltaSinrDistr0 pmBranchDeltaSinrDistr1 pmBranchDeltaSinrDistr2 pmBranchDeltaSinrDistr3 pmBranchDeltaSinrDistr4 pmBranchDeltaSinrDistr5 pmBranchDeltaSinrDistr6 pmCaConfigDlSumEndcDistr pmCaConfigDlSumSaDistr pmCaWaitGoodCov pmDrbEstabAtt5qi pmDrbEstabAttArp pmDrbEstabSucc5qi pmDrbEstabSuccArp pmDrbRelAbnormalAmf5qi pmDrbRelAbnormalAmfAct5qi pmDrbRelAbnormalAmfActArp pmDrbRelAbnormalAmfArp pmDrbRelAbnormalGnb5qi pmDrbRelAbnormalGnbAct5qi pmDrbRelAbnormalGnbActArp pmDrbRelAbnormalGnbArp pmDrbRelNormal5qi pmDrbRelNormalArp pmDrbThpTimeDlQci pmEmfPwrBackoffDynResThrDistr pmEmfPwrBackoffOnPwrDistr pmEmfPwrBackoffPwrDistr pmEmfPwrBackoffStepPwrDistr pmErabEstabAttAddedCsfbArp pmErabEstabAttAddedCsfbQci pmErabEstabAttAddedGbrArp pmErabEstabAttAddedHoOngoingArp pmErabEstabAttAddedHoOngoingQci pmErabEstabAttAddedQci pmErabEstabAttInitGbrArp pmErabEstabAttInitQci pmErabEstabAttRejAdmMsrAddedArp pmErabEstabAttRejAdmMsrAddedPttArp pmErabEstabAttRejAdmMsrAddedRpid pmErabEstabAttRejAdmMsrInitArp pmErabEstabAttRejAdmMsrInitPttArp pmErabEstabAttRejAdmMsrInitRpid pmErabEstabFailUeAdmLicRejArp pmErabEstabSuccAddedGbrArp pmErabEstabSuccAddedQci pmErabEstabSuccInitGbrArp pmErabEstabSuccInitQci pmErabLevRopStartQci pmErabModAttQci pmErabModSuccQci pmErabQciLevSum pmErabQciMax pmErabRelAbnormalEnbActArp pmErabRelAbnormalEnbActCdtAutoPnrQci pmErabRelAbnormalEnbActCdtAutoQci pmErabRelAbnormalEnbActCdtQci pmErabRelAbnormalEnbActGbrArp pmErabRelAbnormalEnbActHoQci pmErabRelAbnormalEnbActHprQci pmErabRelAbnormalEnbActPeQci pmErabRelAbnormalEnbActPeRpid pmErabRelAbnormalEnbActQci pmErabRelAbnormalEnbActTnFailQci pmErabRelAbnormalEnbActUeLostQci pmErabRelAbnormalEnbArp pmErabRelAbnormalEnbCdtAutoPnrQci pmErabRelAbnormalEnbCdtAutoQci pmErabRelAbnormalEnbCdtQci pmErabRelAbnormalEnbGbrArp pmErabRelAbnormalEnbHoQci pmErabRelAbnormalEnbHprQci pmErabRelAbnormalEnbPeQci pmErabRelAbnormalEnbPeRpid pmErabRelAbnormalEnbPsPartFailQci pmErabRelAbnormalEnbQci pmErabRelAbnormalEnbTnFailQci pmErabRelAbnormalEnbUeLostQci pmErabRelAbnormalMmeActArp pmErabRelAbnormalMmeActGbrArp pmErabRelAbnormalMmeActQci pmErabRelAbnormalMmeArp pmErabRelAbnormalMmeGbrArp pmErabRelAbnormalMmeQci pmErabRelAllEnbQci pmErabRelAllMmeQci pmErabRelDlInactGapQci pmErabRelMmeActQci pmErabRelMmeArp pmErabRelMmeGbrArp pmErabRelMmeQci pmErabRelNormalEnbActArp pmErabRelNormalEnbActQci pmErabRelNormalEnbArp pmErabRelNormalEnbGbrArp pmErabRelNormalEnbQci pmErabRelNormalEnbUserInactivityQci pmErabRelUlInactGapQci pmHoExeOutAttQci pmHoExeOutSuccQci pmHoPrepRejInAdmMsrRpid pmInhibitImeisvFeature pmLaaSelectedChannel pmMacContentionDelayDlMaxQos pmMacDrbThpTimeDlQci pmMacDrbThpVolDlQci pmMacLatTimeDlDrxSyncQci pmMacLatTimeDlDrxSyncQos pmMacLatTimeDlDrxSyncSampQos pmMacLatTimeDlNoDrxNoSyncQci pmMacLatTimeDlNoDrxSyncQci pmMacLatTimeDlNoDrxSyncQos pmMacLatTimeDlNoDrxSyncSampQos pmMacLatTimeDlQci pmMacTimeDlDrbLastSlotQos pmMacTimeDlDrbLastSlotSampQos pmMacTimeDlDrbQos pmMacTimeDlDrbSampQos pmMacTimeDlDrxSyncSampQci pmMacTimeDlNoDrxNoSyncSampQci pmMacTimeDlNoDrxSyncSampQci pmMacTimeDlSampQci pmMacVolDlDrbLastSlotQos pmMacVolDlDrbQos pmMeasCellGroupUeThp2DlDistr pmMeasCellGroupUeThp2UlDistr pmMeasGapUeMaxDistr pmMeasGapUeMnMaxDistr pmMeasGapUeMnSumDistr pmMeasGapUeSumDistr pmModImeisvParamSet pmPdcchCceUsedDlQci pmPdcchCceUsedUlQci pmPdcpLatPktTransDlCatMDrxNoSyncQci pmPdcpLatPktTransDlCatMDrxSyncQci pmPdcpLatPktTransDlCatMNoDrxNoSyncQci pmPdcpLatPktTransDlCatMNoDrxSyncQci pmPdcpLatPktTransDlDrxNoSyncQci pmPdcpLatPktTransDlDrxSyncQci pmPdcpLatPktTransDlNoDrxNoSyncQci pmPdcpLatPktTransDlNoDrxSyncQci pmPdcpLatPktTransDlQci pmPdcpLatTimeDlCatMDrxNoSyncQci pmPdcpLatTimeDlCatMDrxSyncQci pmPdcpLatTimeDlCatMNoDrxNoSyncQci pmPdcpLatTimeDlCatMNoDrxSyncQci pmPdcpLatTimeDlDrxNoSyncQci pmPdcpLatTimeDlDrxSyncQci pmPdcpLatTimeDlNoDrxNoSyncQci pmPdcpLatTimeDlNoDrxSyncQci pmPdcpLatTimeDlQci pmPdcpPktDiscDlAqmQci pmPdcpPktDiscDlHoQci pmPdcpPktDiscDlPelrQci pmPdcpPktDiscDlPelrUuQci pmPdcpPktFwdRecDlDiscQos pmPdcpPktFwdRecDlQos pmPdcpPktFwdTransDlDiscQos pmPdcpPktFwdTransDlQos pmPdcpPktLossUl5qi pmPdcpPktLossUlQos pmPdcpPktLossUlTo5qi pmPdcpPktLossUlToDisc5qi pmPdcpPktLossUlToDiscQos pmPdcpPktLossUlToQos pmPdcpPktLostUlMissingPdus2Qci pmPdcpPktLostUlQci pmPdcpPktLostUlRohcFail2Qci pmPdcpPktRecDl5qi pmPdcpPktRecDlDisc5qi pmPdcpPktRecDlDiscAqm5qi pmPdcpPktRecUl5qi pmPdcpPktRecUlDiscIntgQos pmPdcpPktRecUlIntgQos pmPdcpPktRecUlOoo5qi pmPdcpPktRecUlOooQos pmPdcpPktRecUlQos pmPdcpPktReceivedDlQci pmPdcpPktReceivedUlQci pmPdcpPktTransDl5qi pmPdcpPktTransDlAck5qi pmPdcpPktTransDlAckQos pmPdcpPktTransDlAggr5qi pmPdcpPktTransDlAggrQos pmPdcpPktTransDlDisc5qi pmPdcpPktTransDlDiscAqm5qi pmPdcpPktTransDlDiscAqmQos pmPdcpPktTransDlDiscQos pmPdcpPktTransDlIntgQos pmPdcpPktTransDlQci pmPdcpPktTransDlQos pmPdcpPktTransDlRetrans5qi pmPdcpPktTransDlRetransQos pmPdcpPktTransUl5qi pmPdcpVolDlCmpHdrQci pmPdcpVolDlDrbFiltQci pmPdcpVolDlDrbLastTTIQci pmPdcpVolDlDrbQci pmPdcpVolDlDrbTransQci pmPdcpVolDlHdrQci pmPdcpVolRecUl5qi pmPdcpVolRecUlQos pmPdcpVolTransDl5qi pmPdcpVolTransDlAggr5qi pmPdcpVolTransDlAggrQos pmPdcpVolTransDlQos pmPdcpVolTransDlRetrans5qi pmPdcpVolTransDlRetransQos pmPdcpVolTransUl5qi pmPdcpVolUlCmpHdrQci pmPdcpVolUlDrbQci pmPdcpVolUlHdrQci pmPpTestCompressedPdf pmPpTestMcsRpUserPlaneLinkCPdf pmPpTestMcsS1ULinkCPdf pmPpTestMcsX2ULinkCPdf pmPpTestMcsX2UTerminationCPdf pmPrbUsedDlDtchFirstTransQci pmPrbUsedMimoLayersDlDistr pmPrbUsedMimoLayersUlDistr pmPrbUsedUlDtchQci pmRaAttTaZone0Distr pmRaAttTaZone1Distr pmRaAttTaZone2Distr pmRaSuccTaZone0Distr pmRaSuccTaZone1Distr pmRaSuccTaZone2Distr pmRadioUeRepPmiPrimaryCsiRsRank1Distr pmRadioUeRepPmiPrimaryCsiRsRank2Distr pmRadioUeRepPmiPrimaryCsiRsRank3Distr pmRadioUeRepPmiPrimaryCsiRsRank4Distr pmRadioUeRepPmiSecondaryCsiRsRank1Distr pmRadioUeRepPmiSecondaryCsiRsRank2Distr pmRadioUeRepPmiSecondaryCsiRsRank3Distr pmRadioUeRepPmiSecondaryCsiRsRank4Distr pmRadioUeRepRankAggrDistr pmRcTestCompressedPdf pmRlcDelayPktTransDlQci pmRlcDelayPktTransmitDlQos pmRlcDelayTimeDl2Qci pmRlcDelayTimeDlQci pmRlcDelayTimeDlQos pmRpcTestCompressedPdf pmRpcTestMcsNrCellDuCPdf pmRpcTestMcsNrSectorCarrierDuCPdf pmRrcConnReestAttQci pmRrcConnReestFailInAdmMsrRpid pmRrcConnReestSuccQci pmRrcConnResumeTimeDistr pmRrcConnResumeTimeDistrBr pmS1uPktLostDlQci pmS1uPktRecOooQci pmS1uPktRecQci pmServiceTimeDrbQci pmSessionTimeDrb5qi pmSessionTimeDrbQci pmTaInit2Distr pmTaInitBrDistr pmUeCtxtFetchSuccErabActQci pmZtemporary131Compr pmZtemporary132Compr pmZtemporary133Compr pmZtemporary134Compr pmZtemporary135Compr pmZtemporary158Compr pmZtemporary159Compr pmZtemporary16Qci pmZtemporary17Qci pmZtemporary18Qci pmZtemporary224Compr pmZtemporary225Compr pmZtemporary226Compr pmZtemporary230Compr pmZtemporary231Compr pmZtemporary232Compr pmZtemporary233Compr pmZtemporary59 pmZtemporary80 pmZtemporary81 pmZtemporary82 pmZtemporary83',-1);
}

foreach $i ($[ .. $#a) { $CompressedCounters{$a[$i]} = 1; }

# Table containing the list of PDF counters
my %isPdf;

my ($ts, $mo, $counter, $value);

	
# The number of entries (timestamps) for each counter.
my %num_entries;

# Parse input
COUNTERDATA: while (<>) {

	# Is $ts used in global scope?
	($ts, $mo, $counter, $value) = split ';';
	
	# Skip any lines that don't look like valid counters
	next if (! ($ts =~ /(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)/));
	
	my ($yy,$mm,$dd,$hh,$mi,$ss) = ($1,$2,$3,$4,$5,$6);
	
	my $mo_agg;
	
	# Figure out MO Aggregation
	if ($mofilter) {
		
		$mo =~ $mofilter_re or next; # Skip any rows not matched
        $mo_agg=$1;

		if (scalar @relfilter) {
		        for ($i=0; $i<=$#reldef_file; $i++) {
				#print ".. $i $mo_agg $relmo[$i]{$mo_agg}\n";
				if ($relmo[$i]{$mo_agg} =~ $relfilter_re[$i]) {
					if ($relmap_accum) {
					        $mo_agg=sprintf("%s (%s)",$mo_agg,$1);
					        $mo_agg =~ s/RncModule=/Mod=/;
					} else {
					        $mo_agg=$1;
					}
				} else {
					#warn "Warning: Object $mo_agg is not matched by rule $relfilter[$i], file $reldef_file[$i]\n";
					$reldef_filename = $reldef_file[$i];
					$reldef_filename =~ s/^.*\///;
					$warnlist{"Warning: Object $mo_agg is not matched by rule $relfilter[$i], file $reldef_filename\n"} = 1;
					if ($relmap_accum) {
					        $mo_agg=sprintf("%s (%s)",$mo_agg,"Unknown");
					        $mo_agg =~ s/RncModule=/Mod=/;
					} else {
					        $mo_agg="Unknown";
					}
					#next COUNTERDATA;
				}
			}
		}
		
	} else {
		$mo_agg = $mo;
	}
	
	my $tu;
	
	# Define time level aggregation
	if ($hourly) {
		$tu="$yy-$mm-$dd $hh:00";
	} elsif ($daily) {
		$tu="$yy-$mm-$dd";
	} elsif ($all_time) {
		$tu="";
		$min_time = $ts if ($ts lt $min_time or (! $min_time));
		$max_time = $ts if ($ts gt $max_time or (! $max_time));
	} elsif ($secondly) {
		$tu="$yy-$mm-$dd $hh:$mi:$ss";
	} else {
		$tu="$yy-$mm-$dd $hh:$mi";
	}
	if ($fmt eq 'csv') {
		$csv_min_time = $ts if ($ts lt $csv_min_time or (! $csv_min_time));
		$csv_max_time = $ts if ($ts gt $csv_max_time or (! $csv_max_time));
	}
	
	# Maps are used so that we can step through all possible times, mo's and counters
	$ctr_map{$counter}=1;
	$mo_map{$mo_agg}=1;
	$ts_map{$tu}=1;

	# Check if we have an array value (consists of values separated by commas)
	# If it is an array, then the hash contains an array reference instead of
	# a scalar
	# Do not aggregate if the value contains non numeric characters (eg moRef, string, etc) or if the value is empty and we are not aggregating on time basis or ManagedElement level
	# Allow aggregation even if the counter name does not start with pm (used to be: || $counter !~ /^[pP]m/ )
	if ($value =~ /[A-Z]|[a-z]/ || ($value =~ /^[ \t]*$/ and not $all_time and not $hourly and not $daily and not $me_agg) ) {
		chomp $value;
		#replace the traffic descriptor with its peak cell rate
		if ($value =~ /,AtmTrafficDescriptor=[^P]+P([0-9]+).*$/) { $value = $1; }
		$data{$tu}{$mo_agg}{$counter}=$value;
		#Might be used in in formulas, but it does not make a difference if they are removed.
		$sum{$counter}=$value;
		$sum_time{$tu}{$counter}=$value;
		$sum_mo{$mo_agg}{$counter}=$value;
	}
	elsif ($value =~ /,/ || (exists $isPdf{$counter}) || (exists $CompressedCounters{$counter})) {
		chomp $value;
		my @values=split ',',$value;
		$isPdf{$counter}=1;
		array_add(\$data{$tu}{$mo_agg}{$counter}, \@values, $counter);
		#Might be used in in formulas, but it does not make a difference if they are removed..
		array_add(\$sum{$counter},\@values, $counter);
		array_add(\$sum_time{$tu}{$counter},\@values, $counter);
		array_add(\$sum_mo{$mo_agg}{$counter},\@values, $counter);
	} else {
		$data{$tu}{$mo_agg}{$counter}+=$value;
		#Might be used in in formulas, but it does not make a difference if they are removed.
		$sum{$counter}+=$value;
		$sum_time{$tu}{$counter}+=$value;
		$sum_mo{$mo_agg}{$counter}+=$value;
	}
	$num_entries{$mo_agg}{$counter}+=1;

	# Record the different ROPs for each MO.
	$mo_ts{$mo_agg}{$ts} = 1;
}


# If possible, reduce times. If all samples in the same day date is removed.
my ($one_day_only, $date_hdr, $extra_headline) = reduce_time_format;

# Parse Formulas
parse_formulas;

# Execute formulas
exec_formulas;

# Filter the output counters
if ($cntr_filter) {
	foreach my $ctr (keys %ctr_map) {
		delete $ctr_map{$ctr} if ($ctr !~ $cntr_filter_re);
		#If pval=2, the printed format of the $ctr_map entries are the value of formula_pval_filter.
		if ($pval == 2 && defined($ctr_map{$ctr}) && defined($formula_pval_filter{$ctr})) {
			delete $ctr_map{$ctr};
			$ctr_map{$formula_pval_filter{$ctr}}=1;
		};
	}
}

# Check if MOs should be sorted numerically or not
my $numMo=1;
foreach my $mo (sort keys %mo_map) {
	$numMo=0 if ($mo !~ /^[0-9.]+$/);
}

my $sort_function;

# MO List can be sorted Numerically, Alphabetically or by one counter value
# Here we create the subroutine that will be used to sort MOs
if ($sort_key =~ /^(\w+)([\-\+])*$/) {
	if ($2 eq '-') {
		#descending
		my $sort_key1=$1;
		$sort_function = sub { 
			$data{$ts}{$b}{$sort_key1} <=> $data{$ts}{$a}{$sort_key1}
			or
			$numMo ? $a <=> $b : $a cmp $b;
		};
	} else {
		# ascending
		my $sort_key1=$1;
		$sort_function = sub { 
			$data{$ts}{$a}{$sort_key1} <=> $data{$ts}{$b}{$sort_key1}
			or
			$numMo ? $a <=> $b : $a cmp $b;
		};
	}
} elsif ($numMo) {
	$sort_function = sub { $a <=> $b; };
} else {
	$sort_function = sub { $a cmp $b; };
}

# Output data
my $num_rows=1;
if ($cols eq 'ctr') {
	out_headrow($extra_headline, $date_hdr,"Object",sort keys %ctr_map);
	foreach $ts (sort keys %ts_map) {
		foreach $mo (sort $sort_function keys %mo_map) {
			next if ($filter_exp and not eval($filter_exp));
			my @row = ($ts,$mo);
			foreach my $ctr (sort keys %ctr_map) {
				# If using the "avg" option
				if ($avg) {
					pushAvg(\@row, $data{$ts}{$mo}{$ctr}, $num_entries{$mo}{$ctr});
				}else {
					push @row,$data{$ts}{$mo}{$ctr}; 
				}
			}
			out_row(@row);
			last if ($top and $num_rows++ >= $top);
		}
	}
} elsif ($cols eq 'mo') {
	my @mo_list=(sort $sort_function keys %mo_map);
	out_headrow($extra_headline, $date_hdr,$Counter,@mo_list);
	foreach $ts (sort keys %ts_map) {
		foreach my $ctr (sort keys %ctr_map) {
			my @row = ($ts,$ctr);
			my $colcnt=0;
			foreach $mo (@mo_list) {
				# Only try to output if this MO instance
				# actually has this counter, otherwise
				# output a blank column
				if (exists $data{$ts}{$mo}{$ctr}) {
					# If using the "avg" option
					if ($avg) {
						pushAvg(\@row, $data{$ts}{$mo}{$ctr}, $num_entries{$mo}{$ctr});
					} else{
						push @row,$data{$ts}{$mo}{$ctr};
					}
					$colcnt++;
				} else {
					push @row,undef;
				}
			}
			# Only need to output this row if there was some 
			# columns!
			if ($colcnt) {
				out_row(@row);
				last if ($top and $num_rows++ >= $top);
			}
		}
	}
} elsif ($cols eq 'time') {
	if ($fmt eq 'txt' and not $one_day_only and not $daily and not $all_time) {
		# In the case of a text output, we want to break it into days
		# to minimise wrapping
		my (@times, $day, $time, @days, $daystring, %day_times);
		my $current_day="";
		my @timestamps = (sort keys %ts_map);
		foreach $ts (@timestamps) {
			($day,$time) = ($ts =~ /(\d\d\d\d-\d\d-\d\d) (\d\d:\d\d)/);
			$current_day = $day if (! $current_day);
			if ($q || $day eq $current_day) {
				push @days, $day unless ($daystring =~ $day);
				$daystring = join(" ",@days);
			} elsif ($day ne $current_day) {  #and ($ts ne $timestamps[$#days])
				out_headrow("Date: $current_day\n", "Object",$Counter,@times);
				foreach $mo (sort $sort_function keys %mo_map) {
					foreach my $ctr (sort keys %ctr_map) {
						my @row = ($mo,$ctr);
						my $colcnt=0;
						foreach $ts (@times) {
							if (exists $data{"$current_day $ts"}{$mo}{$ctr}) {
								push @row,$data{"$current_day $ts"}{$mo}{$ctr};
								$colcnt++;
							} else {
								push @row, undef;
							}
						}
						if ($colcnt) {
							out_row(@row);
							last if ($top and $num_rows++ >= $top);
						}
					}
				}
				$current_day=$day;
				undef @days;
				$days[0]=$current_day;
				$daystring = $current_day;
				@times=();
			}
			push @times, $time; # Only used to print the list of time slots.
			push @{$day_times{$day}}, $time;
		}
		if (@times) {
			out_headrow("Date: " . join(", ", @days) . "\n", "Object",$Counter,@times);
			foreach $mo (sort $sort_function keys %mo_map) {
				foreach my $ctr (sort keys %ctr_map) {
					my @row = ($mo,$ctr);
					my $colcnt=0;
					foreach my $day (@days) {
						my $tmp_data;
						foreach my $ts (@{$day_times{$day}}) {
							$tmp_data = $data{"$day $ts"}{$mo}{$ctr};
							if (defined $tmp_data) {
								push @row,$data{"$day $ts"}{$mo}{$ctr};
								$colcnt++;
								#last;
							} else {
								push @row, undef;
							}
						}
						push @row, undef unless (defined $tmp_data);
					}
					if ($colcnt) {
						out_row(@row);
						last if ($top and $num_rows++ >= $top);
					}
				}
			}
		}
	} else {
		out_headrow($extra_headline, "Object",$Counter,sort keys %ts_map);
		foreach $mo (sort $sort_function keys %mo_map) {
			foreach my $ctr (sort keys %ctr_map) {
				my @row = ($mo,$ctr);
				my $colcnt=0;
				foreach $ts (sort keys %ts_map) {
					$data{$ts}{$mo}{$ctr};
					if (exists $data{$ts}{$mo}{$ctr}) {
						my $val = $data{$ts}{$mo}{$ctr};
						# If using the "avg" option
						if ($avg) {
							pushAvg(\@row, $val, $num_entries{$mo}{$ctr});
						} else{
							push @row,$data{$ts}{$mo}{$ctr};
						}
						$colcnt++;
					} else {
						push @row,undef;
					}
				}
				if ($colcnt) {
					out_row(@row);
					last if ($top and $num_rows++ >= $top);
				}
			}
		}
	}
	
}

# Calculates the average of $val (can be scalar or array) with regard to number of $entries and pushes it to $rowref.
sub pushAvg {
	my ($rowref, $val, $entries) = @_;
	unless ($val || $entries) {
		push @$rowref, undef;
	} else{
		unless (ref($val) eq 'ARRAY'){
			# If it's scalar, divide by the number of entries to calculate average.
			my $avg_val = $val/$entries;
			push @$rowref, $avg_val;
		} else{
			# If it's an array, divide each value by the number of entries.
			my $avgs = join (',',  map {$_ / $entries} @$val);
			push @$rowref, $avgs;
		}
	}
}

out_footer;
