#!/usr/bin/perl
#
# mobatch 5.0 2022-10-04
# Finn Magnusson, finn.magnusson@ericsson.com 
#        purpose: To send moshell commands to several nodes in parallel
#        Type "mobatch" on its own for help

######################################################################################
#                                                                                    #
#  Ericsson AB 2001-2019    - All Rights Reserved                                   #
#                                                                                    #
# The copyright to the computer program(s) herein is the property   of Ericsson AB,  #
# Sweden. The programs may be used and/or copied only with the written permission    #
# from Ericsson AB or in accordance with the terms and  conditions stipulated in the #
# agreement/contract under which the program(s) have been supplied.                  #
#                                                                                    #
######################################################################################

###########################################################################################
# The below variables correspond to the uservariables starting with "mobatch_" and can be set in the moshellrc or ~/.moshellrc 
# - MAX_P_OPTION: maximum value for the "-p" option (default: 0 = unlimited). If set to greater than 0 and the batch size value given in "-p" is greater then it will be lowered to match the value of MAX_P_OPTION
# - MAX_NODES: maximumn number of nodes in the sitefile (default: 0 = unlimited). If set to greater than 0 and the number of nodes in the sitefile is greater than MAX_NODES then the mobatch will stop after having executed MAX_NODES nodes.
# - MAX_PROCESSES: the default value for "-p" option when "-p" is not specified
###########################################################################################
my $MAX_P_OPTION = 0;
my $MAX_NODES    = 0;
my $MAX_PROCESSES = 10;
my $OFFSET=0;
###########################################################################################


###########################################################################################
# Paths to files:
###########################################################################################
my ($moshelldir) = `dirname "$0"`; chomp($moshelldir); 
if ($moshelldir !~ /^\//) 
{ 
	my $cwd = `pwd`; chomp($cwd) ; 
	$moshelldir = sprintf("%s/%s",$cwd,$moshelldir);
}
my $logdir     =  "$moshelldir/logfiles/logs_mobatch";
my $monode     = "$moshelldir/moshell";
my $pstool     = "$moshelldir/pstool";
my $jarxml	   = "$moshelldir/jarxml";
if ($moshelldir =~ "/moshellroot") {
	$logdir = "/home/moshell/moshell_logfiles/$ENV{'USER'}/logs_mobatch/" ;
	$monode = "$moshelldir/bin/moshell";
} else {
	my $defLogDir  = "$moshelldir/commonjars/defLogDir.sh";
	my $temp_logdir= `$defLogDir mobatch`;
	if ( -d $temp_logdir && -w $temp_logdir ) { $logdir = $temp_logdir ; }
}	
###########################################################################################


###########################################################################################
# Read MAX_P_OPTION, MAX_NODES, MAX_PROCESSES, $OFFSET from moshellrc files:
###########################################################################################
my $filename = "$jarxml/moshellrc";
readFromFile($filename);
my $filename = $ENV{"HOME"} . "/.moshellrc";
readFromFile($filename);
###########################################################################################


if ($moshelldir =~ "/moshellroot") { use lib "/moshellroot/commonjars/lib";}
else 
{
	#use lib sprintf("%s/commonjars/lib",$moshelldir); --> does not work
	use lib substr($0, 0, rindex($0,"/")) . '/commonjars/lib';
	use FindBin;
	use lib "$FindBin::Bin/commonjars/lib";
}
use warnings;
use strict;
use Parallel::ForkManager;
use Getopt::Std;
use File::Basename;
use File::Path;
use FileHandle;
use DirHandle;
use Time::localtime;
use IO::File;

my $totDuration = 0;
my $avgDuration = 0;
my $totSitesOk = 0;
my $commandfile   = "";
my $commanddir    = "";
my $command       = "";
my $sitefile      = "";
my $sitedir       = "";
my $sitefilename  = "";
my $dumpfile      = "";
my $tmp           = "";
my $logdirectory  = "";
my $moswitch      = "";
my $sitelist      = "";
my $settimeout    = 0;
my $setproc       = 0;
my $silent        = 0;
my $recursive     = 0;
my $oldcount      = 0;
my $waitInterval  = 2.5;
my $gz            = 0;
my $optionslist   = "";
my $Dm = 0;
my $Ds = 0;
my $mobatchstart  = time();
my $b_option      = "";
my $amostempfile = sprintf( "/tmp/amostempfile_%04d%02d%02d_%02d%02d%02d_%s", localtime->year() + 1900, localtime->mon() + 1, localtime->mday(), localtime->hour(), localtime->min(), localtime->sec(), $$ );
my @array;
my @words;
my @fileList;
my %SITE;
my %SITELIST;
my %SITESWITCH;
my %IP;
my %RUNNING;
my %DURATION;
my %PID;
my %GAWKPID;
my %LOGFILE;
my %STATUS;

my $output = "> /dev/null";
my $number_of_sites = 0;

my $TIMEOUT=30;
my $TIMEOUT_s=$TIMEOUT*60;

#our($opt_p);
use vars qw/ $opt_p $opt_t $opt_l $opt_v $opt_c $opt_s $opt_o $opt_d $opt_r $opt_i $opt_w $opt_g $opt_n $opt_a $opt_e $opt_f $opt_b $opt_2 $opt_z $opt_j /;

getopts('p:t:lc:v:i:sodrw:gna:efb2zj');

if ($opt_i) {
	if ( $opt_i =~ /^\d+(\.\d+)?$/ )
	{
		$OFFSET = $opt_i;
		$optionslist = sprintf("%s -i %s",$optionslist,$OFFSET);
	}
	else
	{
		die "ERROR: Argument to -i must be a number. Eg: \"-i 1\", \"-i 0.5\", etc.\n";
	}
}
if ($opt_w) {
	if ( $opt_w =~ /^[0-9]+(\.[0-9]+)*$/)
	{
		$waitInterval = $opt_w;
		if ($waitInterval == 0) { $waitInterval = 0.1;}
		$optionslist = sprintf("%s -w %s",$optionslist,$waitInterval);
	}
	else
	{
		die "ERROR: Argument to -w must be a number (integer or float, eg: 1.5)\n";
	}
}
if ($opt_p) {
	if ( $opt_p =~ /^\d+$/ ) {
		$MAX_PROCESSES = $opt_p;
		if ($MAX_P_OPTION > 0 && $MAX_PROCESSES > $MAX_P_OPTION) { $MAX_PROCESSES=$MAX_P_OPTION; }
		$setproc=1;
		$optionslist = sprintf("%s -p %s",$optionslist,$MAX_PROCESSES);
	} else {
		die "ERROR: Argument to -p must be a number.\n";
	}
}
if ($opt_s) { 
	$silent=1 ;
	$optionslist = sprintf("%s -s",$optionslist);
}
if ($opt_l) { 
	$moswitch = "-l" ; 
	$optionslist = sprintf("%s -l",$optionslist);
}
if ($opt_c) { 
	$moswitch = "$moswitch -c $opt_c" ; 
	$optionslist = sprintf("%s -c %s",$optionslist,$opt_c);
}
if ($opt_d) 
{ 
	$moswitch = "$moswitch -d" ; 
	if ($opt_d && $OFFSET==0) 
	{ 
		#avoid sql port collisions by starting each sql session at a 2 seconds interval . Also it is recommended to set sql_random to 3
		$OFFSET=2;
	} 	
	$optionslist = sprintf("%s -d",$optionslist);
}
if ($opt_r) { 
	$recursive=1;
	$optionslist = sprintf("%s -r",$optionslist);
}
#if ($opt_v) { $moswitch = "$moswitch -v $opt_v" ; }
if (defined $opt_t)
{
	if ( $opt_t =~ /^\d+$/ )
	{
		$TIMEOUT = $opt_t;
		$TIMEOUT_s = $TIMEOUT*60;
		$settimeout=1;
		$optionslist = sprintf("%s -t %s",$optionslist,$opt_t);
	}
	else
	{
		die "ERROR: Argument to -t must be a number.\n";
	}
}
if ($opt_g) { 
	$gz=1;
	$optionslist = sprintf("%s -g",$optionslist);
}
if ($opt_o) {
	$optionslist = sprintf("%s -o",$optionslist);
}
if ($opt_v) {
	$optionslist = sprintf("%s -v %s",$optionslist,$opt_v);
}
if ($opt_n) {
	$moswitch = "$moswitch -n" ; 
	$optionslist = sprintf("%s -n",$optionslist);
}
if ($opt_a) {
	$moswitch = "$moswitch -a $opt_a" ; 
	$optionslist = sprintf("%s -a %s",$optionslist,$opt_a);
}
if ($opt_e) {
	$moswitch = "$moswitch -e" ; 
	$optionslist = sprintf("%s -e",$optionslist);
}
if ($opt_f) {
	$moswitch = "$moswitch -f" ; 
	$optionslist = sprintf("%s -f",$optionslist);
}
if ($opt_z) {
	$moswitch = "$moswitch -z" ; 
	$optionslist = sprintf("%s -z",$optionslist);
}
if ($opt_j) {
	$moswitch = "$moswitch -j" ; 
	$optionslist = sprintf("%s -j",$optionslist);
}


my $DATE  = sprintf( "%04d-%02d-%02d", localtime->year() + 1900, localtime->mon() + 1, localtime->mday() );
my $TIME  = sprintf("%02d-%02d", localtime->hour(), localtime->min());
my $TIME2 = sprintf("%02d%02d%02d", localtime->hour(), localtime->min(), localtime->sec());

if ( $#ARGV != 1 && $#ARGV != 2 )
{
	print_usage();
	exit 1;
}

if (-d $ARGV[0])
{
	if (-r $ARGV[0])
	{
		$sitedir = $ARGV[0];
		$sitedir =~ s/[\/]+$//;
		$sitefilename = "offline_" . mybasename($sitedir);
		$moswitch = sprintf("%s -w %s/%s/%s", $moswitch, $DATE, $sitefilename, $TIME2 ) ;
		opendir(DirHandle, $sitedir) || die "Error in opening directory $sitedir\n";
		if ($recursive == 1)
		{
			closedir(DirHandle);
			@fileList = `find $sitedir -type f`;
			foreach my $dumpfile (@fileList) {
				chomp $dumpfile;
				$dumpfile =~ s/\n//;
				if ($dumpfile =~ /[ \t]/) {
					warn "WARNING: skipping file $dumpfile, spaces in filename not supported\n";
					next;
				}
				$number_of_sites++;
				$SITELIST{$number_of_sites} = $dumpfile;
				$SITE{$dumpfile} = $dumpfile;
				$SITESWITCH{$dumpfile} = "$moswitch";
				if ($opt_v) { $SITESWITCH{$dumpfile} ="$SITESWITCH{$dumpfile} -v $opt_v" ;}
				#print "$dumpfile $SITESWITCH{$dumpfile}\n";
			}
		}
		else
		{
			while( ($dumpfile = readdir(DirHandle))){
				$dumpfile = $sitedir . "/" . $dumpfile;
				if ($dumpfile =~ /[ \t]/) {
					warn "WARNING: skipping file $dumpfile, spaces in filename not supported\n";
					next;
				}
				if (-f $dumpfile && -r $dumpfile) 
				{ 
					$number_of_sites++;
					$SITELIST{$number_of_sites} = $dumpfile;
					$SITE{$dumpfile} = $dumpfile;
					$SITESWITCH{$dumpfile} = "$moswitch";
					if ($opt_v) { $SITESWITCH{$dumpfile} ="$SITESWITCH{$dumpfile} -v $opt_v" ;}
					#print "$dumpfile $SITESWITCH{$dumpfile}\n";
				}
			}
			closedir(DirHandle);
		}
	}
	else
	{
		warn "ERROR: cannot read directory $ARGV[0] \n";
		exit 1;
	}
		
}
elsif (-f $ARGV[0] && -r $ARGV[0])
{
	$sitefile     = $ARGV[0];
	$sitefilename = basename( $ARGV[0] );
	$moswitch = sprintf("%s -w %s/%s/%s", $moswitch, $DATE, $sitefilename, $TIME2 ) ; 
	my $sitefile_fh = new FileHandle $sitefile, "r";
	if ( !defined $sitefile_fh ) { die "Error: $sitefile: $!\n"; }
	while ( my $line = $sitefile_fh->getline )
	{
		$line =~ s/\r|^\s+|\&//g;
		chomp $line;
		#if ($line =~ /(\S+)/ && $1 !~ "#") { $SITE{$1} = $1; }
		@words = split(/\s+/,$line);
		if ($line =~ /\S+/ && $words[0] !~ /#/ && $words[0] !~ /^-/) 
		{
			if (defined $SITE{$words[0]}) { next ; }  #to avoid duplicates
			$number_of_sites++;
			$SITELIST{$number_of_sites} = $words[0];
			$SITE{$words[0]} = $words[0] ;
			$SITESWITCH{$words[0]} = "";
			if ($opt_v) { $SITESWITCH{$words[0]} = $opt_v ;}
			if ($line =~ /\S+\s+\S+/ && $words[1] =~ "=" && $words[1] !~ /#/)   { $SITESWITCH{$words[0]} = "$SITESWITCH{$words[0]},$words[1]" ; }
			if ($SITESWITCH{$words[0]} =~ /\S+/) {$SITESWITCH{$words[0]} = "-v $SITESWITCH{$words[0]}";}
			$SITESWITCH{$words[0]} = "$moswitch $SITESWITCH{$words[0]}";
			#print "DEBUG: N=$number_of_sites,SITE=$words[0],SWITCH=$SITESWITCH{$words[0]}\n";
			if ($MAX_NODES > 0 && $number_of_sites >= $MAX_NODES) { last ; }
		}
	}
	undef $sitefile_fh;
}
elsif ($ARGV[0] !~ /\//)
{
	$sitelist = $ARGV[0];
	$sitefilename = $sitelist;
	if ($sitefilename =~ /^[^,]+,/) { $sitefilename =~ s/,.*$/_and_more/; }
	elsif ($sitefilename =~ /^[0-9]+\.[0-9]+\.[0-9]+\.\*$/) { $sitefilename =~ s/\*$/2-254/; }
	elsif ($sitefilename =~ /^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+\.\*$/) { $sitefilename =~ s/\*$/2-254/; }
	elsif ($sitefilename =~ /^[0-9]+\.[0-9]+\.\*\.\*$/) { $sitefilename =~ s/\*\.\*$/0-255.2-254/; }
	$moswitch = sprintf("%s -w %s/%s/%s", $moswitch, $DATE, $sitefilename, $TIME2 ) ;
	my $Last = (@array = split(/,/, $sitelist));
	for (my $i = 0; $i < $Last; $i++) {
		if ($array[$i] !~ /\w/) { next; }
		if (defined $SITE{$array[$i]}) { next; }
		if ($array[$i] =~ /^[0-9]+\.[0-9]+\.[0-9]+\.\*$/)
		{
			$array[$i] =~ s/\*$//;
			for (my $j = 2; $j <= 254 ; $j++) {
				$number_of_sites++;
				my $thesite = $array[$i] . $j;
				$SITELIST{$number_of_sites} = $thesite ;
				$SITE{$thesite} = $thesite;
				$SITESWITCH{$thesite} = "$moswitch";
				if ($opt_v) { $SITESWITCH{$thesite} ="$SITESWITCH{$thesite} -v $opt_v" ;}
			}
		}
		elsif ($array[$i] =~ /^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+\.\*$/)
		{
			my @parts = split(/\.|-/,$array[$i]);
			$array[$i] =~ s/[0-9]+-[0-9]+\.\*$//;
			for (my $z = $parts[2]; $z <= $parts[3] ; $z++) {
				for (my $j = 2; $j <= 254 ; $j++) {
					$number_of_sites++;
					my $thesite = $array[$i] . $z . "." . $j;
					$SITELIST{$number_of_sites} = $thesite ;
					$SITE{$thesite} = $thesite;
					$SITESWITCH{$thesite} = "$moswitch";
					if ($opt_v) { $SITESWITCH{$thesite} ="$SITESWITCH{$thesite} -v $opt_v" ;}
				}
			}
		}
		elsif ($array[$i] =~ /^[0-9]+\.[0-9]+\.\*\.\*$/)
		{
			$array[$i] =~ s/\*\.\*$//;
			for (my $z = 0; $z <= 255 ; $z++) {
				for (my $j = 2; $j <= 254 ; $j++) {
					$number_of_sites++;
					my $thesite = $array[$i] . $z . "." . $j;
					$SITELIST{$number_of_sites} = $thesite ;
					$SITE{$thesite} = $thesite;
					$SITESWITCH{$thesite} = "$moswitch";
					if ($opt_v) { $SITESWITCH{$thesite} ="$SITESWITCH{$thesite} -v $opt_v" ;}
				}
			}
		}
		else 
		{
			$number_of_sites++;
			$SITELIST{$number_of_sites} = $array[$i];
			$SITE{$array[$i]} = $array[$i];
			$SITESWITCH{$array[$i]} = "$moswitch";
			if ($opt_v) { $SITESWITCH{$array[$i]} ="$SITESWITCH{$array[$i]} -v $opt_v" ;}
		}
		#print "DEBUG: N=$number_of_sites,SITE=$array[$i],SWITCH=$SITESWITCH{$array[$i]}\n";
		if ($MAX_NODES > 0 && $number_of_sites >= $MAX_NODES) { last ; }
		
	}
}
else
{
	warn "ERROR: cannot read file $ARGV[0] \n";
	exit 1;
}

if ( -f $ARGV[1] && -r $ARGV[1] ) { $commandfile = $ARGV[1]; }
elsif ( -d $ARGV[1] && -r $ARGV[1] ) { $commanddir = $ARGV[1]; }
else { $command = $ARGV[1]; }

if ( $#ARGV == 2 )
{
	if ( -d $ARGV[2] ) { $logdir = $ARGV[2]; }
	elsif ( mymkpath($ARGV[2]) ) { $logdir = $ARGV[2]; }
	else { die "ERROR: cannot find or create log directory: $ARGV[2]\n"; }
}
else
{
	$logdir = sprintf( "%s/%s/%s/%s", $logdir, $DATE, $sitefilename, $TIME );
	$logdir =~ s/:/./g;
	if ( !-d $logdir ) { if ( !mymkpath($logdir) ) { die "ERROR: $logdir: $!\n"; } }
}

my $number_in_queue = $number_of_sites;
my $number_started  = 0;
my $number_finished = 0;
my $uname = `uname`;
if ((! $opt_2) && $uname =~ /^SunOS/) { $opt_b = 1;}

if ($opt_b && $moshelldir =~ /^\/+opt\/+ericsson\/+amos\/+moshell/ && $uname =~ /^SunOS/ && $sitefile =~ /\w/)
{
	system("rm -f $amostempfile 2>/dev/null");
	system("touch $amostempfile");
	chmod(0666,$amostempfile);
	if (-w $amostempfile)
	{
		print "Running amos request broker...";
		my $amossystem = sprintf( "%s -x %s >> %s", $monode, $sitefile, $amostempfile );
		my $amosres = system($amossystem);
		if ($amosres != 0) { die "\nAmos request broker failed or aborted. Exiting...\n"; }
		print "Done\n";
		$b_option = " -y $amostempfile";
		$optionslist = sprintf("%s -b",$optionslist);
	}
}

open (RESULTFILE, ">$logdir/mobatch_result.txt") or die "unable to open $logdir/mobatch_result.txt $!";

if ($silent==0) 
{ 
	my $script_command = sprintf("%s%s",$0,$optionslist);
	foreach (@ARGV) { $script_command .= /\s/ ?   " \'" . $_ . "\'" :           " "   . $_; }
	customPrint("\nFull command = " . $script_command . "\n");
}
if ($silent==0 && $sitefile =~ /\w/)    { customPrint("Sitefile = $sitefile. Number of sites = $number_of_sites\n"); }
elsif ($silent==0 && $sitedir =~ /\w/)    { customPrint("Sitedir = $sitedir. Number of dumps = $number_of_sites\n"); }
elsif ($silent==0 && $sitelist =~ /\w/) { customPrint("Sitelist = $sitelist. Number of sites = $number_of_sites\n"); }
if ($commandfile !~ /^$/) { if ($silent==0) { customPrint("Command file = $commandfile\n"); } }
elsif ($commanddir !~ /^$/) { if ($silent==0) { customPrint("Command directory = $commanddir\n") ; } }
elsif ($command !~ /^$/)  { if ($silent==0) { customPrint("Command = $command\n"); } }
if ($TIMEOUT) {
	if ($silent==0) { customPrint("Timeout set to $TIMEOUT minutes"); }
	if ($settimeout != 1) { if ($silent==0) { customPrint(" (default value)")}; }
	if ($silent==0) { customPrint("\n"); }
}
else
{
	if ($silent==0) { customPrint("Timeout disabled.\n"); }
}
if ($OFFSET > 0 && $silent==0) { customPrint("Offset set to $OFFSET seconds\n"); }
if ($waitInterval > 0 && $silent==0) { customPrint("Wait interval set to $waitInterval seconds\n"); }
if ($silent==0) { customPrint("Maximum number of parallel processes set to $MAX_PROCESSES"); }
if ($setproc != 1) { if ($silent==0) { customPrint(" (default value)") }; }
if ($silent==0) { customPrint("\n"); }
if ($moswitch =~ "-l") { if ($silent==0) { customPrint("Running in light mode.\n"); } }
if ($silent==0) { customPrint("\nLogfiles stored in $logdir\n"); }

my $pm = new Parallel::ForkManager($MAX_PROCESSES);

if ($silent == 0) { customPrint("\n###########################################################################################\n");}
if ($silent == 0) { customPrint(sprintf( "%s\n", ">>>>> STARTED (pid)"));                                                          }
if ($silent == 0) { customPrint(sprintf( "%-34s", "** FINISHED (result)"));                                                        }
if ($silent == 0) { customPrint(sprintf( "%s\n", "STATUS (started, finished, queue, running, sites running)"));                    }
if ($silent == 0) { customPrint("###########################################################################################\n");  }

$pm->run_on_finish
(
	sub
	{
		my ( $pid, $exit_code, $ident ) = @_;
		#print "DEBUGFINISH: ** Finished $ident with exit code: $exit_code\n";
		if ( !defined $STATUS{$ident} )
		{
			my $fh = new FileHandle $LOGFILE{$ident}, "r";
			seek($fh, -1000, SEEK_END);
			if (defined $fh)
			{
				while ( my $line = $fh->getline )
				{
					#print "$line";
					if ( $line =~ /Bye\.\.\./ ) 
					{ 
						if (defined $STATUS{$ident} && $STATUS{$ident} eq 'WrongPw') { $STATUS{$ident} = "OK - WrongPw";}
						else { $STATUS{$ident} = "OK"; }
					}
					elsif ( $line =~ /!!!!Wrong Password/ ) { $STATUS{$ident} = "WrongPw"; }
					elsif ( $line =~ /Checking ip contact...Not OK|Fetching IOR file...Failed!|Cannot connect to MO service, exiting...|Moshell version too old, exiting.../)
					{
						$STATUS{$ident} = "no contact";
					}
					elsif ( $line =~ /AMOS error, exiting.../)
					{
						$STATUS{$ident} = "amos error";
					}
					elsif ( $line =~ /Not enough free disk space, exiting.../)
					{
						$STATUS{$ident} = "low disk space";
					}
					elsif ( $line =~ /Not enough free RAM, exiting.../)
					{
						$STATUS{$ident} = "low RAM";
					}
					elsif ( $line =~ /Too many sessions started by this user, exiting.../)
					{
						$STATUS{$ident} = "too many sessions";
					}
					elsif ( $line =~ /Too many sessions started on this workstation, exiting.../)
					{
						$STATUS{$ident} = "too many sessions";
					}
				}
				undef $fh;       # automatically closes the file
			}
		}
		if (! defined $STATUS{$ident} ) { $STATUS{$ident} = "Fail" ; }
		if ($silent == 0) {customPrint(sprintf("%-34s","** " . mybasename($ident) . " ($STATUS{$ident})"));}
		delete $RUNNING{$ident};
		$number_finished++;
		my $count = scalar( keys %RUNNING );
		if ($silent == 0) {customPrint(sprintf("%2ss %2sf %2sq %2sr: ",$number_started,$number_finished,$number_in_queue,$count));}
		foreach my $site ( sort keys %RUNNING )
		{
			if ($silent == 0) {customPrint( mybasename($site) . " ");}
		}
		if ($silent == 0) {customPrint("\n");}
		if ($gz == 1) { system(sprintf("gzip -f %s",$LOGFILE{$ident}));}
	}
);

$pm->run_on_start
(
	sub
	{
		my ( $pid, $ident ) = @_;
		if ($silent == 0) {customPrint(sprintf("%-34s",">>>>> " . mybasename($ident) . " ($pid)"));}
		$RUNNING{$ident} = time();
		$PID{$ident} = $pid;
		$GAWKPID{$ident} = '';
		$number_started++;
		$number_in_queue--;
		my $count1 = scalar( keys %RUNNING );
		$oldcount = $count1;
		if ($silent == 0) {customPrint(sprintf("%2ss %2sf %2sq %2sr: ",$number_started,$number_finished,$number_in_queue,$count1));}
		foreach my $site ( sort keys %RUNNING ) { if ($silent == 0) {customPrint( mybasename($site) . " ");} }
		if ($silent == 0) {customPrint("\n");}
	}
);

$pm->run_on_wait
(
	#my $count = scalar( keys %RUNNING );
	#print "DEBUGWAIT: $count\n";
	sub
	{
		#my $count = scalar( keys %RUNNING );
		#print "DEBUGWAIT: $count processes running. $number_started started. $number_finished finished. $number_in_queue in queue.\n";
		my $now = time();
		my $site;
		my $count = scalar( keys %RUNNING );
		#print "DEBUGWAIT: $count\n";
		if ($count == 0) { $pm->{on_wait_period}=0;}
		else { $pm->{on_wait_period}=$waitInterval;}
		foreach $site ( keys %RUNNING )
		{
			if ($GAWKPID{$site} eq '') 
			{
				my $fh = new FileHandle $LOGFILE{$site}, "r";
				if (defined $fh)
				{
					while ( my $line = $fh->getline )
					{
						if ( $line =~ / stopfile=\/tmp\/[0-9]+$/)
						{
							$line =~ s/^.* stopfile=.tmp.|\n//g;
							$GAWKPID{$site} = $line ;
							#print "$site,$GAWKPID{$site}\n";
							last;
						}
					}
					undef $fh;       # automatically closes the file
				}
			}
			else
			{
				if (kill(0,$GAWKPID{$site})==0) 
				{ 
					#the process is not there anymore
					delete $RUNNING{$site};
					next;
				}
			}
				

			####my ($Dd,$Dh,$Dm,$Ds) = Delta_DHMS(@{$RUNNING{$site}}, @Now);
			#print "$site running for $Dm minutes $Ds seconds.\n";
			$DURATION{$site} = $now - $RUNNING{$site};
			if ($TIMEOUT_s && $DURATION{$site} >= $TIMEOUT_s)
			{
				warn "ERROR: $site: Timeout. Killing process $PID{$site}. \n";
				#For linux/solaris:
				my $system;
				$system = sprintf( "%s -k %s 2>&1 >/dev/null", $pstool, $PID{$site});
				system($system);
				#For cygwin:
				if ($GAWKPID{$site} =~ /^[0-9]+$/)      { kill 15, $GAWKPID{$site};      }
				if ($PID{$site}     =~ /^[0-9]+$/)      { kill 15, $PID{$site};          }
				
				delete $RUNNING{$site};
				#$number_finished++;
				$STATUS{$site} = "Timeout";
			}
		}
		#$oldcount = scalar( keys %RUNNING );
		#print "\n\n";
	},
	$waitInterval
);

if($opt_o && $number_of_sites <= 1){ $output = "" }

for (my $i=1 ; $i<=$number_of_sites ; $i++)
{
	my $site = $SITELIST{$i};
	$LOGFILE{$site} = sprintf( "%s/%s.log", $logdir, ($recursive == 1 ? mychangename($site) : mybasename($site) ) );
	
	#if($opt_o && $number_of_sites > 1){ $output = sprintf("|  %s/gawk \' { print \"%s:\",\$0} \'", $moshelldir, $site); }
	if($opt_o && $number_of_sites > 1){ $output = sprintf("| %s","perl -nle \'print \"$site: \$_\";\'"); }	
	
	my $system;

	if ($SITESWITCH{$site} =~ /(^| )-v [^']/) { 
		$SITESWITCH{$site} =~ s/(^| )-v / -v '/; 
		$SITESWITCH{$site} = $SITESWITCH{$site} . "'" ;
	}

	if ($commanddir) 
	{
		if (-r sprintf( "%s/%s.cmd", $commanddir, $site ))
		{
			$commandfile = sprintf( "%s/%s.cmd", $commanddir, $site );
		}
		else 
		{
			$commandfile = sprintf( "%s/%s.mos", $commanddir, $site );
		}
	}
	if ($command) 
	{
		$system = sprintf( "%s %s%s %s \'%s\' %s %s", $monode, $SITESWITCH{$site}, $b_option, $SITE{$site}, $command, $LOGFILE{$site}, $output );
	} 
	else 
	{
		if ( -r $commandfile ) 
		{
			$system = sprintf( "%s %s%s %s %s %s %s", $monode, $SITESWITCH{$site}, $b_option, $SITE{$site}, $commandfile, $LOGFILE{$site},$output );
		} 
		else 
		{
			warn "ERROR: $site: Command file $commandfile: $!\n";
			$STATUS{$site} = "command file not found.";
			next;
		}
	}
	#avoid collisions by starting each moshell session at a interval
	if ($OFFSET > 0) 
	{
		if ($OFFSET =~ /\./) { select(undef, undef, undef, $OFFSET); }
		else { sleep $OFFSET; }
	}
	$pm->start($site) and next;

	mymkfile($LOGFILE{$site});
	
	#print "DEBUG: $system\n";
	system($system);

	$pm->finish;    # Terminates the child process
	
}

#print "Waiting for Children...\n";
$pm->wait_all_children;

if ($silent == 0) {customPrint("###########################################################################################\n");}
if ($silent == 0) {customPrint("\n");}
if ($silent == 0)
{
	for(my $i=1; $i<=$number_of_sites; $i++){
		my $site = $SITELIST{$i};
		if ($DURATION{$site})
		{
			if ($STATUS{$site} eq 'OK')
			{
				$totDuration += $DURATION{$site};
				$totSitesOk ++;
			}
			$Dm = int($DURATION{$site}/60);
			$Ds = $DURATION{$site} % 60;
			$DURATION{$site}=sprintf("%sm%ss",$Dm,$Ds);
		}
		else { $DURATION{$site} = "?m?s";}
		my $resoutput = sprintf( "%-12s  %-6s  %s\n", $STATUS{$site} , $DURATION{$site}, mybasename($site));
		customPrint("$resoutput");
	}
	my $mobatchduration = time() - $mobatchstart;
	customPrint(sprintf("\nTotal mobatch duration: %sm%ss\n",int($mobatchduration/60),$mobatchduration % 60));
	if ($totSitesOk > 0)
	{
		$avgDuration = $totDuration / $totSitesOk;
		customPrint(sprintf("Average duration per site with status OK: %sm%ss\n",int($avgDuration/60),int($avgDuration % 60)));
	}
	customPrint("Logfiles stored in $logdir\n"); 
	close(RESULTFILE);
}
if (-w $amostempfile) { system("rm -f $amostempfile 2>/dev/null");}
exit;

###############################################################################
sub customPrint
{
	open (RESULTFILE, ">>$logdir/mobatch_result.txt");
	my $tmp = $_[0];
	print "$tmp";
	print RESULTFILE "$tmp";
	close(RESULTFILE);
}
sub mybasename
{
	my $tmp = $_[0]; 
	$tmp =~ s/^.*\///;
	$tmp =~ s/:/./g;
	return $tmp;
}
sub mychangename
{
	my $tmp =$_[0];
	$tmp =~ s/$sitedir//;
	$tmp =~ s/\//%/g;
	$tmp =~ s/^%+//;
	return $tmp;
}
sub mymkpath
{
	my $mkpathresult = mkpath($_[0]);
	if ($moshelldir =~ "/moshellroot") { chmod(0777,$_[0]) ; }
	return $mkpathresult;
}
sub mymkfile
{
	if ($moshelldir =~ "/moshellroot" || $moshelldir =~ "/opt/ericsson/amos/moshell")
	{
		open(__TEMPLOG, ">$_[0]") ;
		close(__TEMPLOG);
		chmod(0666,$_[0]);
	}
}

sub readFromFile
{
	my $filename = shift;
	my @inputSplit;
	open(FILE, $filename) or print ("Could not read from $filename");
	while (<FILE>)
	{
		@inputSplit = split(' ', $_);
		next unless defined $inputSplit[0];
		@inputSplit = split('=', $inputSplit[0]);
		next if ($#inputSplit != 1);
		if ($inputSplit[0] eq "mobatch_max_p_option") {
			$MAX_P_OPTION = $inputSplit[1];
		}
		if ($inputSplit[0] eq "mobatch_max_nodes") {
			$MAX_NODES = $inputSplit[1];
		}
		if ($inputSplit[0] eq "mobatch_max_processes") {
			$MAX_PROCESSES = $inputSplit[1];
		}
		if ($inputSplit[0] eq "mobatch_i_option") {
			$OFFSET = $inputSplit[1];
		}
	}
	close FILE;
}

sub print_usage
{

	print <<EOF;
Usage:  mobatch [options] <sitefile>|<sitelist>|<dumpdir>  <command(s)>|<commandfile>|<commanddir> [logdirectory]
	
	Purpose: To send moshell commands to several nodes in parallel.

	Arguments:		
	 - The first argument is the sitefile, sitelist, or dumpdir.
	The sitefile is a file containing the list of sites to connect to. Each line in the sitefile contains:
	          * the IP/DNS addresses and/or site names whose IP address are defined in the IP database.
	          * optionally: the uservariables/scriptingvariables to input with -v option.
	See example of sitefile and ipdatabase in: moshell/examples/mobatch_files/
	If using the sitelist, the sites are listed on the command line and separated by commas. 
	Alternatively it possible to specify a whole subnet of nodes with the IP address syntax x.y.z.* (using wildcard on the last part of the IP address)
	The dumpdir is a directory containing:
	          * MO dumps (format: <kget.log.gz> or <modump.zip>)
	          * or CVs/dbdat/dbdump files (format: <cv.zip> or <db.dat> or <dbdump.zip>) -> only supported with option "-d"
	 - The second argument is the commmands or commandfile.
	See example of commands below and commandfile in moshell/examples/mobatch_files
	If a directory is given, then a different commandfile will be used for each node:
		the name of each commandfile should be <node-name>.cmd or <node-name>.mos
		the <node-name> should be the same as given in the sitefile.
		example: node-name is rbs602 ==> commandfile should be rbs602.cmd or rbs602.mos
	 - The third argument (logdirectory) is optional. If no logdirectory is specified, a default one will be used.
	
	 Options:
         -t <minutes>       Specify the number of minutes before timing out. Set to 0 for no timeout (default=30 minutes)
	 -p <processes>     Specify the maximum number of moshell sessions that will run in parallel (default=10 parallel sessions)
	 -i <seconds>       Specify the interval in seconds between spawning of each moshell session (default=0 seconds). 
	                    Can be a decimal number. Eg: 0.5 for half a second.
	 -w <seconds>       Specify the interval in seconds between the checks on running sessions (default= 2.5 seconds)
	 -v <userVariables> Specify moshell uservariables. Type "moshell" on its own for more info about this option.
	 -o                 Print output of every moshell session both to screen and to logfile.
	 -s                 Silent. No output is printed.
	 -d                 Run moshell in sql mode. Only applicable when first argument is a directory ("dumpdir")
	 -r                 Recursive folder search. Only applicable when first argument is a directory ("dumpdir"). 
	 -g                 Gzip logfiles after completion
	 -n                 Skip the IP connectivity check at moshell startup
	 -e                 Skip sourcing of the global moshellrc file (moshell/jarxml/moshellrc)
	 -f                 Skip sourcing of the user's moshellrc file (~/.moshellrc)
	 -a <rcfile>        Source an additional moshellrc file
	 -b                 Use a common amos request broker for all spawned amos sessions in order to reduce Corba TCP ports usage between UAS and Admin server in OSSRC. Note: this option is default in OSSRC and not applicable for ENM.
	 -2                 Don't use a common amos request broker. This option is only applicable for amosbatch in OSSRC.
	 -z                 Run moshell in YANG mode.
	 -j                 For faster startup in offline mode to a dcgm.
	
	 Examples:
		mobatch -p 15 -t 60 ~/sitefiles/victoria-sites 'lt all ; get'
		mobatch 10.1.128.10 ./cmdfiles/kget.mos
		mobatch 10.1.128.10,rnc34,rbs10,mgw1.ericsson.se ./cmdfiles/kget.mos
		mobatch ./sitefiles/all-rbs.txt ./cmdfiles/do_healthCheck.mos
		mobatch -p 5 -t 1 ./sitefiles/all-rbs.txt 'cv cu ; rbs'
		mobatch ./sitefiles/all-rnc.txt 'lt ^utrancell ; st cell'
		mobatch ./sitefiles/all-bsc.txt ~/bsc-commandfiles/
		mobatch -v security_method=2,sa_credential=~/sam.pbe,sa_password=oemas -p 20 ./all_mgw.txt 'hc'
		mobatch -v ip_database=~/utran_network/ipdatabase ~/utran_network/ipdatabase 'lt all ; get '
		mobatch modumpfolder/ 'str;std;sti;stv;inv'
		mobatch -r modumpfolder/ 'str;std;sti;stv;inv'
		mobatch -dr dbdatfolder/ dbc  

EOF
}
