#! /bin/sh 
#  This sample script is a wrapper for running poe (SP2) jobs under lsbatch.
#  Submit job by saying "bsub [otheroptions] -n k poejob command line",
#  where k is the number of hosts to use; pp2job is the name of this script.
#
#  Note the variable LSB_HOSTS is assigned by lsbatch system when this script
#  is started by lsbatch.

$LSB_TRAPSIGS 

#
# runit.tmpl module will be included upon installation
#
# Do not remove the following two lines!
#RUNIT_MODULE_BEGIN#
#
# runit_unix()
#
# This func takes the global var $COMMANDLINE,
# executes the command and sets the $exstat with the exit
# status of the command
#
runit_unix() {
    eval $COMMANDLINE
    exstat=$?
}

#
# runit_mpl()
#
# This func takes the global var $COMMANDLINE
# and then sets the var $exstat to 
#    (1) first value of $REQUEUE_EXIT_VALUES   if (poe couln't allocate nodes)
#    (2) exit status of the command            otherwise
#
runit_mpl() {
    POE_ERRFILE="/tmp/poejob.$$"
    tmpfiles="$tmpfiles $POE_ERRFILE"

    eval poe $COMMANDLINE 2>$POE_ERRFILE
    exstat=$?

    if [ -n "`grep 't allocate nodes for parallel' $POE_ERRFILE`" ] ; then
        if [ x"$LSB_EXIT_REQUEUE" = "x" ] ; then
	    true
        else 
	    lsf_requeue=1
	    # take the first value of the REQUEUE_EXIT_VALUES
	    if [ x"REQUEUE_EXIT_VALUES" = x ] ; then
	        echo "poejob: fatal error REQUEUE_EXIT_VALUES not set!" 1>&2
	    else
		exstat=`echo $REQUEUE_EXIT_VALUES|awk {print $1}`
            fi
        fi
    fi
}
#RUNIT_MODULE_END#

#
# Temporary files to be removed before exit
#

tmpfiles=

#Set the host.list in the current directory 
HOSTFILE=./host.list
HOSTTMPFILE=./host.list.tmp

CMD=$1
COMMANDLINE="$@"
POE_ERRFILE="/tmp/poejob.$$"

#Save current host.list file to be resotred after job is run */
more $HOSTFILE > $HOSTTMPFILE
rm -f $HOSTFILE

#Generate hostfile
nhosts=0
for word in $LSB_HOSTS 
do
echo $word >> $HOSTFILE
nhosts=`expr $nhosts + 1` 
done

#Set poe specific environment variables 
MP_PROCS=$nhosts; export MP_PROCS

# Run poe job and save exit status
runit_mpl

#Restore host.list file to old host.list file 
cp $HOSTTMPFILE $HOSTFILE

if [ x"$tmpfiles" != "x" ] ; then 
    rm -f $tmpfiles
fi

exit $exstat



