#!/bin/sh

#	ESX Server Support Script
#
#	Collects various configuration and log files
#	for use when troubleshooting the VMware ESX
#	Server
#

#	Function: usage prints how to use this script

DATE=$(date +"%F--%H.%M")
TARFILE=esx-$DATE.$$.tgz
VER=1.30
OUTPUT_DIR=vm-support-$(hostname -s)-$DATE.$$
BACKGROUND_CMD_WAIT=300
COS=$(if [ $(uname) = "Linux" ]; then echo 1; else echo 0; fi)
COREDUMP_DIR="/root"

usage() {
	echo ""
	echo "Usage: $0 [-n] [-N] [-a] [-s] [-S] [-d duration_in_seconds] [-i interval] [-k] [-x] [-X wid] [-q] [-w] [-h] [-f] [-Z wid] [-t wid]"
	echo "  -n causes no cores to be tar'ed up"
	echo "  -N causes no service console cores to be tar'ed up"
	echo "  -a causes all cores to be tar'ed up - even those from previous"
        echo "     runnings of this script"
        echo "  -c gather schedtrace snapshots (only if performance snapshots are enabled)"
        echo "  -s take performance snapshots in addition to other data"
        echo "  -S take only performance snapshots"
        echo "  -d<s> set performance snapshot duration to <s> seconds [default $snap_duration]"
        echo "  -i<s> sleep <s> seconds between snapshots [default autodetect]"
        echo "  -k gather vmware-stats*.log files of each VM" 
        echo "  -x lists wids for running VMs"
        echo "  -X <wid> grab debug info for a hung VM"
        echo "  -q runs in quiet mode"
        echo "  -w <dir> sets the working directory used for the output files"
	echo "  -f allows you to force vm-support to use a VMFS working directory"
	echo "  -l print list of files being collected"
	echo "  -Z <wid> suspends a VM and grabs debug info"
        echo "  -r causes all vmfs3 volumes' volume headers to be tar'ed up"
	echo "  -t <wgid> takes a scsi command trace for I/O from a VM (specify world group id)"
        echo "      Note:- this option consumes a noticeable amount of cpu so enabling it "
	echo "             can negatively impact the performance of the system"
        echo "           - limited to one VM at a time"
        echo "           - trace contains NO customer sensitive data"
        echo "           - only information recorded is: "
        echo "              - serialnumber, timestamp, length of command"
        echo "              - command type, block number"
        echo "           - Therefore, actual data/payload of commands is not stored"
	echo "  -v prints out vm-support version"
	echo "  -h prints this usage statement"
        echo ""
        echo "See man page for more information on vm-support version $VER"
	exit
}

version() {
	echo -e "vm-support v$VER\n"
        exit
}


#
# The command log is for logging the vm-support messages and commands it runs.
# Used by both 'banner' and 'log' (generally just use one of those in the 
# rest of the script.)
#
# The commandlog is ignored until after the basic argument parsing is completed.
#
CMD_LOG=$OUTPUT_DIR/tmp/vm-support.$$.log
commandlog() {
    if [ -f $CMD_LOG ]; then
	echo "${@}" >> $CMD_LOG
    fi
}
commandloge() {
    if [ -f $CMD_LOG ]; then
	echo -e "${@}" >> $CMD_LOG
    fi
}


#	Function: banner prints any number of strings padded with 
#	newlines before and after.

banner() {
	echo
	for option in "$@"
	do
		echo $option
		commandlog "$option"
	done
	echo
}

#
# Log the input to a stream if VERBOSE output is enabled.
#
log() {
   if [ "$VERBOSE" = "1" ]; then
      echo $1
   fi
   commandlog "$1"
}

#
# Log the input to a stream if VERBOSE output is enabled. use
# echo -e instead of the standard echo.
#
loge() {
   if [ "$VERBOSE" = "1" ]; then
      echo -e $1
   fi
   commandloge "$1"
}

#
# Write log message to vmkernel log
vmklog() {
   if [ -w /proc/vmware/log ]; then
	echo -n "vm-support: ${@}" >> /proc/vmware/log
   fi
}


# Match expressions of the following form: key = "value"
# Usage: getConfigVal(<option>, <default value>, <config file>)
getConfigVal() {
   ret=`sed -ne "s/^[ \t]*$1[ \t]*=[ \t]*\"\(.*\)\".*/\1/p" $3 | tail -1`
   if [ "${ret}" = "" ]; then
      echo $2
   else
      echo $ret
   fi
}

UpdateSpinner() {
    # Only show the pretty spinner on a tty
    if [ -t 1 ]; then
        case $SPINNER in
		"|")
                        SPINNER="/"
                ;;

                "/")
                        SPINNER="-"
                ;;

                "-")
                        SPINNER="\\"
                ;;

                "\\")
                        SPINNER="|"
                ;;

                *)
                        SPINNER="|"
                ;;
        esac
        echo -en "\r$1: $SPINNER"
    fi
}

#
# Add all interesting files associated with a 
# VM into the tar. 
#
snarfVMFiles() {
   local VMDIRNAME=$1
   addtar "$VMDIRNAME/*.cfg" IGNORE_ERROR
   addtar "$VMDIRNAME/*.vmx" IGNORE_ERROR
   addtar "$VMDIRNAME/*.vmxf" IGNORE_ERROR
   addtar "$VMDIRNAME/vmware-[^a-z]*.log" IGNORE_ERROR
   addtar "$VMDIRNAME/vmware.log" IGNORE_ERROR
   addtar "$VMDIRNAME/*.vmsd" IGNORE_ERROR
      
   # Only add descriptor files, not the whole disk
   addtar "$VMDIRNAME/*.vmdk" 1024 IGNORE_ERROR
   addtar "$VMDIRNAME/*.dsk" 1024 IGNORE_ERROR

   # Add screenshots (useful for hung vm debugging)
   addtar "$VMDIRNAME/*.png" IGNORE_ERROR
   
   #Add vmware-stats*.log files (helpful for debugging monitor issue)
   if [ $vm_stats -eq 1 ]; then 
      addtar "$VMDIRNAME/vmware-stats*.log" IGNORE_ERROR
   fi

   #
   # If one of the above commands succeeded, then
   # add a list of all the VM's files.  
   #
   if [ -e "$OUTPUT_DIR/$VMDIRNAME" ]; then
      # You might think runcmd would work, but you're wrong.
      # (think spaces, and the fact that it would end up creating
      #  and then deleting files in every VM's directory)
      ls -l "$VMDIRNAME" > "$OUTPUT_DIR/$VMDIRNAME/filelist.$$.txt"
   fi

   if [ $no_cores -eq 0 ]; then
      addtar "$VMDIRNAME/vmware-core*"
      addtar "$VMDIRNAME/vmware64-core*"
      addtar "$VMDIRNAME/core*"                
      addtar "$VMDIRNAME/vmware-vmx-zdump*"
   fi
 
   # We assume the directory has been created by now
   if [ $suspend_hung_vm -eq 1 ]; then
      for FILE in $(ls -d "$VMDIRNAME"/*.vmss 2>/dev/null); do
         gzip -c "$FILE" > "$OUTPUT_DIR"/"$FILE".gz 2>/dev/null
      done
   fi

   #collect RDM lun mapping information of each VM
   grep  .vmdk "$VMDIRNAME"/*.vmx 2>/dev/null | awk -F \" '{ print $2 }' |
   while read VMDK; do
      ( cd  "$VMDIRNAME";
        echo "# vmkfstools -q $VMDK";
        vmkfstools -q "$VMDK" 2>&1 ) >> "$OUTPUT_DIR/$VMDIRNAME/vmdk-types.txt"
   done
        
}

#
# Add all log & core files for VMs managed by VPX into into the tar file.
# (note that we'll copy registered VM's twice.  This seems like a less
# evil than trying to parse the vm-list.)
snarfVirtualCenterFiles() {
   local vpxVMDir=$1
   if [ -d ${vpxVMDir} ];  then
      IFS=$'\n'
      for VMDIRNAME in `find $vpxVMDir -type d`; do
            unset IFS
            snarfVMFiles "$VMDIRNAME"
      done
      unset IFS
   fi
}

#	Function: addtar takes a single paramater, possibly containing wildcards,
#       and recursively copies the resulting files and directories into self 
#       contained output directory for later tar'ing
#
#	Working on copies could slow this down with VERY large files but:
#	1) We don't expect VERY large files
#	2) Since /proc files can be copied this preserves the tree without
#	   having to cat them all into a file.
#	3) tar barfs on open files like logs if it changes while it's tar'ing.
#          Copying file first makes sure tar doesn't complain
addtar() {
	FILES=$1
        if [ "$2" = "IGNORE_ERROR" ]; then
           ignoreError=$2
           SIZELIMIT=""
        else 
           SIZELIMIT=$2
           ignoreError=$3
        fi

	log "addtar $FILES"
	IFS=$'\n'
        for FILE in $(ls -d $FILES 2>/dev/null); do
           unset IFS
           if [ -n "$SIZELIMIT" ] ; then
              local fSize=`ls -l "$FILE" | awk '{print $5}'`
              # If $fSize is too large (>2147483647) $fSize -gt $SIZELIMIT
              # causes a "out of range exception". Thus compare the number
              # of digits first and avoid the second check. see PR 162904
              if [ `expr length "$fSize"` -gt `expr length "$SIZELIMIT"` ] ||
                 [ $fSize -gt $SIZELIMIT ]; then   
                 log "  File '$FILE' is too big ($fSize > $SIZELIMIT), skipping"
                 continue
              fi
           fi 
           log "  Copying $FILE to output directory"
	   if [ $no_cores -eq 0 ] && [ $all_cores -eq 0 ]; then
	       # chkeck if $FILE is $COREDUMP_DIR to copy only cores except old_cores
	       if [ "$FILE" = "$COREDUMP_DIR" ] || [ "$FILE" = "$COREDUMP_DIR/" ]; then
		   for f in $(ls $COREDUMP_DIR/ 2>/dev/null); do
		       if [ $f != "old_cores" ]; then
			   cp -pr --parents "$COREDUMP_DIR/$f" "$OUTPUT_DIR" 2>/dev/null
		       fi
		   done
	       else
		   cp -pr --parents "$FILE" "$OUTPUT_DIR" 2>/dev/null
	       fi
	   else
	       cp -pr --parents "$FILE" "$OUTPUT_DIR" 2>/dev/null
	   fi

	   # If a file from /proc does not copy, ignore - they're 
           # funny.  Otherwise, exit for failed copy.
           # 
           # Also allow callers to specify IGNORE_ERROR, if a file
           # might not be available (perhaps a locked vmfs file)
           rc=$?
           if [ $rc -ne 0 ] && [ "$ignoreError" != "IGNORE_ERROR" ]; then
              if [ "$COS" = "1" ]; then
                 grepExpr='^/proc'
              else
                 # XXX /etc (visorfs) contains special files (.#)
                 # these files cannot be opened for write because of their
                 # special meaning. 
                 # Therefore if vm-support output directory is 
                 # backed by visorfs the error could be because of the special
                 # files.
                 grepExpr='^/proc\|^/etc'
              fi
              echo $FILE | grep "$grepExpr" > /dev/null
              if [ $? != 0 ]; then
                 banner "Could not copy $FILE to tar area (err $rc)." \
                        "Have you run out of disk space?" 
                 if [ -d $OUTPUT_DIR ]; then
                    rm -rf $OUTPUT_DIR
                 fi
                 exit
              fi
           fi

           # Filter out ':'s for Windows -- this is here for vmhba names from
           # procfs which confuse winzip
           TARGET=$(basename "${FILE}" | /bin/awk '{ gsub(/:/, "_"); print $0; }')
           if [[ "$(basename "${FILE}")" != "${TARGET}" ]] ; then
              mv "${OUTPUT_DIR}/${FILE}" "${OUTPUT_DIR}/$(dirname "${FILE}")/${TARGET}" 2> /dev/null
           fi

           if [ $quiet_mode -eq 0 ]; then
	      UpdateSpinner "Preparing files"
           fi
        done
        unset IFS
}


#	Function: runcmd executes the command ($1) 
#	redirected to a file ($2) and then adds that 
#	file to the list of files to tar. 
#	It then deletes the temp file since addtar makes a copy in its own
#	self contained area.
runcmd() {
	cmd="$1"
	log="$2"

	log "Running command: $cmd"

	# if $cmd segfaults (see PR 105045 for an example), bash will print
	# a lovely "/usr/bin/vm-support: line 1338: 19032 Segmentation
	# fault" which is scary and inappropriate (if it said 'dmidecode'
	# we might let it pass).  So, to hide such errors we run the
	# command in a subshell append any shell output to the logfile.
	rm -f "$log"
	sh -c "$cmd >\"$log\" 2>&1" >>"$log" 2>&1
	rc=$?

	if [ $rc -ne 0 ]; then
	    if [ "$3" != "IGNORE_ERROR" ]; then
		if [ $rc -eq 127 ]; then
		    # 127 is command not found (usually)
		    banner "" "Error running $cmd (err $rc).  Assuming command not found." \
			"Continuing..."
		elif [ $rc -gt 128 ]; then
		    banner "" "Error running $cmd or writing to $log (err $rc)." \
			"Continuing..."
		else 
		    banner "" "Error running $cmd or writing to $log (err $rc)." \
                       "Do you have a full disk?" \
		       "Continuing..."
		fi
            fi
	fi

	if [ -e "$log" ]; then
		addtar "$log"
		if ! rm "$log"; then
			banner "" "Could not delete $log (err $rc)." "Continuing..." 
		fi
	fi
}


#	Function: runbackgroundcmd executes the command ($1) 
#	redirected to a file ($2) in the tar directory.
#	
#	The command will be run in the background. vm-support will wait
#	up to BACKGROUND_CMD_WAIT seconds at the end of its runtime for 
#	these commands to complete before giving up.
#
#	The command will be added to a list of commands so that a message
#	can be printed out at the end of vm-support indicating that we're 
#	pausing in the hopes that the following commands complete.
#
#	We also keep track of a list of PIDs so that vm-support doesn't 
#	wait longer than necessary for commands to complete.
#
#	Note that you should be careful to make sure your output file
#	location will be a valid path within OUTPUT_DIR.
runbackgroundcmd() {
	log "Running backgrounded command: $1"

	$1 > "$OUTPUT_DIR/$2" 2>&1 &
	pid=$!

	log "Backgrounded pid $pid for command: $1"

	BACKGROUND_CMD_LIST="${BACKGROUND_CMD_LIST}\n\n${1}";
	BACKGROUND_CMD_PIDS="${BACKGROUND_CMD_PIDS} $pid";
}


#
# Grab info about the running machine (current processes, open files, etc).
# This is as opposed to the persistent configuration state (like mount 
# points or RPMs installed, log files, etc).
#

snarf_running_state() {
   runcmd "date" "/tmp/date.$$.txt"

   local filter="\|/proc/kcore\|/proc/kmsg"

   # Do not expand the globs when creating the command.  We save the
   # current globignore just incase it is used in the future
   OLD_GLOBIGNORE=$GLOBIGNORE
   export GLOBIGNORE=*

   if [[ -f /etc/vmware/vm-support.conf ]] ; then
      filter="${filter}$(/bin/awk '/ignored-proc-nodes/                                \
                                   {                                                   \
                                      split($0, values, "=");                          \
                                      sub(/^[ ]*/, "", values[2]);                     \
                                                                                       \
                                      filters = gensub(/^\([ ]*"(.*)"[ ]*\)$/, "\\1",  \
                                                       "", values[2]);                 \
                                                                                       \
                                      split(filters, f, "\" \"");                      \
                                      for (i in f)                                     \
                                      {                                                \
                                         filter = filter "\\|" f[i]                    \
                                      }                                                \
                                                                                       \
                                      print filter;                                    \
                                   }' /etc/vmware/vm-support.conf)";
   fi

   # stdout redirected to /dev/null.  Some files come and go and confuse find.
   # Just send whatever works and don't scare user.
   for file in $(find /proc -type f 2> /dev/null | /bin/grep -v /proc/$$${filter}) ; do
      addtar "${file}"
   done

   export GLOBIGNORE=$OLD_GLOBIGNORE

   # Traverse the entire VSI tree, saving it to a file.
   runcmd "/usr/lib/vmware/bin/vsi_traverse -s" "/tmp/vsicache.$$"
   sleep 5
   runcmd "/usr/lib/vmware/bin/vsi_traverse -s" "/tmp/vsicache-5-seconds-later.$$"

   if [ "$COS" = "1" ]; then
	   runcmd "uptime" "/tmp/uptime.$$.txt"

	   runcmd "dmesg -s 262144" "/tmp/dmesg.$$.txt"

	   if [ -x /usr/sbin/lsof ]; then
	   	runcmd "lsof -l -n -P -b" "/tmp/lsof-normal-output.$$.txt"
	   	runcmd "lsof -i -n -P -b" "/tmp/lsof-network-output.$$.txt"
	   fi

	   runcmd "netstat -a -n -p" "/tmp/netstat-conn.$$.txt"
	   runcmd "netstat -s" "/tmp/netstat-stats.$$.txt"
	   runcmd "netstat -i -a" "/tmp/netstat-intf.$$.txt" 
	   runcmd "ps -auxwww" "/tmp/ps.$$.txt"
	   runcmd "pstree -ap" "/tmp/pstree.$$.txt"
	   runcmd "top -d 1 -n 10 -b" "/tmp/top.$$.txt"
   fi
}


#
# Get support information about hostd
#

snarf_hostd_state() {
   #	Parse /etc/vmware/hostd/vmInventory.xml and add the virtual 
   #	machine configuration and log files to the 
   #	list of files

   INVENTORY_LOCATION="/etc/vmware/hostd/vmInventory.xml"
   NEW_LOCATION=`sed -n -e "s/^\s*<vmInventory>\s*\(\.*\)/\1/g" -e "s/<\/vmInventory>\s*$//pg" /etc/vmware/hostd/config.xml`
   if [ -n "$NEW_LOCATION" ]; then
      INVENTORY_LOCATION=$NEW_LOCATION
   fi

   # Grab the inventory file itself, in case it is configured differently
   addtar $INVENTORY_LOCATION
   IFS=$'\n'
   for CFGNAME in $(sed -n -e "s/^\s*<vmxCfgPath>\s*\(\.*\)/\1/g" \
       -e "s/<\/vmxCfgPath>//pg" $INVENTORY_LOCATION); do
           unset IFS
	   VMDIRNAME=$(dirname "$CFGNAME")
           snarfVMFiles "$VMDIRNAME"
   done
   unset IFS

   #    grab logs and cores for vms which aren't here anymore
   #    (they won't be in the vm-list after vmotion'ing)
   #    The old default directory was /vpx/vms.  The new default
   #    directory is /home/vmware.  Try to grab both,
   #    since we don't know what version of virtual center is being used.

   vpxVMDir=`getConfigVal "serverd.vpx.defaultVmDir" "/vpx/vms/" "/etc/vmware/config"`
   snarfVirtualCenterFiles "$vpxVMDir"
   if [ "$vpxVMDir" != "/home/vmware" ]; then
      snarfVirtualCenterFiles "/home/vmware"
   fi

   #    Grab a dump of all hostd properties export through the VIM API.
   runbackgroundcmd "/usr/bin/vmware-vimdump" "/tmp/vmware-vimdump.$$.txt"
}


#
#	Function: snarf_system_state grabs all the interesting system state
#       and adds it to the the tar file.
snarf_system_state() {

   snarf_running_state

   snarf_hostd_state
   
   if [ "$COS" = "0" ] && [ $no_cores -eq 0 ]; then
        COREDUMP_DIR="/var/core"
        # For embedded extract the vmk core in vm-support. 
        # In classic esx this is done during init however in embedded we may
        # not have storage at init time. Also because of space
        # constraints we gather only the last core.
        if [ -d "$COREDUMP_DIR/" ]; then
            devname=`/usr/sbin/esxcfg-dumppart --get-config | cut -f2`
            if [ $devname != "none" ]; then
               curTime=`date "+%m%d%y.%H.%M"`
               zDumpName=$COREDUMP_DIR/vmkernel-zdump-"$curTime"
               banner "Checking $devname for new core dumps ..."
               /sbin/vmkdump --zdumpname $zDumpName \
                             --devname $devname   \
                             --newonly
            fi
        else
            banner "Warning: No core partition configured."
            all_cores=0
            no_cores=1
        fi
   fi

   # Grab any core files we can find.  Move "found" ones to a place where
   # we won't include them next time this is run.

   if [ $no_cores -eq 0 ]; then
      addtar "$COREDUMP_DIR/vmkernel-*"
      
      addtar "/vmware-vmx-zdump.*"
      rm -f "/vmware-vmx-zdump.*"

      if [ $all_cores -eq 1 ]; then
         addtar "$COREDUMP_DIR/old_cores/vmkernel-*"
         addtar "$COREDUMP_DIR/old_cores/*.core.gz"
      fi

      if [ ! -d $COREDUMP_DIR/old_cores ]; then
         mkdir $COREDUMP_DIR/old_cores
	 rc=$?
         if [ $rc != 0 ]; then
            banner "Could not mkdir $COREDUMP_DIR/old_cores ($rc). Is something wrong with $COREDUMP_DIR?"
         fi
	 log "mkdir $COREDUMP_DIR/old_cores ($rc)"
      fi

      # move off old cores so we don't keep sending old ones.
      if [ $(ls $COREDUMP_DIR/vmkernel-* 2> /dev/null | wc -l) != 0 ]; then
         if [ -d $COREDUMP_DIR/old_cores ]; then
            mv $COREDUMP_DIR/vmkernel-* $COREDUMP_DIR/old_cores/
	    rc=$?
            if [ $rc != 0 ]; then
               banner "Could not archive cores ($rc). Is something wrong with $COREDUMP_DIR?"
            else
               banner "NOTE: All cores archived from $COREDUMP_DIR into $COREDUMP_DIR/old_cores."
            fi
	    log "moved old cores to $COREDUMP_DIR/old_cores/ ($rc)"
         fi
      fi

      # gzip & archive cos core file.
      if [ $no_service_cos_core -eq 0 ]; then
         cosCoreFile=`esxcfg-advcfg -q -g /Misc/CosCorefile`
         if [ -e "$cosCoreFile" ]; then
            coreBkup=$COREDUMP_DIR/old_cores/`basename $cosCoreFile`.$$.core.gz
            if [ -d $COREDUMP_DIR/old_cores ]; then
               gzip -fc $cosCoreFile > $coreBkup
               if [ $? != 0 ]; then
                  banner "Failed to gzip service console core file."
               else
                  addtar "$coreBkup"
                  rm -f $cosCoreFile
               fi
            fi
         fi
      fi
   fi

   if [ "$COS" = "1" ]; then
         perl -e '$data = `strings /usr/lib/vmware/vmkernel | grep "build-"`; ( $build ) = $data =~ / build-(\d+) /x; `echo "Found vmkernel version $build" >> /var/log/vmkernel-version`'; 
   fi

   #	Add system configuration and log files. Wildcards
   #	may be used.

   addtar "/etc/hosts"
   addtar "/etc/group"
   addtar "/etc/resolv.conf"
   addtar "/etc/nsswitch.conf"
   addtar "/etc/ldap.conf"
   addtar "/etc/sysctl.conf"
   addtar "/etc/inittab"
   addtar "/etc/iproute2/"
   addtar "/etc/iscsi.conf"
   addtar "/etc/ntp.conf"
   addtar "/etc/openldap/"
   addtar "/etc/pam_smb.conf"
   addtar "/etc/exports"
   addtar "/etc/hosts.conf"
   addtar "/etc/hosts.allow"
   addtar "/etc/hosts.deny"
   addtar "/etc/fstab.iscsi"
   addtar "/etc/krb*"
   addtar "/boot/grub/grub.conf"
   addtar "/boot/grub/device.map"
   addtar "/boot/grub/menu.lst"
   addtar "/etc/lilo.conf"
   addtar "/etc/modules.conf"
   addtar "/etc/opt/vmware/vpxa/vpxa.cfg"
   addtar "/etc/rc.d/rc.local"
   addtar "/etc/rc.d/init.d/vmware*"
   addtar "/etc/ssh/ssh_conf"
   addtar "/etc/ssh/sshd_config"
   addtar "/etc/vmkiscsi.conf"
   addtar "/etc/vmware-release"
   addtar "/etc/yp.conf"
   addtar "/etc/yum.conf"
   addtar "/etc/logrotate.conf"
   addtar "/etc/logrotate.d/"
   addtar "/etc/snmp/snmpd.conf"
   addtar "/etc/sysconfig"
   addtar "/etc/vmware/"
   addtar "/etc/opt/vmware/" IGNORE_ERROR
   addtar "/etc/xinetd.d/vmware-authd"
   addtar "/tmp/vmkdump.log"
   addtar "/etc/syslog.conf"
   if [ $no_cores -eq 0 ]; then
      addtar "/var/core/"
      addtar "/var/cores/"
   fi
  
   # For vmvisor we are collecting the whole /var/log directory much
   # earlier
   if [ $COS = "1" ]; then
      if [ $no_cores -eq 0 ]; then
         addtar "/var/log/vmware/core"
      fi
      addtar "/var/log/boot*"
      addtar "/var/log/initrdlogs"
      addtar "/var/log/messages*"
      addtar "/var/log/secure*"
      addtar "/var/log/dmesg"
      addtar "/var/log/vmkproxy"
      addtar "/var/log/vmkernel*"
      addtar "/var/log/vmksummary"
      addtar "/var/log/vmksummary.d/vmksummary*"
      addtar "/var/log/vmkwarning*"
      addtar "/var/log/vmware/"
      addtar "/var/log/cron*"
      addtar "/var/log/rpmpkgs*"
   else 
      addtar "/bootbank/local.tgz"
      addtar "/bootbank/oem.tgz"
      addtar "/bootbank/boot.cfg"
   fi

   addtar "/etc/fstab"
   addtar "/etc/pam.d/"
   addtar "/etc/cron*"
   addtar "/var/lib/iscsi/"
   addtar "/var/pegasus/*.conf*"
   addtar "/var/kerberos/krb5kdc/kdc.conf"
   addtar "/root/anaconda-ks.cfg" IGNORE_ERROR
   addtar ~/.bash_history IGNORE_ERROR

   if [ "$COS" = "1" ]; then
        rpmList=`rpm -qa 'VMware*'`
        runbackgroundcmd "rpm --verify $rpmList" "/tmp/rpm-verify.$$.txt" 
   fi

   if [ "$COS" = "0" ]; then
      # snarf cim classes
      runbackgroundcmd "/bin/cim-diagnostic.sh" "/tmp/cim.$$.txt"
      runbackgroundcmd "/bin/lsi_log" "/tmp/lsi.$$.log"
   fi

   # collect aam information
   if [ -e /opt/LGTOaam512/vmware/aam_config_util.pl ] ; then
   	mkdir /tmp/aamSupport.$$
   	runcmd "/usr/bin/perl /opt/LGTOaam512/vmware/aam_config_util.pl -cmd=support -dir=/tmp/aamSupport.$$" "/tmp/aam_config_util.output"
   	addtar "/tmp/aamSupport.$$/"
        rm -rf /tmp/aamSupport.$$
   fi

   if [ "$COS" = "1" ]; then
   	addtar "/etc/opt/vmware/aam/"
   else
	addtar "/var/run/vmware/aam/"
   fi

   # webAccess config files
   addtar "/usr/lib/vmware/webAccess/tomcat/jakarta-tomcat-*/webapps/ui/WEB-INF/classes/*.properties"
   addtar "/usr/lib/vmware/webAccess/tomcat/jakarta-tomcat-*/webapps/ui/WEB-INF/classes/ui/*.properties"

   # grab the vmauthd and vmkauthd log files, if enabled
   if [ -f /etc/vmware/config ] ; then
      vmauthdLogFile=`getConfigVal "log.vmauthdFileName" "" "/etc/vmware/config"`
      vmkauthLogFile=`getConfigVal "log.vmkauthdFileName" "" "/etc/vmware/config"`
      for logFile in "$vmauthdLogFile" "$vmkauthdLogFile"; do
         if [ -n "$logFile" ] ; then
            if [ `dirname $logFile` = "." ] ; then
               logFile=/var/log/vmware/$logFile
            fi
            addtar $logFile
         fi
      done
   fi 

   # General cmds.
   runcmd "echo vm-support version: $VER" "/tmp/vm-support-version.$$.txt"
   runcmd "uname -a" "/tmp/uname.$$.txt"
   if [ "$COS" = "1" ]; then
	   runcmd "lspci -H1 -M" "/tmp/lspci1.$$.txt"
	   runcmd "lspci -H1 -M -vn" "/tmp/lspci2.$$.txt"
	   runcmd "lspci -vv" "/tmp/lspci3.$$.txt"
	   runcmd "lspci -H1 -t -vv -n" "/tmp/lspci4.$$.txt"
	   runcmd "lspci -v -b" "/tmp/lspci5.$$.txt"
	   runcmd "/sbin/lsmod" "/tmp/modules.$$.txt"
	   # added -l option to df to avoid NFS filesystems should the network
	   # be down
	   runcmd "df -al" "/tmp/df.$$.txt"
	   # specify /vmfs to vdf, also to avoid any NFS filesystems
	   runcmd "vdf -h /vmfs" "/tmp/vdf.$$.txt"
	   runcmd "ifconfig -a" "/tmp/ifconfig.$$.txt"
	   runcmd "ping -c5 `/sbin/route -n | awk '{ if ($4 ~ /G/) print $2; }'`" "/tmp/ping-gateway.$$.txt"
	   runcmd "ifconfig -a" "/tmp/ifconfig_after.$$.txt"
	   runcmd "mii-tool -vv" "/tmp/mii-tool-ethN.$$.txt" IGNORE_ERROR
	   runcmd "route" "/tmp/route.$$.txt"
	   runcmd "mount" "/tmp/mount.$$.txt"
	   runcmd "rpm -qa" "/tmp/rpm.$$.txt"
	   runcmd "chkconfig --list" "/tmp/chkconfig.$$.txt"
	   runcmd "/usr/sbin/dmidecode" "/tmp/dmidecode.$$.txt" IGNORE_ERROR
	   runcmd "ls -lR /vmfs/devices" "/tmp/vmfsDevices.$$.txt"
   else
           runcmd "df" "/tmp/df.$$.txt"
	   runcmd "ls -lR /dev" "/tmp/dev.$$.txt"
   fi
   runbackgroundcmd "fdisk -l" "/tmp/fdisk.$$.txt"

   if [ -e /sbin/hplog ]; then
      # snarf hardware logs
      runcmd "/sbin/hplog -v" "/tmp/hplog.$$.txt"
   fi

   # ESX-specific.
   runcmd "vmware -v" "/tmp/vmware.$$.txt"
   if [ "$COS" = "1" ]; then
	   runcmd "esxupdate -l query" "/tmp/esxupdate-patch-history.$$.txt"
   fi

   if [ -x /usr/bin/omreport ]; then
      # snarf Dell OpenManage information
      runcmd "/usr/bin/omreport system alertlog" "/tmp/omreport-alertlog.$$.txt"
      runcmd "/usr/bin/omreport system cmdlog" "/tmp/omreport-cmdlog.$$.txt"
      runcmd "/usr/bin/omreport system esmlog" "/tmp/omreport-esmlog.$$.txt"
      runcmd "/usr/bin/omreport system postlog" "/tmp/omreport-postlog.$$.txt"
      runcmd "/usr/bin/omreport chassis temps" "/tmp/omreport-temps.$$.txt"
      runcmd "/usr/bin/omreport chassis fans" "/tmp/omreport-fans.$$.txt"
      runcmd "/usr/bin/omreport chassis memory" "/tmp/omreport-memory.$$.txt"
   fi

   # VMware kernel specific stuff.  Do not run if it's not loaded
   # /proc/vmware/config will not exist unless kernel is running.
   if [ -d /proc/vmware/config ] || [ "$COS" = "0" ]; then
           runcmd "/usr/sbin/vmkload_mod -v10 -l" "/tmp/vmkmod.$$.txt"
 
           #volume information added for vpa
           runcmd "ls -l /vmfs/volumes" "/tmp/volume_list.vpa.txt"
           for VMFS in $(ls -l /vmfs/volumes | awk '/^d/ {print $9}'); do

                   runcmd "vmkfstools -P /vmfs/volumes/$VMFS" "/tmp/$VMFS.vpa.txt"

                   if [ $dump_vmfs_resfile -eq 1 ]; then 
                           echo -e "\nSaving volume header for $VMFS."

                           # we just save the volume header file for now.
                           addtar "/vmfs/volumes/$VMFS/.vh.sf"
                           # addtar "/vmfs/volumes/$VMFS/.fbb.sf"
                           # addtar "/vmfs/volumes/$VMFS/.pbc.sf"
                           # addtar "/vmfs/volumes/$VMFS/.sbc.sf"
                           # addtar "/vmfs/volumes/$VMFS/.fdc.sf"
                   fi
	   done
   
           if [ "$COS" = "1" ]; then
                   runcmd "vmkchdev -L" "/tmp/vmkchdev.$$.txt"
                   runcmd "/usr/sbin/esxcfg-vmhbadevs -m" "/tmp/esxcfg-vmhbadevs-vmfs.$$.txt"
                   runcmd "/usr/sbin/esxcfg-vmhbadevs" "/tmp/esxcfg-vmhbadevs.$$.txt"
                   runcmd "/usr/sbin/esxcfg-vswif -l" "/tmp/esxcfg-vswif.$$.txt"
                   runcmd "/usr/sbin/esxcfg-firewall -q" "/tmp/esxcfg-firewall.$$.txt"
           fi

           runcmd "/usr/sbin/vmkerrcode -l" "/tmp/vmkerrcode.$$.txt"

           runcmd "/usr/sbin/esxcfg-dumppart -c" "/tmp/esxcfg-vmkdump.$$.txt"
           runcmd "/usr/sbin/esxcfg-info -a" "/tmp/esxcfg-info.$$.txt"
           runcmd "/usr/sbin/esxcfg-nas -l" "/tmp/esxcfg-nas.$$.txt"
           runcmd "/usr/sbin/esxcfg-vswitch -l" "/tmp/esxcfg-vswitch.$$.txt"
           runcmd "/usr/sbin/esxcfg-vmknic -l" "/tmp/esxcfg-vmknic.$$.txt"
           runcmd "/usr/sbin/esxcfg-swiscsi -q" "/tmp/esxcfg-swiscsi.$$.txt"
           runcmd "/usr/sbin/esxcfg-route -l" "/tmp/esxcfg-route.$$.txt"
           runcmd "/usr/sbin/esxcfg-resgrp -l" "/tmp/esxcfg-resgrp.$$.txt"
	   runcmd "/usr/sbin/esxcfg-mpath -lv" "/tmp/esxcfg-mpath.$$.txt"
           runcmd "/usr/sbin/esxcfg-nics -l" "/tmp/esxcfg-nics.$$.txt"

           runcmd "/usr/sbin/vmkping -D -v" "/tmp/vmkping.$$.txt"
   fi

}

#
# Which files are necessary for esxtop (+ additional ones we
# care about).
#
SNAP_SHOT_FILES="cpuinfo net/*/*.info scsi/*/*"

#
# snapshotOne: takes a single snapshot
#
snapshotOne() {
   local snapShotFiles=$1
   local dstDir=$2
   local num=$3
   local zip=$4
   local ERR_FILE=/dev/null

   # For some reason tar doesn't like to read directly from /proc
   # so 'tar -czf proc$num.tgz $FILES' doesn't work.

   mkdir -p $dstDir/proc > $ERR_FILE 2>&1 || exit 1
   for p in  $snapShotFiles ; do
      for f in `ls /proc/$p 2>/dev/null` ; do 
         cp -a --parents $f $dstDir > $ERR_FILE 2>&1
      done
   done
   date > $dstDir/proc/date
   mv $dstDir/proc $dstDir/proc$num > $ERR_FILE 2>&1 || exit 1
   if [ "$zip" != "0" ]; then
      tar -czf $dstDir/proc$num.tgz -C $dstDir proc$num > $ERR_FILE 2>&1
      rm -rf $dstDir/proc$num
   fi

   # Add a snapshot of the VSI tree as well.
   mkdir -p $dstDir/vsi > $ERR_FILE 2>&1 || exit 1
   if [ -x /usr/lib/vmware/bin/vsi_traverse ]; then
      /usr/lib/vmware/bin/vsi_traverse -s  > $dstDir/vsi/vsi.$num
   fi

   # Add a snapshot for the plm stats.
   # Vsh plugin requires an absolute path to the log directory.
   local plmDir=`pwd`/$dstDir/plm
   local vshCmd=/usr/lib/vmware/hostd/vsh
   mkdir -p $plmDir > $ERR_FILE 2>&1 || exit 1
   if [ -x $vshCmd ]; then
      local hostdLibDir=/usr/lib/vmware/hostd
      local plugin=$hostdLibDir/libsupportsvc.so
      LD_LIBRARY_PATH="$hostdLibDir" $vshCmd -e "puse $plugin; supportsvc_cmds/connect; supportsvc_cmds/login; supportsvc_cmds/printstatstofile $plmDir; quit" > /dev/null 2>&1
   fi
   return 0
}

#
# calculateSnapInterval: measures how long a single snapshot
# takes, and sets the sleep interval to twice that.
#
calculateSnapInterval() {
   local startTime=`date +"%s"`
   snapshotOne "$1" "$2" "0" "$3" || exit 1
   local endTime=`date +"%s"`
   local interval=`expr '(' $endTime - $startTime ')' \* 2`
   if [ "$interval" = "0" ]; then
      interval=2
   fi
   echo "$interval"
}

#
# createSnapshots: creates multiple perf snapshots, 
# sleeping the specified (or calculated) amount between
# each snapshot.
#
createSnapshots() {
   local snapShotFiles=$1
   local dest=$2
   local duration=$3
   local sleepInterval=$4
   local zip=$5
   local i=0
   local startTime=`date +"%s"`

   echo -e "\nTaking performance snapshots.  This will take about $duration seconds."
   mkdir -p $dest
   echo -e '#!/bin/sh\n\nfor f in `ls proc*.tgz`; do\n   tar -xzf $f\n   rm $f\ndone\n' > $dest/untar.sh
   chmod +x $dest/untar.sh

   if [ "$sleepInterval" = "-1" ]; then
      i=1
      sleepInterval=`calculateSnapInterval "$snapShotFiles" "$dest" "$zip"`
      if [ "$?" != 0 ]; then
         echo "Error calculating interval: $sleepInterval" 
         exit
      fi

      echo "Snapshot interval is $sleepInterval" | tee $dest/interval
   fi

   schedStatsEnable  # Enable detailed scheduler stats.

   while [ 1 ]; do
      local curTime=`date +"%s"`
      if [ $(($startTime + $duration)) -lt $curTime ]; then
         echo -e "\nDone.  $i snapshots created."
         break
      fi
      if [ $i -eq 0 ]; then
	  echo -e "\nStarting vscsiStats."   # only print this message once
          log "starting vscsi stats collection"
      fi
      startVscsiStats $i # "start" vscsiStats each time to reset 30-min timeout
      sleep $sleepInterval
      if [ $quiet_mode -eq 0 ]; then
          echo -en "\rSnapping $i:" `expr $startTime + $duration - $curTime` " seconds left."
      fi
      snapshotOne "$snapShotFiles" "$dest" "$i" "$zip"
      i=`expr $i + 1`
   done
   schedStatsDisable # Disable detailed scheduler stats.
   stopVscsiStats $OUTPUT_DIR # stop vscsiStats collection
}

SCHEDTRACE_MOD=/usr/lib/vmware/vmkmod/schedtrace
SCHEDTRACE_FILE=/tmp/schedtrace
CHANNELLOGGER=/usr/sbin/logchannellogger
# starts schedtrace by loading the schedtrace module
# also starts the logchannellogger
startSchedTrace() {
    if [ -e $SCHEDTRACE_MOD ]; then       
        echo "Starting schedtrace."
	log "Starting schedtrace."
        /usr/sbin/vmkload_mod $SCHEDTRACE_MOD
        $CHANNELLOGGER schedtrace $SCHEDTRACE_FILE 2>/dev/null &
    fi
}

# unload schedtrace module
# also copy the file to vm-support destination directory
stopSchedTrace() {
    local dest=$1
    local ERR_FILE=/dev/null
    
    # check if schedtrace is loaded
    /usr/sbin/vmkload_mod -l |grep schedtrace > /dev/null
    if [ $? -eq 0 ]; then       
        echo -e "\nStopping schedtrace."
	log "Stopping schedtrace."
        /usr/sbin/vmkload_mod -u schedtrace
        sleep 10 #wait logchannellogger finish
        mkdir -p $dest/schedtrace > $ERR_FILE 2>&1 || exit 1
        mv $SCHEDTRACE_FILE $dest/schedtrace/
        cp /usr/sbin/schedtrace_format $dest/schedtrace/
    fi
}

SCHED_STATS=/usr/bin/sched-stats

# Enable detailed scheduler stats.
schedStatsEnable() {
    if [ -e $SCHED_STATS ]; then
        echo -e "\nStarting detailed scheduler stats."
        log "Starting detailed scheduler stats collection."
        # Enabling cpu state histograms."
        $SCHED_STATS -s 1
    fi
}

# Disable detailed scheduler stats.
schedStatsDisable() {
    if [ -e $SCHED_STATS ]; then
        echo -e "\nStopping detailed scheduler stats"
        log "Stopping detailed scheduler stats collection."
        # Disabling cpu state histograms.
        $SCHED_STATS -s 0
    fi
}

VSCSISTATS_BIN=/usr/lib/vmware/bin/vscsiStats
VSCSISTATS_TMP_PATH=/tmp/
VSCSISTATS_TMP_FILE=$VSCSISTATS_TMP_PATH/vscsiStatsTraceList.tmp
# starts vscsiStats collection
# (data is only logged in vm-support output unless cmd tracing is enabled)
startVscsiStats() {
    local vscsiChannel=
    local initialized=$1
    if [ -e $VSCSISTATS_BIN ]; then       
	$VSCSISTATS_BIN -s > /dev/null
	if [ $vscsi_trace -eq 1 ]; then
	    if [ $initialized -eq 0 ]; then
		echo -e "\nStarting vscsi cmd trace for world group $vscsi_trace_wgid."
		log "\nStarting vscsi cmd trace for world group $vscsi_trace_wgid."
		$VSCSISTATS_BIN -s -t -w $vscsi_trace_wgid > $VSCSISTATS_TMP_FILE
		for VSCSITRACEHANDLES in $(grep traceChannel $VSCSISTATS_TMP_FILE); do
		    vscsiChannel=`echo $VSCSITRACEHANDLES | sed -n -e "s/<vscsiStats-traceChannel>//g" -e "s/<\/vscsiStats-traceChannel>//pg"`
		    $CHANNELLOGGER $vscsiChannel $VSCSISTATS_TMP_PATH/$vscsiChannel.vscsitrace 2>/dev/null &
		done
	    fi
	fi
    fi
}

# stops vscsiStats collection
# (data is only logged in vm-support output)
stopVscsiStats() {
    local dest=$1
    local loggers=
    if [ -e $VSCSISTATS_BIN ]; then       
	echo -e "\nStopping vscsiStats."
        log "stopping vscsi stats collection"
	$VSCSISTATS_BIN -x >  /dev/null
	if [ $vscsi_trace -eq 1 ]; then
	    sleep 2
	    mv $VSCSISTATS_TMP_PATH/*.vscsitrace $dest
	    rm $VSCSISTATS_TMP_FILE
	fi
    fi
}

#
# expungeSensitiveFiles
#
# Dont keep any sensitive customer information that was collected 
# during recursive copies.  vm-support.conf can be used for files
# that may be optionally removed.
#
expungeSensitiveFiles() {
   # list of files that shouldn't be included
   EXPUNGELIST="/etc/vmware/ssl/rui.key "

   for FILE in ${EXPUNGELIST}
   do
      log "Removing $FILE from vm-support."
      rm -rf $OUTPUT_DIR$FILE 2>/dev/null
   done
}

#
# createTar
#
# Create the final tar file &  cleanup.
#	Perform the tar ('S' for sparse core files)
#
createTar() {
   expungeSensitiveFiles

   banner "Creating tar archive ..."

   if [ $print_file_list -eq 1 ]; then
       tar -czSvf $TARFILE $OUTPUT_DIR
   else
       tar -czSf $TARFILE $OUTPUT_DIR
   fi

   if [ $? != 0 ]; then
           banner "The tar did not successfully complete!" \
                   "If tar reports that a file changed while" \
                   "reading, please attempt to rerun this script."
   else
           fullPath="$(pwd | sed s/[\/]$//)/$TARFILE"
           size=`stat -t $TARFILE | cut -f2 -d " "`
           if [ $size -gt 10000000 ]; then
                   banner "File: $fullPath" \
                          "NOTE: $TARFILE is greater than 10 MB." \
                          "Please do not attach this file when submitting an incident report." \
                          "Please contact VMware support for an ftp site." \
                          "To file a support incident, go to http://www.vmware.com/support/sr/sr_login.jsp"
           else
                   banner "File: $fullPath" \
                          "Please attach this file when submitting an incident report." \
                          "To file a support incident, go to http://www.vmware.com/support/sr/sr_login.jsp"
           fi
           banner "To see the files collected, run: tar -tzf $fullPath"
   fi


   #	Clean up temporary files

   rm -rf $OUTPUT_DIR

   if [ $? != 0 ]; then
           banner "$OUTPUT_DIR was not successfully removed.  Please remove manually."
   fi
   echo "Done"
}

# 
# MakeOutputDir
#
#  Tries to make a subdir to put all your files in.  Dies if it does not create.
MakeOutputDir() {
   mkdir $OUTPUT_DIR

   if [ $? != 0 ]; then
	banner "Could not create ./${OUTPUT_DIR}... Exiting..." \
               "Please cd to a directory to which you can write" # Thanks Adam!
	exit
   fi
  
   # This is a guarantee that the tmp directory will exist for blocking commands
   # to write their output to.
   mkdir $OUTPUT_DIR/tmp
   
   if [ $? != 0 ]; then
	banner "Could not create ./${OUTPUT_DIR}/tmp... Exiting..." \
               "${OUTPUT_DIR} must be writeable." 
	exit
   fi

   touch $CMD_LOG; # enable logging of vm-support commands
}

#
# VMDumper
#
# Run the vmdumper command and add its output to the vmdumper log
#
VMDumper() {
    # Since vmdumper tends to mutate running VMs, also put a hint in the
    # vmkernel log
    vmklog "vm-support: running vmdumper $@"

    echo "running vmdumper $@" >> $OUTPUT_DIR/hungvm/vmdumper-commands.$$.txt
    /usr/lib/vmware/bin/vmdumper "$@" 2>&1 | tee -a $OUTPUT_DIR/hungvm/vmdumper-commands.$$.txt 
}


#
# DebugHungVM
#
#  Tries to capture some useful information about a single hung VM.
#  (with out relying on any other services being alive and well (ie serverd)
#
DebugHungVM() {
   local wid=$1; # wid of the vmm0 world
   local sendAbort="no"
   local sendNMI="no"
   local takeScreenShot="no"
   local name=`/usr/lib/vmware/bin/vmdumper -l 2>/dev/null | grep "^vmid=$wid"`

   if [ -z "$name" ] ; then
      banner "Cannot find world '$wid'"
      exit 1
   fi

   MakeOutputDir

   local vmxCfg=$(echo $name | sed -n 's/.*cfgFile="\(.*\)".*uuid=.*/\1/p')
   local vmxDirectory=$(dirname "$vmxCfg")

   runcmd "echo 'Suspected hung vm is: $vmxCfg, wid=$wid on $DATE'" "README.hungVM"

   echo ""
   echo ""
   read -p "Can I include a screenshot of the VM $wid? [y/n]: " takeScreenShot

   if [ $suspend_hung_vm -eq 0 ] ; then 

   	read -p "Can I send an NMI (non-maskable interrupt) to the VM $wid? \
This might crash the VM, but could aid in debugging [y/n]: " sendNMI

   	read -p "Can I send an ABORT to the VM $wid? \
This will crash the VM, but could aid in debugging [y/n]: " sendAbort

   fi

   # Grab info about the running processes before poking anything through the
   # hungVM probes
   #
   # Replace OUTPUT_DIR with a different location for the pre-hungvm
   # files, so they don't get overwritten by the post-hungvm support run.
   old_OUTPUT_DIR=${OUTPUT_DIR}
   OUTPUT_DIR="${OUTPUT_DIR}/pre-hungvm"
   MakeOutputDir ## create the pre-hungVM output dir
   snarf_running_state
   OUTPUT_DIR="${old_OUTPUT_DIR}"

   banner "Grabbing data & core files for world $wid.  This will take 5 - 10 minutes."
   local HUNG_VM_FILES="vmware/sched/cpu-verbose vmware/vm/$wid/userRPC"
   local vcpus=""

   if [ "$COS" = "1" ]; then
      vcpus=`cat /proc/vmware/sched/cpu | awk '{print $1 " " $2}' | grep $wid | awk '{print $1}'`
      for vcpu in $vcpus ; do
         HUNG_VM_FILES="$HUNG_VM_FILES vmware/vm/$vcpu/userRPC"
      done
   fi
   
   createSnapshots "$HUNG_VM_FILES" "$OUTPUT_DIR/hungvm" $snap_duration $snap_interval 1

   trap "VMDumper $wid samples_off; exit;" HUP INT QUIT TERM EXIT KILL

   banner "Collecting VM samples (this may take several minutes) ..."
   VMDumper $wid samples_on
   sleep 60

   #
   # grab vmm backtraces
   #
   for i in `seq 1 5` ; do
      for vcpu in $vcpus ; do
         VMDumper $vcpu backtrace
      done
      sleep 1
   done

   if [ "$takeScreenShot" = "yes" ] || [ "$takeScreenShot" = "y" ]; then
      VMDumper $wid screenshot 
   fi

   #
   # grab core files.
   #
   banner "Creating core files (this may take several minutes) ..."
   VMDumper $wid unsync 
   sleep 60
   for unysncCore in "$vmxDirectory"/vmware*-core*
      do
         mv -f "$unysncCore" "${unysncCore}-unsync" > /dev/null 2>&1 
   done
   VMDumper $wid sync 
   sleep 60
   VMDumper $wid vmx
   sleep 120

   if [ ! -f "$vmxDirectory"/vmware-vmx-zdump.* ]; then
      VMDumper $wid vmx_force
      sleep 120
   fi

   banner "Done with core dumps"
   
   VMDumper $wid samples_off
   trap - HUP INT QUIT TERM EXIT KILL

   if [ $suspend_hung_vm -eq 1 ]; then

      touch $vmxDirectory/temp.vmsupport.timestamp.$$
      local time_before_suspend=`stat -c "%Z" $vmxDirectory/temp.vmsupport.timestamp.$$`
      rm $vmxDirectory/temp.vmsupport.timestamp.$$

      VMDumper $wid suspend_vm
      banner "Note: This includes your VM's memory state in the vm-support file."\
             "Depending on the memory size of the VM, this operation could take >15 minutes."

      local suspend_vmss_file=""
      local count=0

      while [ $count -lt 180 ] && ! [ -f /$suspend_vmss_file ]; do
	 UpdateSpinner "Waiting for memory file write to begin"
         count=`expr $count + 1`
         sleep 1

	 for possible_vmss in `ls $vmxDirectory/*.vmss 2>/dev/null` ; do
	    local possible_vmss_timestamp=`stat -c "%Z" $possible_vmss 2>/dev/null`

	    if [ $possible_vmss_timestamp -gt $time_before_suspend ] ; then
	       suspend_vmss_file=$possible_vmss
	       break
	    fi
	 done
      done

      echo ""

      if [ -f $suspend_vmss_file ] ; then
         
	 while ! [ "`/usr/lib/vmware/bin/vmdumper -l 2>/dev/null | grep "^vmid=$wid"`" = "" ] ; do
	    
	    UpdateSpinner "Waiting for memory file write to finish"
	    sleep 1
	    
	 done

	 count=0

	 # wait 10 more seconds to guarantee everything gets finalized
	 while [ $count -lt 10 ] ; do
	    
	    UpdateSpinner "Waiting for memory file write to finish"
	    sleep 1
	    count=`expr $count + 1`
	 
	 done

	 banner "Memory file write finished."
      
      else 
         banner "Could not suspend VM... Please try again or contact VMware support."
      fi
   fi

   if [ "$sendNMI" = "yes" ] || [ "$sendNMI" = "y" ]; then
      VMDumper $wid nmi
   fi

   if [ "$sendAbort" = "yes" ] || [ "$sendAbort" = "y" ]; then
      local proxyPid=""
      
      if [ "$COS" = "1" ]; then
         # XXX A somewhat fragile lookup
         proxyPid=$(pgrep -f "vmkload_app.*${vmxCfg}")
      else
         proxyPid=$(echo $name | sed -n 's/.*vmxCartelID=\(.*\)/\1/p')
      fi

      if [ -n "$proxyPid" ]; then
         # Send SIGUSR1 to the proxy. The signal handler in the proxy will 
         # send a SIGABRT to the cartel and also dump core once the VM is killed
	 kill -USR1 $proxyPid > /dev/null 2>&1

         # Wait until the vmx and vmm worlds die and the proxy dumps core
         local proxyKilled=0
         local count=0
         local vmkloadAppCore=""
         
         while [ $count -lt 18 ] && [ $proxyKilled -eq 0 ]; do
	    sleep 10 
            count=`expr $count + 1`
            kill -0 $proxyPid >/dev/null 2>&1
            if [ $? -eq 1 ]; then
               proxyKilled=1
            fi
         done

         if [ "$COS" = "1" ]; then
            if [ $proxyKilled != 1 ]; then
               banner "Proxy failed to respond after $count tries. Forcing proxy Abort"
               kill -ABRT $proxyPid > /dev/null 2>&1
               #Look in /var/log/vmware/hostd for the proxy core file
               local coreFile=`ls /var/log/vmware/hostd/core.* 2>/dev/null | grep $proxyPid`
               if [ -n "$coreFile" ]; then
                  vmkloadAppCore="/var/log/vmware/hostd/$coreFile"
               fi
            else 
               # get the location of the core file and add it to the tar file
               vmkloadAppCore=$(tail -100 /var/log/vmkproxy | egrep '.*\['$proxyPid'\].*vmkload_app core file.*' | sed 's/.*file:\(.*\)/\1/')
            fi
         else 
             if [ $proxyKilled != 1 ]; then
                kill -ABRT $proxyPid > /dev/null 2>&1
             fi
         fi

         if [ -n "$vmkloadAppCore" ]; then
            banner "Adding proxy core file to tar"
            addtar $vmkloadAppCore
         fi
      else 
         banner "Cannot find pid for world $wid, not aborted." 
      fi
   fi
   banner "Adding all files in '$vmxDirectory' to tar"

   snarfVMFiles "$vmxDirectory"
   
   
   if [ "$COS" = "1" ]; then
      addtar /proc/vmware/log
      runcmd 'cat /proc/vmware/log' "/tmp/vmkernel-after.$$.log"
   fi

   rm -f "$vmxDirectory"/vmware-vmx-zdump.*
   rm -f "$vmxDirectory"/vmware*-core*
}

#
# finishSupportDataCollection
#
#  Last part of vm-support that runs... Waits on background tasks and creates the
#  tar file.
#
finishSupportDataCollection() {

        if [ $snapshot_only -ne 1 ]; then
	   # Grab the multitude of random host files we might be interested in
           snarf_system_state
	fi

	banner "Waiting up to ${BACKGROUND_CMD_WAIT} seconds for background commands to complete:"

	loge "Blocking commands were:${BACKGROUND_CMD_LIST}\n"

	# do the necessary cleanup on the following signals: HUP INT QUIT TERM EXIT KILL 
	stopBackgroundWaitingOnException() {
	    createTar
	    trap - HUP INT QUIT TERM EXIT KILL
	    exit
	}
	   
	trap stopBackgroundWaitingOnException HUP INT QUIT TERM EXIT KILL
	
	totalBackgroundWait=0
	while [ ! $totalBackgroundWait -gt $BACKGROUND_CMD_WAIT ]; do
	
		LIVE_PIDS=""
		
		if [ "${BACKGROUND_CMD_PIDS}" = "" ]; then 
			break;
		fi
	
		for p in ${BACKGROUND_CMD_PIDS}; do
			if ! kill -0 $p 2>/dev/null; then
				# pid $p is dead
	    
				# Check exit status ...
				if ! wait $p; then
					log "Background process $p had error."
				fi
			else
				# Pid p is still running ...
				LIVE_PIDS=${LIVE_PIDS}" ${p}"
			fi
		done
	
		# Remove the dead pids
		BACKGROUND_CMD_PIDS="${LIVE_PIDS}"
	
		# Let some more backgrounded pids go
		sleep 2

		UpdateSpinner "Waiting for background commands"
	
		totalBackgroundWait=$(( $totalBackgroundWait + 2 ))
	
		if [ $totalBackgroundWait -gt ${BACKGROUND_CMD_WAIT} ]; then
			# took too long ...
			log "Background process wait time expired: $LIVE_PIDS."
		fi
	done
    
	trap - HUP INT QUIT TERM EXIT KILL
	
	#Now tar up everything into one file.
	createTar
}

snap_duration=300
snap_interval=10
all_cores=0
no_cores=0
no_service_cos_core=0
schedtrace=0
snapshot=0
snapshot_only=0
hung_vm=0
hung_vm_wid=0
suspend_hung_vm=0
quiet_mode=0
print_file_list=0
force_vmfs_working_dir=0
working_dir=
dump_vmfs_resfile=0
vscsi_trace_wgid=
vscsi_trace=0
vm_stats=0

# main()
while getopts lnNkahfcsSd:i:xX:Z:t:qw:rv arg ; do
   case "$arg" in
      "n") no_cores=1 ;;
      "N") no_service_cos_core=1 ;;
      "a") all_cores=1 ;;
      "s") snapshot=1 ;;
      "c") schedtrace=1 ;;
      "S") snapshot_only=1; ;;
      "d") snap_duration=$OPTARG ;;
      "i") snap_interval=$OPTARG ;;
      "k") vm_stats=1;;
      "Z") hung_vm=1; suspend_hung_vm=1 ; hung_vm_wid=$OPTARG ;;
      "X") hung_vm=1; hung_vm_wid=$OPTARG ;;
      "x") hung_vm=1 ;;
      "t") vscsi_trace=1; vscsi_trace_wgid=$OPTARG ;;
      "q") quiet_mode=1 ;;
      "l") print_file_list=1 ;;
      "f") force_vmfs_working_dir=1 ;;
      "w") working_dir=$OPTARG ;;
      "r") dump_vmfs_resfile=1 ;;
      "v") version ;;
      "h") usage ;;
        *) usage ;;
   esac
done

#
# Exclude args parsed by getopts, there should only be nothing left.
# Make sure there are no extraneous options on the cmd line
#
shift `expr $OPTIND - 1`
if [ $# -gt 1 ]; then usage ; fi

if [ "$COS" = "0" ]; then
        force_vmfs_working_dir=1
        no_service_cos_core=1
	if [ -z $working_dir ]; then
	   working_dir="/var/tmp"
	fi
fi

# Sanity check args
if [ $all_cores -eq 1 ] && [ $no_cores -eq 1 ]; then
	banner "The -n and -a flags are mutually exclusive."
	exit
fi

if [ $snapshot -eq 1 ] && [ $snapshot_only -eq 1 ]; then
	banner "The -S and -s flags are mutually exclusive."
	exit
fi

#	Start message

banner "VMware ESX Server Support Script $VER"

#	Check for root privilege

if [ $(whoami) != "root" ]; then
	banner "You must be root to run this script."
	exit
fi

#       Set the working directory if specified

if [ -n "$working_dir" ]; then
        if ! cd "$working_dir"; then
                banner "Could not set working directory to '$working_dir'."
                exit
        fi
fi

if [ $force_vmfs_working_dir -ne 1 ]; then

	case "$(pwd)" in 
	
		/vmfs/volumes/*) 
	
		banner "The working directory should not be on a VMFS volume." \
			"" \
			"Please run vm-support from another directory or specify a different" \
			"working directory using the -w flag."

		exit

	esac

fi

# Source /etc/profile.  If we can't find it, it's the users problem to get
# their paths straight.

if [ -f /etc/profile ]; then
	. /etc/profile
fi

# Protect against non-default values of $IFS (Not all scripts in /etc/profile.d/ 
# are good citizens).
unset IFS

if [ $hung_vm -eq 1 ];then
  if [ $hung_vm_wid -eq 0 ]; then
     #
     # List the available VMs to dump (don't actually dump anything)
     #
     runningVMs=`/usr/lib/vmware/bin/vmdumper -l 2>/dev/null | wc -l`
     if [ $runningVMs -eq 0 ]; then
        banner "Couldn't find any running VMs"
     else 
        banner "Available worlds to debug:"
        /usr/lib/vmware/bin/vmdumper -l | sed -n 's/pid=.*displayName="\(.*\)".*$/\1/p'
     fi
     exit
  else 
     #
     # Dump state for a single hung VM
     #
     # This code calls MakeOutputDir on its own
     #
     DebugHungVM $hung_vm_wid
  fi
else
  MakeOutputDir
fi


# In quiet mode, print this here since the spinner won't be printed/updated
if [ $quiet_mode -eq 1 ]; then
    banner "Preparing files ..."
fi

   # Generate the human readable IPMI SEL dump
   if [ -x /sbin/decodeSel.sh ]; then
      /sbin/decodeSel.sh
   fi
   # In vmvisor everything in /var/log is useful so we don't need to be
   # selective.
if [ $COS = "0" ]; then
   addtar "/var/log"
else
   if [ -e /var/log/ipmi_sel ]; then
	addtar "/var/log/ipmi_sel"
   fi
   
   if [ -e /var/log/ipmi_sel.raw ]; then
	addtar "/var/log/ipmi_sel.raw"
   fi
fi

# do the necessary cleanup on the following signals: HUP INT QUIT TERM EXIT KILL 
cleanupOnException() {
    trap - HUP INT QUIT TERM EXIT KILL
    stopSchedTrace $OUTPUT_DIR  #stop schedtrace if it's loaded
    schedStatsDisable
    stopVscsiStats $OUTPUT_DIR # stop vscsiStats collection
    finishSupportDataCollection
    exit
}

#
# Before dumping host-wide state, do (performance-related)
# periodic snapshots
#
if  [ $snapshot -eq 1 ] || [ $snapshot_only -eq 1 ]; then
   # 
   # People might get bored waiting for the snapshoting to finish, so
   # install a signal handler that will create the final tar even if the snapshot
   # process gets interrupted.
   trap cleanupOnException HUP INT QUIT TERM EXIT KILL

   zip=1
   if [ $schedtrace -eq 1 ]; then
       startSchedTrace    # start schedtrace
   fi
   
   createSnapshots "$SNAP_SHOT_FILES" "$OUTPUT_DIR/snapshots" \
                   $snap_duration $snap_interval $zip

   if [ $schedtrace -eq 1 ]; then
       stopSchedTrace $OUTPUT_DIR  #stop schedtrace if it's loaded
   fi   
   echo -e "\rDone with performance snapshots."

   trap - HUP INT QUIT TERM EXIT KILL
fi

#
# Collect all the host-wide state and tar it up
#
finishSupportDataCollection

#eof
