#!/bin/ksh
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
#  
#  
# Licensed Materials - Property of IBM 
#  
# (C) COPYRIGHT International Business Machines Corp. 2000,2007 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
###################################################################
#
# Description:
#     The ctsnap command gathers configuration, log and trace information
#     about the Reliable Scalable Cluster Technology (RSCT) components that
#     are installed with BOS LINUX or AIX. This command only collects
#     the data on one local node on which this command is running.
#
#     The RSCT components include the following:
#
#       - Resource Monitoring and Control subsystem (RMC)
#
#       All IBM.* resource managers, such as:
#
#         - Audit Log resource manager (IBM.AuditRM)
#
#         - Event Response resource manager (IBM.ERRM)
#
#         - File System resource manager (IBM.FSRM)
#
#         - Host resource manager (IBM.HostRM)
#
#         - DMS resource manager (IBM.DMSRM)
#
#         - Sensor resource manager (IBM.SensorRM)
#
#       - High Availability Topology Services(HATS)
#
#       - High Availability Group Services(HAGS)
#
#     This command is typically executed when a problem is encountered with
#     any of these components in order to provide service information to the
#     IBM Support Center.
#
# Syntax:
#       ctsnap [-h] [-a] [-d output_directory]
#
# Flags:
#       -h writes the usage of this command to standard output
#       -a collects only information pertinent to HACMP clusters
#          on Linux
#       -d is used to specify the output directory (absolute path)
#          default is /tmp/ctsupt
#
# Ouputs:
#     tar file : ctsnap.host_name.nnnnnnnn.tar.Z 
#     log file : ctsnap.host_name.nnnnnnnn.log
#     (where host_name is the base name of the host name of the
#     node on which the command is running; nnnnnnnn is a timestamp).
#     By default these files are placed in the /tmp/ctsupt directory.
#
# Exit Status:
#     0 - run successfully
#     1 - error occured
#
# Security:
#     This command can only be run by a root user.
#
# Contents of the information collected:
# (LINUX or AIX in the parenthesis means now
# it is collected only on the node with LINUX
# or AIX since it is not available yet on
# another platform)
#
# 1. Files:
#
#   All files in /var/ct/
#   All files in /var/adm/ffdc/
#   All files in /usr/sbin/rsct/trctmplts/
#
#   Collect the following daemons if the
#   corresponding core files exist, otherwise
#   collect what strings on AIX or
#   strings xxx|grep "@(#)" on LINUX:
#     /usr/sbin/rsct/bin/rmcd
#     /usr/sbin/rsct/bin/IBM.*d, such as:
#       /usr/sbin/rsct/bin/IBM.SensorRMd
#       /usr/sbin/rsct/bin/IBM.Sampled
#       /usr/sbin/rsct/bin/IBM.LANRMd
#       /usr/sbin/rsct/bin/IBM.HostRMd
#       /usr/sbin/rsct/bin/IBM.FSrmd
#       /usr/sbin/rsct/bin/IBM.ERrmd
#       /usr/sbin/rsct/bin/IBM.DMSRMd
#       /usr/sbin/rsct/bin/IBM.AuditRMd
#     /usr/sbin/rsct/bin/hagsd
#     /usr/sbin/rsct/bin/hatsd, hats_nim
#
#   /etc/resol.conf
#   /etc/services
#   /var/log/messages* (LINUX)
#
# 2. Output files of commands:
#
#   errpt -a (AIX)
#
#   If a RSCT subsystem is running collect
#   lssrc -ls subsys_name, such as:
#     lssrc -ls  ctrmc
#     lssrc -ls  IBM.AuditRM
#     lssrc -ls  IBM.ERRM
#     lssrc -ls  IBM.FSRM
#     lssrc -ls  IBM.DMSRM
#     lssrc -ls  IBM.HostRM
#     lssrc -ls  IBM.SensorRM
#     and similarly to other IBM.*
#     resource managers
#     lssrc -l -s cthats
#     lssrc -l -s cthags
#     lsaudrec -l
#   Persistent and dynamic attributes of configRM classes:     
#     lsrsrc -a b -t IBM.SharedResourceCluster
#     lsrsrc -a b -t IBM.PeerNode
#     lsrsrc -a b -t IBM.NetworkInterface
#     lsrsrc -a b -t IBM.CommunicationGroup
#     lsrsrc -a b -t IBM.RSCTParameters
#   CSM Specific
#     lsrsrc -ab IBM.ManagedNode
#     lsrsrc -ab IBM.ManagementServer
#     lsnode -l
#
#   ct_hats_info          and the command return code
#   ct_topology_info      and the command return code
#   ct_read_machines_lst  and the command return code
#
#   ct_hags_info  and the command return code
#   hagsns -c -s cthags
#   hagsvote -l -c -s cthags
#   hagspbs -s cthags
#   hagsgr -s cthags
#   hagsmg -s cthags
#   hagscl -l -s cthags
#   hagscounts -s cthags
#
#   ct_clusterinfo   and the command return code
#   lsclcfg          and the command return code
#
#   For network information:
#
#   ifconfig -a
#   netstat
#   netstat -r
#   netstat -i
#   netstat -rn
#   netstat -in
#   netstat -s
#   netstat -D(AIX)
#   netstat -m(AIX)
#   netstat -A(AIX)
#   no -a(AIX)
#   (for LINUX, we collect all files in
#   /proc/sys/net/ipv4/conf/ and
#   /proc/sys/dev/ )
#
#   Miscellaneous command:
#
#   df -k /tmp
#   df -k /var/ct
#
#   ps -edfl|grep -E -e "IBM|rmc|hats|hags"
#
#   lssrc -a | grep -E -e "rsct|hats|hags"
#
#
#   lslpp -ha(AIX)
#   rpm -q rsct.core rsct.core.utils rsct.basic rsct.sdk src (LINUX)
#
#   vmstat 1 3
#
#   vmstat -s(AIX) (on Linux vmstat already
#   shows info what the vmstat -s on AIX)
#
#
####################################################################
#sccsid="@(#)67   1.39   src/rsct/utils/ctsnap.sh, common_utils, rsct_relgh, relghs001a 4/11/07 18:56:37";

VERSION=1.39

PATH=/usr/bin:/etc:/usr/sbin:/usr/sbin/rsct/bin:/sbin:/bin:$PATH

export LANG=C
export LC_MESSAGES=C
export LC_TIME=C

ROOTDIR=/
LOGDIR=/tmp/ctsupt
total_bytes=0

SUBSYSGRP=rsct_rm       # group name for all the resouce manager subsystems
RSCTGRP=rsct
HATSGRP=cthats
HAGSGRP=cthags
HAGSGLSMGRP=cthagsglsm

RMC=ctrmc           # rmc subsystem
HATS=cthats
HAGS=cthags
HAGSGLSM=cthagsglsm

RSCTBINDIR=/usr/sbin/rsct/bin
CTDIR=/var/ct

hacmp_on_linux=0

#####################################################################
# FUNCTIONs
#####################################################################

usage () {
  echo "Usage: ctsnap [-h] [-d output_directory]"
  echo "    -h Display this usage statement."
  echo "    -d Directory to put information (default : /tmp/ctsupt)."
} # End of usage


valid_dir () {
  case "$1" in
        /*) echo "..................."
            ;;
        *)  echo "Invalid -d argument. You must specify an absolute path."
            exit 1
            ;;          # Not valid path.
  esac
} # End of valid_dir


function check_space {

  echo "    Checking for required and free space ......." >> $LOGFILE 2>&1

  # adding an extra 20 kbytes into total number of bytes needed in order to
  # keep filesystem from being 100 percent full on systems tight on space

  total_bytes=`expr $total_bytes + 20`
  echo "        Required space : $total_bytes kbytes" >> $LOGFILE 2>&1

  if [ "$BASEOS" = "AIX" ]
  then
      free_space=`df -k ${LOGDIR} | tail -n +2 | awk '{print $3}'`
  else
      free_space=`df -k ${LOGDIR} | tail -n +2 | awk '{print $4}'`
  fi
  echo "        Free space : $free_space kbytes" >> $LOGFILE 2>&1

  if [ "$free_space" -lt "$total_bytes" ]
  then
    echo "    Not enough space in ${LOGDIR}" 
    echo "    Not enough space in ${LOGDIR}" >> $LOGFILE 2>&1
    echo "    Exiting utility" >> $LOGFILE 2>&1
    exit 1
  else
    echo "    Space checking is done" >> $LOGFILE 2>&1
  fi

} # End of check_space

function check_and_tar_symbolic_dirs {
  i=0
  circular=0
  fulldir=$new_name
  pre_dir=$new_name

  while [[ -L $fulldir ]]
    do
      if (( $i > $max_link_count )) # after max_link_count stop searching
      then
          echo "The symbolic link count exceeds max_link_count:$max_link_count." >> $LOGFILE 
          break 1
      fi
      #get the real directory name to tar
      fulldir=$(ls -ld $fulldir |awk '{print $NF}')

      # check if it is a relative dir and get the absolute path without ..
      # so that it is easy for comparison for checking circular link
      if (( $i > 0 ))
      then
          ((ii=$i-1))
          pre_dir=${dir_array[$ii]}
      fi
      if [[ $fulldir = ..* ]]
      then
          abdir=${pre_dir%/*}
          while [[ $fulldir = ..* ]]
          do
            fulldir=${fulldir#../}
            abdir=${abdir%/*}
          done
          fulldir=$abdir/$fulldir
      elif [[ $fulldir = .* ]]
      then
          fulldir=${fulldir#.}
          abdir=${pre_dir%/*}
          fulldir=$abdir$fulldir
      elif [[ $fulldir != /* ]]
      then
          abdir=${pre_dir%/*}
          fulldir=$abdir/$fulldir
      fi
      dir_array[$i]=$fulldir

      # check circular link since now they are all absolute paths
      j=0
      while (( $j < $i )) 
        do
          if [[ ${dir_array[$i]} = ${dir_array[$j]} ]]  
          then
              echo "${dir_array[$i]} is a circular link." >> $LOGFILE 
              circular=1
              break 2
          fi
          ((j=$j+1))
      done
      flist=".$fulldir $flist" # keep the original middle dirs structure
                                   # so that we can trace the links
      ((i=$i+1))
  done
  if [[ $circular != 1 ]] 
  then 
      if [[ -e ${TARFILE} ]]
      then
          tar -uf ${TARFILE} .$fulldir 2>/dev/null 
      else
          tar -cf ${TARFILE} .$fulldir 2>/dev/null 
      fi
  fi
} # End of check_and_tar_symbolic_dirs

function timeout_cmd {
    typeset sleepTime=$1 # sleep for $1 secs
    typeset timeOut=$2   # timeout in $2 secs
    typeset waitTime=0   # increment value
    typeset pid=""

    # run the cmd
    eval $cmd &
    pid=$!

    while [[ $waitTime -lt $timeOut ]]
    do
      waitTime=$(($waitTime + $sleepTime))

      if ! kill -s 0 $pid 2>/dev/null
      then
        isTimeOut=0 
        break
      fi
      sleep $sleepTime
    done
    if [[ $waitTime -ge $timeOut ]]                              
    then                                                         
        kill $pid 2>/dev/null                                    
        isTimeOut=1                                              
        print -- "command $cmd Timed out!" >> $LOGFILE           
    fi                                                           
                                                                 
} # end of timeout_cmd

function run_cmd {
  exec 2>/dev/null
  if [ "$verbal" = 1 ]                                                       
  then                                                                       
      echo "command $cmd is running."                                      
  fi                                                                         
  # Section to convert "ls" type commands
  if [[ `echo "$cmd" | awk '{print $1}'` = "ls" ]]
  then
      exec 2>/dev/null
      set $cmd
      path_name=$2
	  print -n "."
      new_name=$path_name      
      if [[ -L $new_name ]] 
      then
          check_and_tar_symbolic_dirs #handle symbolic link
      fi 
      flist=".$new_name $flist"  # keep the original dir structure but
                                 # if it is a symbolic link then it's 
                                 # only a symbolic link when tarred
                                 # so there is no duplication
                                 #  If it's not a symbolic link then
                                 # get the file list for archive
  else
    echo "$cmd" | egrep "lsrsrc|lsrpnode|lscomg"  >/dev/null 2>/dev/null       
    if [ "$?" = 0 ]
    then
        timeout_cmd 5 30
    else
        eval $cmd
    fi

    print -n "."

    exec 2>/dev/null
  fi
} # End of run_cmd

##############################################################################
# Function: collect_hags_command_outputs
#
# Required argument: HAGS subsystem name (e.g. cthags, grpsvcs)
##############################################################################

function collect_hags_command_outputs {
    #collect hags command outputs
    #-----------------------------
    comp=hagsns 
    cmd="hagsns -c -s $1 > .${OUTDIR}/$comp.out"
    run_cmd

    comp=hagsvote
    cmd="hagsvote -l -c -s $1 > .${OUTDIR}/$comp.out"
    run_cmd

    comp=hagspbs 
    cmd="hagspbs -s $1 > .${OUTDIR}/$comp.out"
    run_cmd

    comp=hagsgr 
    cmd="hagsgr -s $1 > .${OUTDIR}/$comp.out"
    run_cmd

    comp=hagsmg 
    cmd="hagsmg -s $1 > .${OUTDIR}/$comp.out"
    run_cmd
    
    comp=hagscl 
    cmd="hagscl -l -s $1 > .${OUTDIR}/$comp.out"
    run_cmd

    comp=hagscounts 
    cmd="hagscounts -s $1 > .${OUTDIR}/$comp.out"
    run_cmd
}

##############################################################################
# Function: doit
#
# To utilize this facility and add new function to collect more data from a
# system you must use one of three methods :
#
# 1) single redirection in the form of " > "
#     Notice the spaces surrounding the greater than sign
# 2) append redirection in the form of "  >> "
#     Notice the spaces surrounding the greater than signs
# 3) The ls command in the form of "ls xxxx"
#     Notice the xxxx can be either a file name or a directory name
#     (but no wildcard)
#
#   All of these forms are very important for supporting conversion of commands
#   to forms that will pipe their output to the "wc -c" command to count the
#   number of bytes for the particular operation.
#   If the commands are not in this form, you will get an error message.
#   Again, this is important to understand when adding new functionality into
#   the script for collecting more documentation
#
#   For the first (with ">") and the second (with ">>") methods you must use 
#   the following 3-line sequence to perform the appropriate space checking.
#
#   for example, for collecting "errpt -a" information :
#           comp=errpt
#           cmd="errpt -a > ./errpt.out 2>&1"
#           run_cmd
#    
#    comp    => This is the component name that will be used as the output
#               file name. This output file will be save in log directory.
#               There is no need to specify the component for the ls command.
#    cmd     => This is the actual command you would like to run in one of the
#               three appropriate formats as noted above.  Notice the " > " in
#               this form of the command.
#    run_cmd => This is just the function call to take appropriate action
#               depending on which pass we are on.
#
#   For the third (ls) method, there is no need to specify the component name,
#   only cmd and run_cmd lines are needed.
#
##############################################################################
function doit {


  #1. collect files
  #-------------------
  #handle the files in /var/ct
  #-------------------------
  cmd="ls /var/ct >/dev/null 2>&1"
  run_cmd

  #handle the files in /var/adm/ffdc/
  #-------------------------
  cmd="ls /var/adm/ffdc >/dev/null 2>&1"
  run_cmd

  #handle the template files
  #-------------------------
  cmd="ls /usr/sbin/rsct/trctmplts >/dev/null 2>&1"
  run_cmd
 

  #check if RMC, HATS, HAGS configured
  #------------------------------------
  export LC_ALL=C
  lssrc -s $RMC >/dev/null 2>&1
  if [ "$?" = 0 ]
  then
    IF_RMC_CONFIGED="yes"
    #check if in /var/ct/<cluster>/run/mc/ core files exist
    #or to check if rmgr core files exist, as long as there is one
    #exists collect rmc and rmgr daemons: rmcd, IBM.*d
    #---------------------------------------------------
    #cluster_name=`ct_clusterinfo |grep CLUSTER_NAME | awk '{print $2}'`
    cluster_name=`lsclcfg |tail -n +2 | awk '{print $1}'`
    RMCRUNDIR=$CTDIR/$cluster_name/run
    search_dir_list="$RMCRUNDIR/mc" 
    if [[ -d $search_dir_list ]]
    then
        corefile=`find $search_dir_list -name 'core*' -print` 
        if [ "$corefile" != "" ]
          then
            HAS_RMC_CORE="yes"
        fi
    fi
    if [ "$HAS_RMC_CORE" = "no" ]
    then
      daemon_list=`ls $RSCTBINDIR/IBM*`
      tmp_daemon_list=""
      for i in $daemon_list
      do
        tmp=`basename $i`
        tmp_daemon_list="$tmp $tmp_daemon_list"
      done
      daemon_list="rmcd $tmp_daemon_list"
      if [ "$BASEOS" = "AIX" ]
      then
        for i in $daemon_list
        do
          comp=what$i
          cmd="what \$RSCTBINDIR/\$i > .${OUTDIR}/$comp.out"
          run_cmd
        done
      elif [ "$BASEOS" = "Linux" ]
      then
        for i in $daemon_list
        do
          comp=strings$i
          cmd="strings \$RSCTBINDIR/\$i|grep \"@(#)\" > .${OUTDIR}/$comp.out"
          run_cmd
        done
      fi
    else #collect rmc, rmgr daemons
      daemon_list=`ls $RSCTBINDIR/IBM*`
      daemon_list="$RSCTBINDIR/rmcd $daemon_list"
      for i in $daemon_list
      do
        cmd="ls $i >/dev/null 2>&1"
        run_cmd
      done
    fi
  fi

  lssrc -s $HATS >/dev/null 2>&1
  if [ "$?" = 0 ]
  then
    IF_HATS_HAGS_CONFIGED="yes"
    #check if in /var/ct/<cluster>/run/cthats/ core files exist
    #
    if [[ -d $RMCRUNDIR/$HATS ]]
    then
        corefile=`find $RMCRUNDIR/$HATS -name 'core*' -print` 
        if [ "$corefile" != "" ]
        then
          HAS_HATS_CORE="yes"
        fi
    fi
    #check if in /var/ct/<cluster>/run/cthags/ core files exist
    if [[ -d $RMCRUNDIR/$HAGS ]]
    then
        corefile=`find $RMCRUNDIR/$HAGS -name 'core*' -print` 
        if [ "$corefile" != "" ]
        then
          HAS_HAGS_CORE="yes"
        fi
    fi
    #check if in /var/ct/<cluster>/run/cthagsglsm/ core files exist
    if [[ -d $RMCRUNDIR/$HAGSGLSM ]]
    then
        corefile=`find $RMCRUNDIR/$HAGSGLSM -name 'core*' -print` 
        if [ "$corefile" != "" ]
        then
          HAS_HAGSGLSM_CORE="yes"
        fi
    fi

    if [ "$HAS_HATS_CORE" = "yes" ]
    then
      daemon_list="$RSCTBINDIR/hatsd $RSCTBINDIR/hats_nim"
      what_list="hagsd"
      if [ "$HAS_HAGS_CORE" = "yes" ]
      then
        daemon_list="$RSCTBINDIR/hagsd $daemon_list"
        what_list=""
      fi
    elif [ "$HAS_HAGS_CORE" = "yes" ]
    then
      daemon_list="$RSCTBINDIR/hagsd"
      what_list="hatsd hats_nim"
    else
      daemon_list=""
      what_list="hagsd hatsd hats_nim"
    fi

    if [ "$HAS_HAGSGLSM_CORE" = "yes" ]
    then
      daemon_list="$RSCTBINDIR/hagsglsmd"
      what_list="hagsglsmd"
    fi

    if [ "$what_list" != "" ]
    then
      if [ "$BASEOS" = "AIX" ]
      then
        for i in $what_list
        do
          comp=what$i
          cmd="what \$RSCTBINDIR/\$i > .${OUTDIR}/$comp.out"
          run_cmd
        done
      elif [ "$BASEOS" = "Linux" ]
      then
        for i in $what_list
        do
          comp=strings$i
          cmd="strings \$RSCTBINDIR/\$i|grep \"@(#)\" > .${OUTDIR}/$comp.out"
          run_cmd
        done
      fi
    fi
    #collect hats or hags or both daemons
    if [ "$daemon_list" != "" ]
    then
        for i in $daemon_list
        do
          cmd="ls $i >/dev/null 2>&1"
          run_cmd
        done
    fi
  fi

  # get /etc/resolv.conf
  cmd="ls /etc/resolv.conf >/dev/null 2>&1"
  run_cmd

  #get /etc/services
  cmd="ls /etc/services >/dev/null 2>&1"
  run_cmd

  #get /etc/hosts
  cmd="ls /etc/hosts >/dev/null 2>&1"
  run_cmd

  #get /etc/inittab
  cmd="ls /etc/inittab >/dev/null 2>&1"
  run_cmd

  if [[ $ISSERVER = 1 ]]
  then
    new_name=`ls /var/log`
    for i in $new_name
    do
      cmd="ls /var/log/$i >/dev/null 2>&1"
      run_cmd
    done
  elif [ "$BASEOS" = "Linux" ]
  #get /var/log/messages*(LINUX)
  then
    new_name=`ls /var/log/messages*`
    for i in $new_name
    do
      cmd="ls $i >/dev/null 2>&1"
      run_cmd
    done
  fi

  #2. output of commands:
  #-------------------------------
  #get the errpt information(AIX)
  #------------------------------
  if [ "$BASEOS" = "AIX" ]
  then
    comp=errpt
    cmd="errpt -a > .${OUTDIR}/$comp.out"
    run_cmd

    comp=svmon
    cmd="svmon -P > .${OUTDIR}/$comp.out"
    run_cmd
  fi

  #If a RSCT subsystem is running collect lssrc -ls subsys_name output 
  #subsystem may have a status of inoperative;
  #in this case, no need to calculate the space and to run the command
  #-------------------------------------------------------------------
  #get name list of all the rm and rmc subsystems
  if [ "$IF_RMC_CONFIGED" = "yes" ]
  then
    sysname_list=`lssrc -g ${SUBSYSGRP} | tail -n +2 | awk '{print $1}'`
    sysname_list="$RMC $sysname_list"                      # add ctrmc
    if [ "$IF_HATS_HAGS_CONFIGED" = "yes" ]
    then
       sysname_list="$HATS $HAGS $HAGSGLSM $sysname_list"
    fi 
    for i in $sysname_list
    do
      sys_status=`lssrc -s $i | tail -n +2 | awk '{print $3}'`
      if [ "$sys_status" = "inoperative" ]
      then
        echo "    Subsystem $i is inoperative, command \"lssrc -ls $i\" is ignored" >> $LOGFILE 2>&1
      else
        comp=$i
        cmd="lssrc -ls $i > .${OUTDIR}/lssrc.$comp.out"
        run_cmd
      fi
    done
  fi
 
  #HATS, HAGS command outputs
  #---------------------------
  if [ "$IF_HATS_HAGS_CONFIGED" = "yes" ]
  then
    #collect ctXXX command outputs except ct_clusterinfo which
    #will be collected as long as RMC is configured
    #----------------------------------------------------------
    ctcommand_list="ct_hats_info ct_topology_info ct_hags_info"
    for i in $ctcommand_list
    do
      #if the cmd exist
      comp=$i
      cmd="$i > .${OUTDIR}/$comp.out"
      run_cmd
      cmd=$i
      eval $cmd >/dev/null 2>&1
      echo "$i return code: $?" >> .${OUTDIR}/$comp.out
    done
    # special treatment for ct_read_machines_lst since its syntax is different
    # The output file is ".out" format 
    cluster_name=`lsclcfg |tail -n +2 | awk '{print $1}'`
    if [[ -e $CTDIR/$cluster_name/cfg/machine.lst ]]
    then
          cmd="ct_read_machines_lst .${OUTDIR}/ct_read_machines_lst.out"
          run_cmd
          cmd="ct_read_machines_lst .${OUTDIR}/ct_read_machines_lst.out"
          eval $cmd  2>/dev/null
          echo "ct_read_machines_lst return code: $?" >> .${OUTDIR}/ct_read_machines_lst.out
    fi

	collect_hags_command_outputs cthags
  fi

  if [ "$IF_RMC_CONFIGED" = "yes" ]
  then
    comp=ct_clusterinfo 
    cmd="ct_clusterinfo > .${OUTDIR}/$comp.out"
    run_cmd
    cmd="ct_clusterinfo"
    eval $cmd >/dev/null 2>&1
    echo "ct_clusterinfo return code: $?" >> .${OUTDIR}/$comp.out

    comp=lsclcfg 
    cmd="lsclcfg > .${OUTDIR}/$comp.out"
    run_cmd
    cmd="lsclcfg"
    eval $cmd >/dev/null 2>&1
    echo "lsclcfg return code: $?" >> .${OUTDIR}/$comp.out
  fi

  # collect rpttr
  cmd="ls /usr/sbin/rsct/bin/rpttr >/dev/null 2>&1"
  run_cmd

  #collect ctversion output
  comp=ctversion
  cmd="/usr/sbin/rsct/install/bin/ctversion > .${OUTDIR}/$comp.out"
  run_cmd

  # Some security component command output
  comp=ctsvhbac
  cmd="/usr/sbin/rsct/bin/ctsvhbac > .${OUTDIR}/$comp.out"
  run_cmd

  comp=ctsvhbal
  cmd="/usr/sbin/rsct/bin/ctsvhbal > .${OUTDIR}/$comp.out"
  run_cmd

  comp=ctskeygen
  cmd="/usr/sbin/rsct/bin/ctskeygen -d > .${OUTDIR}/$comp.out"
  run_cmd

  comp=ctsthl
  cmd="/usr/sbin/rsct/bin/ctsthl -l > .${OUTDIR}/$comp.out"
  run_cmd

  # 
  cmd="ls /usr/sbin/rsct/cfg/ctsec_map.global > /dev/null 2>&1"
  run_cmd

  cmd="ls /usr/sbin/rsct/cfg/ctsec_map.local > /dev/null 2>&1"
  run_cmd

  # network informations:
  #-----------------------
  comp=ifconfig 
  cmd="ifconfig -a > .${OUTDIR}/$comp.out"
  run_cmd

  comp=netstat
  cmd="netstat > .${OUTDIR}/$comp.out"
  # check if netstat cmd hangs, if hangs then skip the cmds
  isTimeOut=0
  timeout_cmd 5 30                                                    
  if [ "$isTimeOut" = 0 ]                                                 
  then
    run_cmd

    comp=netstatr
    cmd="netstat -r > .${OUTDIR}/$comp.out"
    run_cmd

    comp=netstati
    cmd="netstat -i > .${OUTDIR}/$comp.out"
    run_cmd

    comp=netstatrn
    cmd="netstat -rn > .${OUTDIR}/$comp.out"
    run_cmd

    comp=netstatin
    cmd="netstat -in > .${OUTDIR}/$comp.out"
    run_cmd

    comp=netstats
    cmd="netstat -s > .${OUTDIR}/$comp.out"
    run_cmd
  fi
   
  if [ "$BASEOS" = "AIX" ]
  then 
    if [ "$isTimeOut" = 0 ]                                                 
    then
      comp=netstatd
      cmd="netstat -D > .${OUTDIR}/$comp.out"
      run_cmd

      comp=netstatm
      cmd="netstat -m > .${OUTDIR}/$comp.out"
      run_cmd
     
      comp=netstatA
      cmd="netstat -A > .${OUTDIR}/$comp.out"
      run_cmd

    fi

    comp=no
    cmd="no -a > .${OUTDIR}/$comp.out"
    run_cmd

  elif [ "$BASEOS" = "Linux" ]
  then
    #collect files in /proc/sys/net/ipv4/conf/ and /proc/sys/dev/
      new_dir=$LOGDIR/proc.sys.net.ipv4.conf
      if [[ -e $new_dir ]]
      then
      rm -rf $new_dir
      fi
      mkdir $new_dir

      cp -r /proc/sys/net/ipv4/conf $new_dir

      temp_bytes=`du -s -k $new_dir | awk '{print $1}'`
      total_bytes=$(($total_bytes + $temp_bytes))
      tmp_file_list="proc.sys.net.ipv4.conf $tmp_file_list"
      delete_list="$new_dir $delete_list"

      new_dir=$LOGDIR/proc.sys.dev
      if [[ -e $new_dir ]]
      then
        rm -rf $new_dir
      fi
      mkdir $new_dir

      cp -r /proc/sys/dev $new_dir

      temp_bytes=`du -s -k $new_dir | awk '{print $1}'`
      total_bytes=`expr $total_bytes + $temp_bytes`
      tmp_file_list="proc.sys.dev $tmp_file_list"
      delete_list="$new_dir $delete_list"
  fi

  # Miscellaneous command output
  #-----------------------------

  mkdir -p .${OUTDIR}/lsrsrc
  comp=lsrsrc_IBM.PeerDomain                                                    
  cmd="lsrsrc -t IBM.PeerDomain > .${OUTDIR}/lsrsrc/$comp.out"                  
  run_cmd                                                                       
                                                                                
  if [[ $ISSERVER = 1 || $ISHMC = 1 ]]
  then
      comp=lsrsrc_IBM.ManagedNode
      export CT_MANAGEMENT_SCOPE=1
      cmd="lsrsrc -ab IBM.ManagedNode > .${OUTDIR}/lsrsrc/$comp.out"
      run_cmd
  else
      comp=lsrsrc_IBM.ManagementServer
      export CT_MANAGEMENT_SCOPE=1
      cmd="lsrsrc -ab IBM.ManagementServer > .${OUTDIR}/lsrsrc/$comp.out"
      run_cmd
  fi

  CLNAME=$(/usr/sbin/rsct/bin/ct_clusterinfo -c)                                
  if [[ $? -ne 0 || "$CLNAME" != "IW" ]]                                      
  then                                                                          
      comp=lsrsrc_IBM.NetworkInterface                                          
      export CT_MANAGEMENT_SCOPE=2                                              
      cmd="lsrsrc -t IBM.NetworkInterface > .${OUTDIR}/lsrsrc/$comp.out"        
      run_cmd                                                                   
  else                                                                          
      comp=lsrsrc_IBM.NetworkInterface                                          
      cmd="lsrsrc -t IBM.NetworkInterface > .${OUTDIR}/lsrsrc/$comp.out"        
      run_cmd                                                                   
  fi                                                                            
                                                                                
  comp=lscomg                                                                   
  cmd="lscomg > .${OUTDIR}/lsrsrc/$comp.out"                                    
  run_cmd                                                                       


  comp=lsrpnode_i
  cmd="lsrpnode -i > .${OUTDIR}/lsrsrc/$comp.out"
  run_cmd

  MACHINE=`uname -m`
  if [[ $BASEOS = "Linux" && $MACHINE = "s390" ]]
  then
       if [[ -f /sbin/lsdasd ]]
       then
           comp=lsdasd
           cmd="/sbin/lsdasd  > ./${OUTDIR}/$comp.out"
       else
           comp=dasd
           cmd="cat /proc/dasd/devices > ./${OUTDIR}/$comp.out"
       fi
       run_cmd
  fi

  comp=df
  cmd="df -k  > .${OUTDIR}/$comp.out"
  run_cmd

  comp=ps
  cmd="ps -ef >  .${OUTDIR}/$comp.out"
  run_cmd

  comp=lssrc.a
  cmd="lssrc -a > .${OUTDIR}/$comp.out"
  run_cmd

  comp=vmstat
  cmd="vmstat 1 3 > .${OUTDIR}/$comp.out"
  run_cmd

  comp=lsaudrec
  cmd="lsaudrec -l | tail -100 > .${OUTDIR}/$comp.out"
  run_cmd

  if [ "$BASEOS" = "AIX" ]
  then
    comp=vmstats
    cmd="vmstat -s > .${OUTDIR}/$comp.out"
    run_cmd

    comp=lslpp
    cmd="lslpp -ha > .${OUTDIR}/$comp.out"
    run_cmd

    comp=lppchk
    cmd="lppchk -c rsct.* > .${OUTDIR}/$comp.out"
    run_cmd
  fi

  comp=rpm
  cmd="rpm -qa > .${OUTDIR}/$comp.out"
  run_cmd

  comp=uptime
  cmd="uptime > .${OUTDIR}/$comp.out"
  run_cmd

  comp=rmcdomainstatus
  cmd="rmcdomainstatus -s ctrmc > .${OUTDIR}/$comp.out"
  run_cmd

  if [[ $ISHMC = 1 ]]
  then
    comp=hsc
    cmd="hsc version > .${OUTDIR}/$comp.out"
    run_cmd

    comp=lslpars
    cmd="lslpars > .${OUTDIR}/$comp.out"
    run_cmd

    comp=getcimstatus
    cmd="getcimstatus > .${OUTDIR}/$comp.out"
    run_cmd
	
    comp=lspartition
    cmd="lspartition -dlpar > .${OUTDIR}/$comp.out"
    run_cmd

    comp=lsnode
    cmd="lsnode -H > .${OUTDIR}/$comp.out 2>&1"
    run_cmd

  elif [[ $ISSERVER = 1 ]]
  then
    set -A hwclist `lsrsrc IBM.ManagedNode HWControlPoint | grep "HWControlPoint =" | cut -f2 -d '"'`
    set -A hwplist `lsrsrc IBM.ManagedNode PowerMethod | grep "PowerMethod =" | cut -f2 -d '"'`
    let i=0
    firstone=0
    rm tmpfile 2> /dev/null
    while [[ -n ${hwclist[$i]} ]]
    do
      echo "${hwclist[$i]} ${hwplist[$i]}" >> tmpfile
      let i=i+1
    done
    sort -u tmpfile > tmpfile2
    rm tmpfile
    {
      while read hwc hwp
      do
        comp=lshwinfo
        cmd="lshwinfo -p $hwp -c $hwc >> .${OUTDIR}/$comp.out"
        run_cmd 
      done
    } < tmpfile2
    rm tmpfile2


    comp=lsnode
    cmd="lsnode -l > .${OUTDIR}/$comp.out 2>&1"
    run_cmd

  fi


} # End of doit

##############################################################################
# Function: do_hacmp_on_linux
#
# Collect information pertinent to HACMP running on Linux. The "doit" funtion
# (above) is NOT executed when this function is.
##############################################################################

function do_hacmp_on_linux {

  #-------------------
  #collect the files in /var/ha (subdirs: lck, log, run only)
  #-------------------------
  cmd="ls /var/ha/lck >/dev/null 2>&1"
  run_cmd
  cmd="ls /var/ha/log >/dev/null 2>&1"
  run_cmd
  cmd="ls /var/ha/run >/dev/null 2>&1"
  run_cmd

  #-------------------
  #collect the ls -lR info of /var/ha/soc
  #-------------------------
  ls -lR /var/ha/soc > .${OUTDIR}/var_ha_soc_ls_lR.out

  #-------------------
  #override subsystem names when clustering is HACMP on Linux
  #------------------------------------
  HATS=topsvcs
  HAGS=grpsvcs

  #-------------------
  #collect HATS/HAGS info if installed
  #------------------------------------
  export LC_ALL=C

  lssrc -s $HATS >/dev/null 2>&1
  if [ "$?" = 0 ]
  then
    comp=lssrc_ls_$HATS
    cmd="lssrc -l -s $HATS > .${OUTDIR}/$comp.out"
    run_cmd

    comp=lssrc_ls_$HAGS
    cmd="lssrc -l -s $HAGS > .${OUTDIR}/$comp.out"
    run_cmd

	collect_hags_command_outputs $HAGS

    #-------------------
    #if any core files exist, collect daemon executables
    #------------------------------------
    total_core_files=`find /var/ha/run -name 'core*' -print 2> /dev/null | wc -l`

	if [ $total_core_files -gt 0 ]
	then
		for i in hagsd hatsd hats_nim
		do
		  cmd="ls $RSCTBINDIR/$i >/dev/null 2>&1"
		  run_cmd
		done
	fi
  fi

  #-------------------
  #collect various things clsnap does not
  #------------------------------------

  new_name=`ls /var/log/messages*`
  for i in $new_name
  do
     cmd="ls $i >/dev/null 2>&1"
     run_cmd
  done

  #collect files in /proc/sys/net/ipv4/conf/ and /proc/sys/dev/

  new_dir=$LOGDIR/proc.sys.net.ipv4.conf
  if [[ -e $new_dir ]]
  then
  rm -rf $new_dir
  fi
  mkdir $new_dir

  cp -r /proc/sys/net/ipv4/conf $new_dir

  temp_bytes=`du -s -k $new_dir | awk '{print $1}'`
  total_bytes=$(($total_bytes + $temp_bytes))
  tmp_file_list="proc.sys.net.ipv4.conf $tmp_file_list"
  delete_list="$new_dir $delete_list"

  new_dir=$LOGDIR/proc.sys.dev
  if [[ -e $new_dir ]]
  then
    rm -rf $new_dir
  fi
  mkdir $new_dir

  cp -r /proc/sys/dev $new_dir

  temp_bytes=`du -s -k $new_dir | awk '{print $1}'`
  total_bytes=`expr $total_bytes + $temp_bytes`
  tmp_file_list="proc.sys.dev $tmp_file_list"
  delete_list="$new_dir $delete_list"

  comp=df
  cmd="df -k  > .${OUTDIR}/$comp.out"
  run_cmd

  comp=vmstat
  cmd="vmstat 1 3 > .${OUTDIR}/$comp.out"
  run_cmd

  comp=lsaudrec
  cmd="lsaudrec -l | tail -100 > .${OUTDIR}/$comp.out"
  run_cmd

  comp=rpm
  cmd="rpm -qa > .${OUTDIR}/$comp.out"
  run_cmd

  comp=uptime
  cmd="uptime > .${OUTDIR}/$comp.out"
  run_cmd

} # End of do_hacmp_on_linux

#############################################
# MAIN
#############################################
logdate=`date +\%m\%d\%H\%M`

userid=`id -ru`
if [ "$userid" != 0 ]
then
  echo "Must be root user [0] to use this utility."
  exit 1
fi
verbal=0
while getopts "ahvd:" flag
do
    case $flag in
    a)   hacmp_on_linux=1
         ;;
    h)   usage
         exit 0
         ;;
    v)   verbal=1
         ;;
    d)   d_argument=$OPTARG
         LOGDIR=$d_argument
         valid_dir $LOGDIR
         ;;
    :)   print "You forgot to enter argument to $OPTARG"
         usage
         exit 1
         ;;
    \?)  usage
         exit 1
         ;;
    esac
done

BASEOS=`uname`

if [[ "1" = $hacmp_on_linux ]] && [[ $BASEOS != "Linux" ]]
then
	echo "-a flag is not valid for $BASEOS Operating system"
	exit 1
fi

cd $ROOTDIR
flist=""
tmp_file_list="ctsnap_out"
out_file=""
delete_list=""
max_link_count=32

mkdir -p ${LOGDIR}

if [[ "0" = $hacmp_on_linux ]]
then
	if [[ $BASEOS = "Linux" ]]
	then
		/opt/hsc/bin/hsc version > /dev/null 2> /dev/null
		if [[ $? = 0 ]]
		then
			ISHMC=1			# this is an HMC
			PATH="$PATH:/opt/hsc/bin:/opt/csm/bin"
	    else	
	    rpm -qa | grep csm.server- > /dev/null 2> /dev/null
	    if [[ $? = 0 ]]
	    then
	        value=`runact-api -c IBM.DmsCtrl::::isLicenseValid | cut -f5 -d':'`
	        if [[ $value = 1 ]]
            then
                 ISSERVER=1     #this is a Linux CSM Management Server
                 PATH="$PATH:/opt/csm/bin"
	        fi
	    fi
	fi
	else
		lslpp -L csm.server > /dev/null 2> /dev/null
		if [[ $? = 0 ]]
		then
    		value=`runact-api -c IBM.DmsCtrl::::isLicenseValid | cut -f5 -d':'`
    		if [[ $value = 1 ]]
			then
				ISSERVER=1      #this is an AIX CSM Management Server
				PATH="$PATH:/opt/csm/bin"
			fi
		fi
	fi
fi

HOST=`hostname -s`
TARFILE=${LOGDIR}/ctsnap.${HOST}.${logdate}.tar
LOGFILE=${LOGDIR}/ctsnap.${HOST}.${logdate}.log
exec 3> $LOGFILE

OUTDIR=${LOGDIR}/ctsnap_out
if [[ -e $OUTDIR ]]
then
  rm -rf $OUTDIR
fi
mkdir -p ${OUTDIR}

IF_RMC_CONFIGED="no"
IF_HATS_HAGS_CONFIGED="no"
HAS_RMC_CORE="no"
HAS_HATS_CORE="no"
HAS_HAGS_CORE="no"
HAS_HAGSGLSM_CORE="no"

date >> $LOGFILE 2>&1
echo "Log directory set to ${LOGDIR}" >> $LOGFILE 2>&1
echo "Log directory set to ${LOGDIR}" 

echo "Gathering information......" >> $LOGFILE 2>&1

if [[ "1" = $hacmp_on_linux ]]
then
	do_hacmp_on_linux
else
	doit
fi

echo ""

echo "Starting tar/compress process......" >> $LOGFILE 2>&1
flist="$flist"
set $flist

if [[ -e ${TARFILE} ]]
then
    tar -uf ${TARFILE}  -C $LOGDIR $tmp_file_list  -C $ROOTDIR "$@" 2>/dev/null
else
    tar -cf ${TARFILE}  -C $LOGDIR $tmp_file_list  -C $ROOTDIR "$@" 2>/dev/null 
fi

if [[ -e ${TARFILE} ]]
then
    which gzip  > /dev/null 2>&1
    if [[ "$?" = 0 ]]
    then
        gzip -f ${TARFILE}
    else
        which compress  > /dev/null 2>&1
        if [[ "$?" = 0 ]]
        then
            compress -f ${TARFILE}
        else
            echo "gzip and compress commands don't exist and the tar file ${TARFILE} is not compressed." >> $LOGFILE 2>&1
        fi
    fi
fi

GZFILE=${TARFILE}.gz
ZFILE=${TARFILE}.Z
out_file=".$OUTDIR $delete_list"
set $out_file
if [[ -e ${TARFILE} || -e ${GZFILE} || -e ${ZFILE} ]]
then
    rm -rf "$@"
fi

echo "*******done*******" >> $LOGFILE 2>&1

chmod 400 $LOGFILE 
if [[ -e ${GZFILE} ]]
then
    chmod 400 ${GZFILE}
elif [[ -e ${ZFILE} ]]
then
    chmod 400 $ZFILE
elif [[ -e ${TARFILE} ]]
then
    chmod 400 $TARFILE
else
    chmod 400 $out_file
fi
exit 0
