#!/bin/ksh
#
#pragma ident "@(#)loghost_sync   1.24     98/06/15 SMI"
#
#	Copyright (c) 1997 Sun Microsystems, Inc.
#
# loghost_sync - Logical Host synchronization command
#
# The purpose of this command is to execute on all nodes whenever the
# row corresponding to the LOGHOST_CM format is updated in the
# CCD. Whenever a row of this format is removed, the corresponding
# logical host along with its associated data services is shut down on
# the master node of the logical host. When a row of this format is
# added to the CCD, the corresponding logical host and its associated
# data services is brought up on the current master.
#
# The arguments obtained by this command are:
#
# <keyword> - can be one of CCDSYNC_PRE_ADDU, CCDSYNC_POST_ADDU,
#			    CCDSYNC_PRE_REMOVE, CCDSYNC_POST_REMOVE
#
# <row>     - the row being added/removed
#
# <ccdfile> - the full pathname of the dynamic CCD file. This is
#	      required because the dynamic CCD cannot be queried in
#	      the conventional manner from this command. It has to be
#	      queried as a static database.
#
# <numnodes> - integer identifying the number of nodes in the static
#	       cluster configuration.
#
# <curr_members> - list of node ids identifying the current cluster
#		   membership.
#
# <localnodeid> - node id of this node.

keyword=$1

case ${keyword} in
	CCDSYNC_PRE_RESTORE | CCDSYNC_PRE_RESTOREKEY | \
		CCDSYNC_POST_RESTORE | CCDSYNC_POST_RESTOREKEY)
			row=""
			CCDFILE=$2
			numnodes=$3
			curr_members=$4
			localnodeid=$5 ;;
	*)
			row=$2
			CCDFILE=$3
			numnodes=$4
			curr_members=$5
			localnodeid=$6 ;;
esac
		
# This should be removed later
export CCDFILE

pre="SUNWcluster.loghost"
INCLUDE=.
RECONF_SCRIPTS=/opt/SUNWcluster/etc/reconf/scripts/
${INCLUDE} ${RECONF_SCRIPTS}/reconf_ener.disks

###########################################################################
#
# function init
#
# The init function just initializes a lot of global variables. It also
# computes the dependency of the data services sitting on top of this
# logical host, and initializes arrays that corresponds to the data
# services's state, the logical hosts of the data service mastered on this
# node, not mastered on this node, etc.

function init
{
typeset loghost_row
typeset r rows
typeset ds_list dsname
typeset ordered_ds_list
typeset all_loghosts lm_loghosts nm_loghosts
typeset n
typeset pdbapps


# Cluster Application Bit Assignment. Refer to the cdb file for all
# assignments

CVM=3
VxVM=4
Netdisk=6

CLUSTERBIN=/opt/SUNWcluster/bin
CLUSTERVAR=/var/opt/SUNWcluster
CLUSTERETC=/etc/opt/SUNWcluster/conf
SSACLI=${CLUSTERBIN}/scssa

CLUSTERNAME=$(/bin/cat ${CLUSTERETC}/default_clustername)
cdbfile=${CLUSTERETC}/${CLUSTERNAME}.cdb
localhostname=$(${CLUSTERBIN}/cdbmatch cluster.node.${localnodeid}.hostname \
			${cdbfile})
clusterstate=$(${CLUSTERBIN}/clustm getstate ${CLUSTERNAME})

pdbapps=$(${CLUSTERBIN}/cdbmatch cluster.pdbapps ${cdbfile})
vm=$(${CLUSTERBIN}/appmatch ${pdbapps} ${CVM})
if [[ "${vm}" = "1" ]]; then
  vm=cvm
else
  vm=$(${CLUSTERBIN}/appmatch ${pdbapps} ${VxVM})
  if [[ "${vm}" = "1" ]]; then
    vm=vxvm
  fi
fi
nd=$(${CLUSTERBIN}/appmatch ${pdbapps} ${Netdisk})

let n=0
allnodes=""
while (( n < ${numnodes} )); do
  allnodes="${allnodes} ${n}"
  let n=n+1
done

# retrieve name of logical host and current master

lname=${row%:*}
lname=${lname#*:}
curr_master=${row#*:*:}

loghost_row=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} LOGHOST query \
			lname ${lname})
if [[ -z "${loghost_row}" ]]; then
  log_info "${pre}.4010" "$0 ${keyword} ${row}: The definition of logical host ${lname} does not exist in ${CCDFILE}"
  exit 1
fi

# split the information in loghost_row

dglist=${loghost_row%:*:*}
dglist=${dglist#*:*:*:}
dglist=$(print ${dglist} | tr ',' ' ')

nodelist=${loghost_row%:*:*:*}
nodelist=${nodelist#*:*:}
nodelist=$(print ${nodelist} | tr ',' ' ')

let in_nodelist=0
for nodename in ${nodelist}; do
  if [[ "${nodename}" = "${localhostname}" ]]; then
    let in_nodelist=1
    break
  fi
done

iplist=${loghost_row%:*}
iplist=${iplist#*:*:*:*:}
iplist=$(print ${iplist} | tr ',' ' ')

# get list of all data services on this logical host

rows=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} LOGHOST_DS query \
		lname ${lname})
ds_list=""
for r in ${rows}; do
  dsname=${r#*:*:}
  ds_list="${ds_list} ${dsname}"
done

ordered_ds_list=""
get_ordered_ds_list "${ds_list}" ordered_ds_list
ordered_ds_list=$(print ${ordered_ds_list} | tr ',' ' ')

# initialize arrays

set -A ordered_ds_array ${ordered_ds_list}
set -A all_loghosts_array
set -A lm_loghosts_array
set -A nm_loghosts_array
set -A ds_state_array

let n=0
while (( n < ${#ordered_ds_array[*]} )); do
  all_loghosts=""
  lm_loghosts=""
  nm_loghosts=""
  construct_method_args_list ${ordered_ds_array[n]} all_loghosts lm_loghosts \
		nm_loghosts
  all_loghosts_array[n]=${all_loghosts}
  lm_loghosts_array[n]=${lm_loghosts}
  nm_loghosts_array[n]=${nm_loghosts}

  # retrieve the state of the data service

  rows=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} DS_STATE query \
			ds_name ${ordered_ds_array[n]})
  ds_state_array[n]=${rows#*:*:}
  let n=n+1
done
}

###########################################################################
#
# function cvm_deport dglist
#
# dglist - The list of disk groups to be deported
#
function cvm_deport
{
typeset g retval retval2

if [ $# -gt 0 ]; then
  for g in $*; do
    retval=0
    /usr/sbin/vxdg list "${g}" > /dev/null 2>&1 || retval=$?
    
    if [[ "${retval}" -eq 0 ]]; then
      log_info "$pre.1180" "deporting ${g}"

      retval2=0
      /usr/sbin/vxdg deport "${g}" || retval2=$?
      
      if [ "${retval2}" -eq 31 ]; then
	  log_info "$pre.5020" "Disk group ${g} busy. Deport failed"
	  return 1
      fi
    fi
  done
fi

return 0
}

###########################################################################
#
# function cvm_import dglist
#
# dglist - list of diskgroups to be imported
#
function cvm_import
{
typeset -i defined_vols
typeset -i startable_vols
typeset g retval retval2

if [ $# -gt 0 ]; then
  for g in $*; do
    retval=0
    /usr/sbin/vxdg list "${g}" > /dev/null 2>&1 || retval=$?
    
    if [[ "${retval}" -ne 0 ]]; then
	log_info "$pre.1170" "importing ${g}"
	retval2=0
	/usr/sbin/vxdg -fCt import "${g}" || retval2=$?
	if [[ "${retval2}" -ne 0 ]]; then
	   if [[ "${retval2}" -eq 20 ]]; then
	     log_info "$pre.4035" "NOTICE: Check disk group ${g} for failed disks \
or other missing components"
	   fi
	   log_info "$pre.4030" "Import of disk group ${g} failed"
	   return ${retval2}
	else
# now we want to check to see if this dg is actually ok do work with.
	  startable_vols=$(/usr/sbin/vxrecover -g "${g}" -np | wc -l )
	  defined_vols=$(/usr/sbin/vxprint -g "${g}" -S -v | /usr/bin/tail -1 \
		| awk '{print $1}')
	  if (( defined_vols == startable_vols )); then
# we are ok, ie we have at least one plex in each volume that is ok
# and thus we are startable.
		/usr/sbin/vxrecover -g "${g}" -sb  
	  else
# the dg has at least some plexes that cannot even be assembled. stop
# what we are doing and abort this logical host.
	        log_info "$pre.4030" "Import of disk group ${g} failed"
		log_info "$pre.4035" "NOTICE: Check disk group ${g} for \
			failed disks or other missing components"
# just to be safe, deport the dg that is in question.
		/usr/sbin/vxdg deport "${g}" >/dev/null 2>&1
		return 255

          fi
        fi
    fi
  done
fi

return 0
}

#########################################################################
#
#	Remove DA name for disks of private diskgroups that have been
#	ndshared. Only arguments expected are dg name and local host name.
#
function unndshare_privdg_disks
{
nodename="${2}";
netdisks=`vxdisk -s list | nawk 'BEGIN {
		rmdg="'${1}'";
	}
	(/type:/ || /masters:/ || /diskid:/ || /dgid:/ || /hostid:/) {
		next;
	}
	($1 == "Disk:") {
		  daname = $2;
		next;
	}
	($1 == "flags:") {
		$1 = "";
		flags = $0;
		next;
	}
	($1 == "dgname:") {
		if ($2 == rmdg)
		{
			if (match(flags, "private") &&
			match(flags, "remote"))
			print daname;
			# print rmdg, daname, flags >> "'${CLUSTERVAR}'/cvm.log";
		}
		next;
	}' | tr '\012' ' '`;
	
[ -n "${netdisks}" ] || return 0;

nddevs=`/usr/sbin/vxdisk list ${netdisks} |
sed -n -e "s/pubpaths:.*\/nd//p" -e "s/privpaths:.*\/nd//p"`;

for da in ${netdisks};
do
  ndev=`/usr/sbin/vxdisk list ${da}|sed -n -e "s/pubpaths:.*\/nd//p"`;
  vxdisk rm ${da} || true;
  [ -n "${ndev}" ] || continue;
#  path=`${CLUSTERBIN}/ndshare -q -m ${ndev} | nawk -F: '{
#    nhost = split($4, hosts, ",");
#    npath = split($5, paths, ",");
#    for (i = 1; i <= nhost; i++)
#      if (hosts[i] == "'${nodename}'")
#	{
#	  print paths[i];
#	  exit;
#	}
#  }'`;
  
  [ -n "${path}" ] && vxdisk define `basename ${path} s4` || true;
done;

#for ndev in ${nddevs}; do
#  ${CLUSTERBIN}/ndshare -u -n nd${ndev} || true;
#done;

unset da nddevs ndev netdisks path;
return 0;
}

###########################################################################
#
# function construct_method_args_list i_dsname o_all_loghosts o_lm_loghosts
#				      o_nm_loghosts
#
# i_dsname - The data service name forms the input parameter
#
# o_all_loghosts - An output parameter that is a list of all the logical
#		   hosts in the cluster configuration offering this data
#		   service.
#
# o_lm_loghosts - An output parameter that is a list of all logical hosts
#		  that this physical node is supposed to master that is
#		  offering this data service.
#
# o_nm_loghosts - An output parameter that is a list of all logical hosts
#		  that this physical node is not supposed to master that is
#		  offering this data service.

function construct_method_args_list
{
typeset i_dsname
typeset o_all_loghosts
typeset o_lm_loghosts
typeset o_nm_loghosts
typeset cm_row cm
typeset r rows
typeset loghost_name
typeset tmp_list1 tmp_list2
i_dsname=$1

# get list of all logical hosts offering this data service

rows=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} LOGHOST_DS query \
			dsname ${i_dsname})

o_all_loghosts=""
o_lm_loghosts=""
o_nm_loghosts=""

for r in ${rows}; do
  loghost_name=${r%:*}
  loghost_name=${loghost_name#*:}
  if [[ -z "${o_all_loghosts}" ]]; then
    o_all_loghosts=${loghost_name}
  else
    o_all_loghosts="${o_all_loghosts},${loghost_name}"
  fi

  # special handling is required for the abort case, because the LOGHOST_CM
  # entries are not removed from the CCD and no decisions about mastered and
  # not-mastered lists should be made based on these entries. After the abort
  # transition, we know that this node will not own anything, so we stick
  # everything into the not-mastered list.

  if [[ "${clusterstate}" = "abort" ]]; then
    if [[ -z "${o_nm_loghosts}" ]]; then
      o_nm_loghosts=${loghost_name}
    else
      o_nm_loghosts="${o_nm_loghosts},${loghost_name}"
    fi
    continue
  fi

  # retrieve the current master of the logical host

  cm_row=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} LOGHOST_CM query \
			lname ${loghost_name})
  cm=${cm_row#*:*:}

  if [[ ( -n "${cm}" && "${cm}" = "${localhostname}" ) || \
        ( "${loghost_name}" = "${lname}"  && "${curr_master}" = "${localhostname}" ) \
     ]]; then
    if [[ -z "${o_lm_loghosts}" ]]; then
      o_lm_loghosts=${loghost_name}
    else
      o_lm_loghosts="${o_lm_loghosts},${loghost_name}"
    fi
  else
    if [[ -z "${o_nm_loghosts}" ]]; then
      o_nm_loghosts=${loghost_name}
    else
      o_nm_loghosts="${o_nm_loghosts},${loghost_name}"
    fi
  fi
done

# if we are removing the LOGHOST_CM row corresponding to lname, remove
# lname from the o_lm_loghosts list, and add it to the o_nm_loghosts list
 
if [[ "${keyword}" = "CCDSYNC_PRE_REMOVE" ]]; then
  tmp_list1=$(print ${o_lm_loghosts} | tr ',' ' ')
  tmp_list2=""
  for l in ${tmp_list1}; do
    if [[ "${l}" != "${lname}" ]]; then
      if [[ -z "${tmp_list2}" ]]; then
	tmp_list2=${l}
      else
	tmp_list2="${tmp_list2},${l}"
      fi
    else
      if [[ -z "${o_nm_loghosts}" ]]; then
	o_nm_loghosts=${l}
      else
	o_nm_loghosts="${o_nm_loghosts},${l}"
      fi
    fi
  done
  o_lm_loghosts=${tmp_list2}
fi


eval $2=${o_all_loghosts}
eval $3=${o_lm_loghosts}
eval $4=${o_nm_loghosts}
}

###########################################################################
#
# function ifconfig_ip mode logif_list
#
# mode - The mode of the logical interface - can be "up" or "down"
#
# logif_list - The list of logical interfaces that has to be "ifconfig'ed"
#	       according to the specified mode for the logical host
#	       identified by the global variable ${lname}

function ifconfig_ip
{
typeset mode
typeset logif_list l iprow
typeset ip_nodelist nodearray
typeset iflist ifarray
typeset ipaddr
typeset localnode_index n found
typeset retval retval2
typeset arp_list
typeset nafogroup
typeset nafoif


mode=$1
shift
logif_list=$*
arp_list=""

for l in ${logif_list}; do

  # retrieve the logical IP information for this logical interface
  iprow=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} LOGIP query \
		logif ${l})
  if [[ -z "${iprow}" ]]; then
    continue
  fi

  ip_nodelist=${iprow%:*:*:*}
  ip_nodelist=${ip_nodelist#*:}
  ip_nodelist=$(print ${ip_nodelist} | tr ',' ' ')

  iflist=${iprow%:*:*}
  iflist=${iflist#*:*:}
  iflist=$(print ${iflist} | tr ',' ' ')

  ipaddr=${iprow%:*}
  ipaddr=${ipaddr#*:*:*:}

  set -A nodearray ${ip_nodelist}
  set -A ifarray ${iflist}

  # search for my node index

  let n=0
  let found=0
  while (( n < ${#nodearray[*]} )); do
    if [[ "${nodearray[n]}" = "${localhostname}" ]]; then
      let found=1
      let localnode_index=${n}
      break
    fi
    let n=n+1
  done
  if (( found == 0 )); then
    continue
  fi

  #
  # Usage of PNM.
  # To the interface which is of real significance , get the active
  # interface for the backup group which the interface belongs
  # XXX: For temporary stuff , check if PNM package exists,if
  # assuming that people will use PNM.
  #
  USE_NAFO=0
  /usr/bin/pkginfo SUNWpnm >/dev/null 2>&1
  RC=$?
  if [ ${RC} -eq 0 ]; then
        USE_NAFO=1
  fi
  if [ ${USE_NAFO} -eq 1 ]; then
        nafogroup=`/opt/SUNWpnm/bin/pnmrtop ${ifarray[localnode_index]}`
        RC=$?
        if [ ${RC} -ne 0 ]; then
                if [ "${mode}" = "up" ]; then
                  #
                  # pnm should have been up and configured.
                  # hence this is an error.
                  #
                  log_info "$pre.4900" "error in getting the PNM backup group for ${ifarray[localnode_index]}. Please Check if interface is configured on a PNM backup group  OR check console for errors messages from PNM"
                  return 1
                fi
                # 
                # PNM failed but it is possible that loghost/pnm could be
                # shutdown before calling the logical host.(reconfiguration)
                # to have an idempotent operation, if we supposed to shutdown
                # (ifconfig down) then ignore the error and return.
                # Hence we do nothing and assume that real adaptor will
                # is the one cosnideration.

        else
          # 
          # now that pnm group is found, we get the active interface.
          #
          nafoif=`/opt/SUNWpnm/bin/pnmptor ${nafogroup}`
          RC=$?
          if [ ${RC} -ne 0 ]; then
                #
                # we cannot get the pnmptor and this means that we do
                # do not have any active interface on the PNM
                #
                if [ "${mode}" = "up" ]; then
                  log_info "$pre.4901" "There seems to be no active interface for PNM backup group ${nafogroup}. Please Check if interface is configured on a PNM backup group  OR check console for errors messages from PNM"
                  return 1
                fi
         else
                ifarray[localnode_index]=${nafoif}
         fi
      fi   
  fi






  # check whether the logical interface is already up

  retval=0
  /usr/sbin/ifconfig -au | /bin/grep -w ${ifarray[localnode_index]}:${l} \
	> /dev/null 2>&1 || retval=$?
  if [[ "${retval}" -eq 0 ]]; then
    
    # interface is up. ifconfig it down if mode is down
    if [[ "${mode}" = "down" ]]; then
      retval2=0
      /usr/sbin/ifconfig ${ifarray[localnode_index]}:${l} 0.0.0.0 down || \
		retval2=$?
      if [[ "${retval2}" -ne 0 ]]; then
	log_info "$pre.4020" "ifconfig ${ifarray[localnode_index]}:${l} down failed"
	return 1
      fi
    fi
    
  elif [[ "${retval}" -eq 1 ]]; then

    # interface is down. ifconfig it up if mode is up

    if [[ "${mode}" = "up" ]]; then
      retval2=0
      /usr/sbin/ifconfig ${ifarray[localnode_index]}:${l} ${ipaddr} netmask + \
		broadcast + -trailers up || retval2=$?
      if [[ "${retval2}" -ne 0 ]]; then
	log_info "$pre.4010" "ifconfig ${ifarray[localnode_index]}:${l} ${ipaddr} up failed"
	return 1
      fi
      arp_list="${arp_list} ${ipaddr}@${ifarray[localnode_index]}"
    fi
  fi
done

if [[ "${mode}" = "up" && -n "${arp_list}" ]]; then
  ${CLUSTERBIN}/sc_retrans_arp ${CLUSTERNAME} 5 retransarp.${lname} ${arp_list} &
elif [[ "${mode}" = "down" ]]; then
  killarp ${lname}
fi

return 0
}

###########################################################################
#
# function killarp i_lname
#
# i_lname - input logical host name
#
# This function kills any remnants of the arp program running on this node
# for the input logical host.

function killarp
{
typeset i_lname
typeset dummy pid

i_lname=$1
if [[ -f "${CLUSTERVAR}/retransarp.${i_lname}" ]]; then
  pid=$(/bin/cat ${CLUSTERVAR}/retransarp.${i_lname})
  kill -9 ${pid} || dummy=$?
  if [[ -f "${CLUSTERVAR}/retransarp.${i_lname}" ]]; then
    /bin/rm -f ${CLUSTERVAR}/retransarp.${i_lname}
  fi
fi
}

###########################################################################
#
# function get_ordered_ds_list i_ds_list o_ds_list
#
# i_ds_list - The input list of data services.
#
# o_ds_list - The output list of data services is the ordered list of input
#	      data services ordered according to the dependency between the
#	      serivces.
#
function get_ordered_ds_list
{
typeset i_ds_list
typeset o_ds_list tmp_ds_list
typeset rescan_list dependency_list
typeset found
typeset i j k

i_ds_list=$1
rescan_list=""

while (( 1 )); do
  for i in ${i_ds_list}; do

    dependency_list=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} \
		DS_DEPSVC_LIST query ds_name ${i})
    dependency_list=${dependency_list#*:*:}
    dependency_list=$(print ${dependency_list} | tr ',' ' ')

    # If the data service has no dependencies, put it in the output list.
    # Otherwise, evaluate each of the dependent data services for other
    # dependencies

    if [[ -z "${dependency_list}" ]]; then
      if [[ -z "${o_ds_list}" ]]; then
	o_ds_list=${i}
      else
	o_ds_list="${o_ds_list},${i}"
      fi
    else
      let found=0
      for j in ${dependency_list}; do
	tmp_ds_list=$(print ${o_ds_list} | tr ',' ' ')
	for k in ${tmp_ds_list}; do
	  if [[ "${j}" = "${k}" ]]; then
	    let found=1
	    break
	  fi
	done
	if (( found == 0 )); then
	  break
	fi
      done

      if (( found == 0 )); then
	rescan_list="${rescan_list} ${i}"
      else
	if [[ -z "${o_ds_list}" ]]; then
	  o_ds_list=${i}
	else
	  o_ds_list="${o_ds_list},${i}"
	fi
      fi
    fi
  done

  i_ds_list=${rescan_list}
  rescan_list=""
  if [[ -z "${i_ds_list}" ]]; then
    break
  fi
done

eval $2=${o_ds_list}
}

###########################################################################
#
# function exec_ds_method method_type
#
# method_type - input keyword which has one of the following values: "start",
#		"start_net", "stop", "stop_net", "abort", "abort_net",
#		"fm_init", "fm_start", "fm_stop", "fm_check".
#
# This function executes the method identified by the method_type parameter
# for all the data services in the ordered_ds_list array according to their
# dependency order.

function exec_ds_method
{
typeset method_type
typeset dsname dsrow
typeset method_path method_to
typeset n
method_type=$1
if [[ "${method_type}" = "start" || "${method_type}" = "start_net" || \
      "${method_type}" = "fm_init" || "${method_type}" = "fm_start" ]]; then
  
  let n=0
  while (( n < ${#ordered_ds_array[*]} )); do
    dsname=${ordered_ds_array[n]}
    if [[ "${method_type}" = "start" ]]; then

      dsrow=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} DS_METHOD_PATH \
		query ds_name ${dsname})
      method_path=${dsrow%:*:*:*:*:*}
      method_path=${method_path#*:*:}

      method_to=${dsrow%:*:*:*:*}
      method_to=${method_to#*:*:*:}

    elif [[ "${method_type}" = "start_net" ]]; then

      dsrow=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} DS_NM_PATH \
		query ds_name ${dsname})
      method_path=${dsrow%:*:*:*:*:*}
      method_path=${method_path#*:*:}

      method_to=${dsrow%:*:*:*:*}
      method_to=${method_to#*:*:*:}

    elif [[ "${method_type}" = "fm_init" || "${method_type}" = "fm_start" ]]; then

      dsrow=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} DS_FM_PATH \
		query ds_name ${dsname})
      
      if [[ "${method_type}" = "fm_init" ]]; then
	method_path=${dsrow%:*:*:*:*:*:*:*}
	method_path=${method_path#*:*:}

	method_to=${dsrow%:*:*:*:*:*:*}
	method_to=${method_to#*:*:*:}
	
      elif [[ "${method_type}" = "fm_start" ]]; then
	method_path=${dsrow%:*:*:*:*:*}
	method_path=${method_path#*:*:*:*:}

	method_to=${dsrow%:*:*:*:*}
	method_to=${method_to#*:*:*:*:*:}
      fi
    fi

    if [[ -n "${method_path}" ]]; then
      call_method ${method_type} ${n} ${method_path} ${method_to}
    fi

    let n=n+1
  done

elif [[ "${method_type}" = "stop" || "${method_type}" = "stop_net" || \
	"${method_type}" = "abort" || "${method_type}" = "abort_net" || \
	"${method_type}" = "fm_stop" ]]; then

  let n=${#ordered_ds_array[*]}-1
  while (( n >= 0 )); do
    dsname=${ordered_ds_array[n]}

    if [[ "${method_type}" = "stop" || "${method_type}" = "abort" ]]; then
      dsrow=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} DS_METHOD_PATH \
		query ds_name ${dsname})

      if [[ "${method_type}" = "stop" ]]; then
	method_path=${dsrow%:*:*:*}
	method_path=${method_path#*:*:*:*:}

	method_to=${dsrow%:*:*}
	method_to=${method_to#*:*:*:*:*:}
	
      elif [[ "${method_type}" = "abort" ]]; then
	method_path=${dsrow%:*}
	method_path=${method_path#*:*:*:*:*:*:}

	method_to=${dsrow#*:*:*:*:*:*:*:}
      fi

    elif [[ "${method_type}" = "stop_net" || "${method_type}" = "abort_net" ]]; then
      dsrow=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} DS_NM_PATH \
		query ds_name ${dsname})

      if [[ "${method_type}" = "stop_net" ]]; then
	method_path=${dsrow%:*:*:*}
	method_path=${method_path#*:*:*:*:}

	method_to=${dsrow%:*:*}
	method_to=${method_to#*:*:*:*:*:}

      elif [[ "${method_type}" = "abort_net" ]]; then
	method_path=${dsrow%:*}
	method_path=${method_path#*:*:*:*:*:*:}

	method_to=${dsrow#*:*:*:*:*:*:*:}
      fi

    elif [[ "${method_type}" = "fm_stop" ]]; then
      dsrow=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} DS_FM_PATH \
		query ds_name ${dsname})

      method_path=${dsrow%:*:*:*}
      method_path=${method_path#*:*:*:*:*:*:}

      method_to=${dsrow%:*:*}
      method_to=${method_to#*:*:*:*:*:*:*:}
    fi

    if [[ -n "${method_path}" ]]; then
      call_method ${method_type} ${n} ${method_path} ${method_to}
    fi
    let n=n-1
  done
fi
}

###########################################################################
#
# function call_method method_type ds_index method_path method_to
#
# method_type - the type of the method to be called. Same as the first
#		argument to exec_ds_method
#
# ds_index - the index of the data service in the ordered_ds_array.
#
# method_path - the full path of the method to be called.
#
# method_to - the timeout registered with the method

function call_method
{
typeset method_type
typeset ds_index
typeset method_path
typeset method_to
typeset retval method_called
method_type=$1
ds_index=$2
method_path=$3
method_to=$4

retval=0
let method_called=0

if [[ "${ds_state_array[ds_index]}" = "1" && -f "${method_path}" ]]; then
  
  if [[ "${method_type}" = "stop" || "${method_type}" = "stop_net" || \
        "${method_type}" = "abort" || "${method_type}" = "abort_net" || \
        "${method_type}" = "fm_stop" ]]; then

	  # for stop/stopnet let us run as REAL time

 	  ${CLUSTERBIN}/timed_run ${method_to} ${method_path} \
		"${lm_loghosts_array[ds_index]}" "${nm_loghosts_array[ds_index]}" \
			${method_to} || retval=$?
  else
	 # run start/start_net as TIME SHARED.

	  /usr/bin/priocntl -c TS -e ${CLUSTERBIN}/timed_run \
		${method_to} ${method_path} "${lm_loghosts_array[ds_index]}" \
			 "${nm_loghosts_array[ds_index]}"  ${method_to} 
	  retval=$?
  fi
  let method_called=1

elif [[ "${ds_state_array[ds_index]}" = "0" && -f "${method_path}" ]]; then
  if [[ "${method_type}" = "stop" || "${method_type}" = "stop_net" || \
        "${method_type}" = "abort" || "${method_type}" = "abort_net" || \
        "${method_type}" = "fm_stop" ]]; then
    
    ${CLUSTERBIN}/timed_run ${method_to} ${method_path} "" \
		"${all_loghosts_array[ds_index]}" ${method_to} || retval=$?
    let method_called=1
  fi
fi

if (( method_called == 1 )); then
  if [[ "${retval}" -eq 0 ]]; then
    log_info "${pre}.1050" "${method_type} method of data service ${ordered_ds_array[ds_index]} completed successfully."
  else
    log_info "${pre}.3010" "${method_type} method of data service ${ordered_ds_array[ds_index]} failed"
  fi
fi

}

case ${keyword} in
  
  CCDSYNC_PRE_ADDU)
	init
	export CLUSTERNAME
	if [[ "${curr_master}" = "${localhostname}" ]]; then
	  log_info "${pre}.1030" "Taking over logical host ${lname}"
	  
	  if [[ "${vm}" = "cvm" && "${nd}" -eq 1 ]]; then
	    for dg in ${dglist}; do
	      unndshare_privdg_disks ${dg} ${localhostname}
	    done
	  fi

	  touch ${CLUSTERVAR}/loghost.${lname}
	  cvm_import ${dglist} 
	  RC=$?

	  if [[ "${RC}" -ne 0 ]]; then
		echo ${RC} > ${CLUSTERVAR}/loghost.${lname}
		exit 0
	  fi

	  curr_nodes=""

	  for n in ${curr_members}; do
	    nodename=$(${CLUSTERBIN}/cdbmatch cluster.node.${n}.hostname \
				${cdbfile})
	    curr_nodes="${curr_nodes} ${nodename}"
	  done

	  if anyone_else_present "${nodelist}" "${curr_nodes}" ${localhostname} ; then
	    failure_fencing release private ${dglist}
	  else
	    failure_fencing reserve private ${dglist}
	  fi

	  ${CLUSTERBIN}/scnfs ${CLUSTERNAME} mount ${lname}
	  RC=$?
	  if [[ "${RC}" -ne 0 ]]; then
		echo ${RC} > ${CLUSTERVAR}/loghost.${lname}
		exit 0
	  fi

	fi

	if (( in_nodelist == 1 )); then
	  exec_ds_method start
	fi

	if [[ "${curr_master}" = "${localhostname}" ]]; then

	  ifconfig_ip up ${iplist} 
	  RC=$?
	  if [ "${RC}" -ne 0 ]; then
	 	echo ${RC} > ${CLUSTERVAR}/loghost.${lname}
		exit 0
	  fi
	fi

	if (( in_nodelist == 1 )); then
	  exec_ds_method start_net
	  exec_ds_method fm_init
	fi

	rm -rf ${CLUSTERVAR}/loghost.${lname}

	;;

  CCDSYNC_POST_ADDU)
	CLUSTERBIN=/opt/SUNWcluster/bin
        CLUSTERVAR=/var/opt/SUNWcluster
        CLUSTERETC=/etc/opt/SUNWcluster/conf
        SSACLI=${CLUSTERBIN}/scssa

	lname=${row%:*}
	lname=${lname#*:}
	if [ -f ${CLUSTERVAR}/loghost.${lname} ]; then
		rm -rf ${CLUSTERVAR}/loghost.${lname}
		exit 1
	fi
	init
	export CLUSTERNAME
	if [[ "${curr_master}" = "${localhostname}" ]]; then
	  log_info "${pre}.1040" "Take over of logical host ${lname} succeeded"
	fi
	exec_ds_method fm_start
	;;


  CCDSYNC_PRE_REMOVE)
	init
	export CLUSTERNAME
	if [[ "${curr_master}" = "${localhostname}" ]]; then
	  log_info "${pre}.1010" "Giving up logical host ${lname}"
	fi

	#
	# since fm_stop is already called by loghostreconfig
	# for abort cases , there is no need to do it here.
	#
	if [[ "${clusterstate}" != "abort" ]]; then
		exec_ds_method fm_stop
	fi

	if (( in_nodelist == 1 )); then
	  if [[ "${clusterstate}" = "abort" ]]; then
	    exec_ds_method abort_net
	  else
	    exec_ds_method stop_net
	  fi
	fi

	touch ${CLUSTERVAR}/loghost.${lname}

	if [[ "${curr_master}" = "${localhostname}" ]]; then

	  ifconfig_ip down ${iplist} 
	  RC=$?
	  if [[ "${RC}" -ne 0 ]]; then
		echo $? > ${CLUSTERVAR}/loghost.${lname}
		#	
		# no exiting here because methods may
		# complain if i exit here.
		#
	  fi 

	fi 

	if (( in_nodelist == 1 )); then 
		if [[ "${clusterstate}" = "abort" ]]; then 
			exec_ds_method abort 
		else
	    		exec_ds_method stop
	  	fi
	fi

	if [[ "${curr_master}" = "${localhostname}" ]]; then

	  ${CLUSTERBIN}/scnfs ${CLUSTERNAME} unmount ${lname}
	  
	  if [[ "${vm}" = "cvm" && "${nd}" -eq 1 ]]; then
	    for dg in ${dglist}; do
	      unndshare_privdg_disks ${dg} ${localhostname}
	    done
	  fi
	  cvm_deport ${dglist}
	  #
	  # the deport operation has failed. We will make one last attempt
	  # by calling the abort methods to close the open volumes and
	  # retrying the deport. If this fails too, we return an error and
	  # let the higher layers of software handle the error. Failure
	  # fencing will be removed in the abort transition.
	  #
	  if [[ "$?" -ne 0 ]]; then
	    exec_ds_method abort
	    cvm_deport ${dglist} || echo $? > ${CLUSTERVAR}/loghost.${lname}
	  fi
	      
	  failure_fencing release private ${dglist}
	fi

	rm -rf ${CLUSTERVAR}/loghost.${lname}

	;;

  CCDSYNC_POST_REMOVE)

	CLUSTERBIN=/opt/SUNWcluster/bin
	CLUSTERVAR=/var/opt/SUNWcluster
	CLUSTERETC=/etc/opt/SUNWcluster/conf
	SSACLI=${CLUSTERBIN}/scssa
	CLUSTERNAME=$(/bin/cat ${CLUSTERETC}/default_clustername)
	cdbfile=${CLUSTERETC}/${CLUSTERNAME}.cdb
	localhostname=$(${CLUSTERBIN}/cdbmatch \
		cluster.node.${localnodeid}.hostname ${cdbfile})
	lname=${row%:*}
	curr_master=${row#*:*:}
	lname=${lname#*:}

	if [ -f ${CLUSTERVAR}/loghost.${lname} ]; then
		rm -rf ${CLUSTERVAR}/loghost.${lname}
		exit 1
	fi

	if [[ "${curr_master}" = "${localhostname}" ]]; then
	  log_info "${pre}.1020" "Give up of logical host ${lname} succeeded"
	fi
	;;

  CCDSYNC_PRE_RESTORE)
	#
	# first check if there are logical hosts mastered
	# if any exit with 1.
	#
	CLUSTERBIN=/opt/SUNWcluster/bin
	CLUSTERVAR=/var/opt/SUNWcluster
	CLUSTERETC=/etc/opt/SUNWcluster/conf

	CLUSTERNAME=$(/bin/cat ${CLUSTERETC}/default_clustername)
	cdbfile=${CLUSTERETC}/${CLUSTERNAME}.cdb
	localhostname=$(${CLUSTERBIN}/cdbmatch \
		cluster.node.${localnodeid}.hostname ${cdbfile})
  	cm_row=$(${CLUSTERBIN}/scccd -f ${CCDFILE} ${CLUSTERNAME} \
		LOGHOST_CM query curr_master ${localhostname})

	if [ "${cm_row}" !=  "" ]; then
	log_info "${pre}.4090" "Cannot Restore CCD database when Logical Hosts are mastered"
		exit 1;
	fi
	;;

  CCDSYNC_PRE_RESTOREKEY | CCDSYNC_POST_RESTORE | CCDSYNC_POST_RESTOREKEY)
		;;

esac

exit 0

