##############################################################################
# FileName  : GetPoolNameAndPhysicalMem
# Date      : 2014-08-07
# Description   : Check wether the physic memory in one pool as the same
#
##############################################################################

#!/bin/sh
set +x

source /opt/huawei/snas/script/inspect_mml/CommonFunc.sh
G_REMOTE_TMP_FILE="/tmp/tmpfile$$"
LOG_FILE="/var/log/inspect.log"
snasIniFile="/opt/huawei/snas/etc/snas.ini"
G_EXEC_DIAGNOSE_CMD="/opt/huawei/snas/script/inspect_mml/exec_diagnose_cmd"
G_DBG_CLIENT="/opt/huawei/snas_rep/bin/dbg_client"
G_SNAS_REP="snas_rep"

function LOG
{
    time=$(date)
    echo [${time}][$$]$@ >> $LOG_FILE
}

DiagnoseOutPut()
{
    local diagCmd="$1"
    local tmpFile="$2"
    local firstLine=0
    local endLine=0

    firstLine=`sed -n "/Diagnose>>${diagCmd}/=" $tmpFile 2>>${LOG_FILE}`
    if [ -z "$firstLine" ]; then
        LOG "[$FUNCNAME][$LINENO] Failed, tmpFile($tmpFile) no find line Diagnose>>${diagCmd}"
        cat $tmpFile >> ${LOG_FILE} 2>&1
        return 1
    fi
    firstLine=`expr $firstLine + 1 2>>${LOG_FILE}`
    endLine=`sed -n '/Diagnose>>exit/=' $tmpFile 2>>${LOG_FILE}`

    if [ -z "$endLine" ]; then
        LOG "[$FUNCNAME][$LINENO] Failed, tmpFile($tmpFile) no find line Diagnose>>exit"
        cat $tmpFile >> ${LOG_FILE} 2>&1
        return 1
    fi
    endLine=`expr $endLine - 2 2>>${LOG_FILE}`
    if [ $firstLine -gt $endLine ]; then
        LOG "[$FUNCNAME][$LINENO] Failed, firstLine($firstLine) is bigger and equal than endLine($endLine)"
        return 1
    fi

    sed -n "${firstLine},${endLine}p" $tmpFile 2>>${LOG_FILE}
    return 0
}


function IsReplicationMasterNode
{
    local iRet=""
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"

    diagCmd="rnm ismasternode"
    ${G_EXEC_DIAGNOSE_CMD}  "${diagCmd}" > $tmpFile 2>>${LOG_FILE}
    ret=$?
    if [ $ret -ne 0 ]; then
        LOG "[$FUNCNAME][$LINENO] Failed to exec ${G_EXEC_DIAGNOSE_CMD} $diagCmd."
        return 2
    fi

    if [ `grep "Local node is master node" $tmpFile | wc -l` -ge 1 ] ; then
        return 1
    else
        return 0
    fi
}

function IsReplicationCtrlNode
{
    local iRet=""
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"

    diagCmd="rnm isctrlnode"
    ${G_EXEC_DIAGNOSE_CMD}  "${diagCmd}" > $tmpFile 2>>${LOG_FILE}
    ret=$?
    if [ $ret -ne 0 ]; then
        LOG "[$FUNCNAME][$LINENO] Failed to exec ${G_EXEC_DIAGNOSE_CMD} $diagCmd."
        return 2
    fi

    if [ `grep "Command exec success" $tmpFile | wc -l` -ge 1 ] && [ `grep "Local node is ctrl node" $tmpFile | wc -l` -ge 1 ] ; then
        return 1
    elif [ `grep "Command exec success" $tmpFile | wc -l` -eq 1 ]; then
        return 0
    else
        LOG "[$FUNCNAME][$LINENO] Failed to exec ${G_EXEC_DIAGNOSE_CMD} $diagCmd."
        cat $tmpFile >> LOG_FILE 2>&1
        return 2
    fi
}

function GetRepTargetNum
{
    local target="$1"
    local keyWord="$2"
    local targetRow="$3"
    local iRet=""
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"

    if [ "$target" == "pair" ]; then
        diagCmd="rcl showpairnum"
    elif [ "$target" == "node" ]; then
        diagCmd="rnm showrepzonenum"
    elif [ "$target" == "channel" ]; then
        diagCmd="rnm showchannelnum"
    else
        LOG "[$FUNCNAME][$LINENO] $target unexpect."
        return 1
    fi
    ${G_EXEC_DIAGNOSE_CMD}  "${diagCmd}" > $tmpFile 2>>${LOG_FILE}
    ret=$?
    if [ $ret -ne 0 ]; then
        LOG "[$FUNCNAME][$LINENO] Failed to exec ${G_EXEC_DIAGNOSE_CMD} $diagCmd."
        return 1
    fi

    cmdOutPut=`DiagnoseOutPut "${diagCmd}" "${tmpFile}"`
    ret=$?
    if [ $ret -ne 0 ]; then
        LOG "[$FUNCNAME][$LINENO] Failed to DiagnoseOutPut diagCmd($diagCmd) tmpFile($tmpFile), ret($ret)."
        cat $tmpFile >> LOG_FILE 2>&1
        return 1
    fi

    if [ `echo "$cmdOutPut" | grep -w "$keyWord" | wc -l` -ne 1 ]; then
        LOG "[$FUNCNAME][$LINENO] parse ${G_EXEC_DIAGNOSE_CMD} $diagCmd output unexpect."
        cat $tmpFile >> LOG_FILE 2>&1
        return 1
    fi
    info=`echo "$cmdOutPut" | grep -v "$keyWord"  | grep -v "^$"`
    if [ `echo "$info" | wc -l` -ne 1 ]; then
        LOG "[$FUNCNAME][$LINENO] parse ${G_EXEC_DIAGNOSE_CMD} $diagCmd output unexpect."
        cat $tmpFile >> LOG_FILE 2>&1
        return 1
    fi

    echo "$info" | awk '{print $'$targetRow'}' | sed 's/\r/\ /g' | awk '{print $1}'
    return 0
}


#add filed:NodeStatus isGetSuccess isMasterNode isCtrlNode
#add filed:  0    Fail/InspectPass/InspectFail  MasterNode  -
#original filed:$NodeNum $ChannelNum
function GetMasterNodeChannelInfo
{
    NodeNum=`GetRepTargetNum "node" "NodeNum" 2`
    ret=$?
    if [ $ret -ne 0 ]; then
        echo "0 Fail MasterNode -"
        return 1
    fi

    if [ "X$NodeNum" != "X0" ]; then
        return 0
    fi

    ChannelNum=`GetRepTargetNum "channel" "ChannelNum" 2 `
    ret=$?
    if [ $ret -ne 0 ]; then
        echo "0 Fail MasterNode -"
        return 1
    fi

    echo "0 Succ MasterNode - $NodeNum $ChannelNum"

    return 1
}

#add filed:NodeStatus isGetSuccess isMasterNode isCtrlNode
#add filed:  0    Fail/Empty/Succ    MasterNode/NotMasterNode    CtrlNode/NotCtrlNode/-
#original filed:LogicId     ArrayID     LinkCnt     LiveLink     Status    Ref    Compatible
function  GetReplicationLinkInfo
{
    local isMasterNode="$1"
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"
    IsReplicationCtrlNode
    isCtrlNode=$?

    if [ $isCtrlNode -eq 1 ]; then
        diagCmd="epl showldev"
        ${G_EXEC_DIAGNOSE_CMD}  "${diagCmd}" > ${tmpFile} 2>>${LOG_FILE}
        ret=$?
        if [ $ret -ne 0 ]; then
            LOG "[$FUNCNAME][$LINENO] Failed to exec ${G_EXEC_DIAGNOSE_CMD} $diagCmd."
            echo "0 Fail $isMasterNode CtrlNode"
            return 1
        fi

        cmdOutPut=`DiagnoseOutPut "${diagCmd}" "${tmpFile}"`
        ret=$?
        if [ $ret -ne 0 ]; then
            LOG "[$FUNCNAME][$LINENO] Failed to DiagnoseOutPut diagCmd($diagCmd) tmpFile($tmpFile), ret($ret)."
            cat $tmpFile >> LOG_FILE 2>&1
            echo "0 Fail $isMasterNode CtrlNode"
            return 1
        fi

        if [ `echo "$cmdOutPut"  | grep -w "^LogicId" |wc -l` -eq 0 ]; then
            if [ `echo "$cmdOutPut"  | grep "Command exec success" |wc -l` -ge 1 ]; then
                echo "0 Empty $isMasterNode CtrlNode"
                return 0
            else
                echo "0 Fail $isMasterNode CtrlNode"
                return 1
            fi
        fi

        if [ `echo "$cmdOutPut"  | grep -v LiveLink | grep -v "^$" |wc -l` -eq 0 ]; then
            echo "0 Empty $isMasterNode CtrlNode"
        else
            echo "$cmdOutPut"  | grep -v LiveLink | grep -v "^$" | while read item
            do
                if [ -z "$item" ]; then
                    continue
                fi
                echo "0 Succ $isMasterNode CtrlNode ${item}" | sed 's/\r/\ /g'
            done
        fi
    elif [ $isCtrlNode -eq 0 ]; then
        echo "0 Succ $isMasterNode NotCtrlNode"
    else
        LOG "[$FUNCNAME][$LINENO] Failed to exec ${G_EXEC_DIAGNOSE_CMD} $diagCmd."
        cat $tmpFile >> LOG_FILE 2>&1
        echo "0 Fail $isMasterNode -"
    fi
}

#add filed:NodeStatus isGetSuccess isMasterNode isCtrlNode
#original filed:LogicId     ArrayID     LinkCnt     LiveLink     Status    Ref    Compatible
#original filed:$NodeNum $ChannelNum
function GetReplicationLink
{
    local iRet=""
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"

    nodeStatus=$(egrep '[[]|^status=' $snasIniFile|tr -d '\n'|grep -Po '(?<=[[]NODE[]]status=)[0-9]+')
    if [ $nodeStatus -ne 0 ]; then
        echo "$nodeStatus"
        return 0
    fi

    IsReplicationMasterNode
    isMasterNode=$?
    if [ $isMasterNode -eq 1 ]; then
        GetMasterNodeChannelInfo
        isContinue=$?
        if [ $isContinue -eq 0 ]; then
            GetReplicationLinkInfo "MasterNode"
        fi
    elif [ $isMasterNode -eq 0 ]; then
        GetReplicationLinkInfo "NotMasterNode"
    else
        echo "0 Fail - -"
    fi
}

#nodeStatus isGetSuccess status repPid
function GetReplicationProcess
{
    local iRet=""
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"

    # 用于日志记录复制进程具体信息，方便巡检失败时定位问题
    repProcess=$(ps -ef | grep "/opt/huawei/snas_rep/bin/snas_rep")
    # 获取pid时，pgrep更安全
    repPid=$(pgrep -fl "/opt/huawei/snas_rep/bin/snas_rep" | grep -v "grep" | grep -v "gdb" | grep "snas_rep" | awk '{print $1}')
    if [ "$repPid" = "" ]; then
        repPid="-"
    fi
    status=$(egrep '[[]|^status=' $snasIniFile|tr -d '\n'|grep -Po '(?<=[[]NODE[]]status=)[0-9]+')
    echo "$status ${repPid}" | sed 's/\r/\ /g'
    LOG "[$FUNCNAME]status=$status, repPid=${repPid}, repProcess=${repProcess}"
}

#add filed:NodeStatus isGetSuccess isMasterNode isCtrlNode
#add filed:  0    Fail/Empty/Succ    MasterNode/NotMasterNode    CtrlNode/NotCtrlNode/-
#original filed:
function  GetReplicationPairStatusInfo
{
    local isMasterNode="$1"
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"
    IsReplicationCtrlNode
    isCtrlNode=$?

    if [ $isCtrlNode -eq 1 ]; then
        CheckPairStatus  #控制节点新增检查 反复断建链、执行时间过长--超过60天未完成

        diagCmd="rcl showpairinfo"
        ${G_EXEC_DIAGNOSE_CMD}  "${diagCmd}" > ${tmpFile} 2>>${LOG_FILE}
        ret=$?
        if [ $ret -ne 0 ]; then
            LOG "[$FUNCNAME][$LINENO] Failed to exec ${G_EXEC_DIAGNOSE_CMD} $diagCmd."
            echo "0 Fail $isMasterNode CtrlNode"
            return 1
        fi

        cmdOutPut=`DiagnoseOutPut "${diagCmd}" "${tmpFile}"`
        ret=$?
        if [ $ret -ne 0 ]; then
            LOG "[$FUNCNAME][$LINENO] Failed to DiagnoseOutPut diagCmd($diagCmd) tmpFile($tmpFile), ret($ret)."
            cat $tmpFile >> LOG_FILE 2>&1
            echo "0 Fail $isMasterNode CtrlNode"
            return 1
        fi

        if [ `echo "$cmdOutPut"  | grep -w "^PairId" |wc -l` -eq 0 ]; then
            if [ `echo "$cmdOutPut"  | grep "Command exec success" |wc -l` -ge 1 ]; then
                echo "0 Empty $isMasterNode CtrlNode"
                return 0
            else
                echo "0 Fail $isMasterNode CtrlNode"
                return 0
            fi
        fi

        if [ `echo "$cmdOutPut"  | grep -v "Command exec success" | grep -v RunningStatus | grep -v "^$" |wc -l` -eq 0 ]; then
            echo "0 Empty $isMasterNode CtrlNode"
        else
            echo "$cmdOutPut"  | grep -v RunningStatus  | grep -v "^$" | while read item
            do
                if [ -z "$item" ]; then
                    continue
                fi
                echo "0 Succ $isMasterNode CtrlNode ${item}" | sed 's/\r/\ /g'
            done
        fi
    elif [ $isCtrlNode -eq 0 ]; then
        echo "0 Succ $isMasterNode NotCtrlNode"
    else
        LOG "[$FUNCNAME][$LINENO] Failed to exec ${G_EXEC_DIAGNOSE_CMD} $diagCmd."
        cat $tmpFile >> LOG_FILE 2>&1
        echo "0 Fail $isMasterNode -"
    fi
}

#add filed:NodeStatus isGetSuccess isMasterNode isCtrlNode
#add filed:  0    Fail/InspectPass/InspectFail  MasterNode  -
#original filed:$NodeNum $PairCount
function GetMasterNodePairInfo
{
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"

    NodeNum=`GetRepTargetNum "node" "NodeNum" 2`
    ret=$?
    if [ $ret -ne 0 ]; then
        echo "0 Fail MasterNode -"
        return 1
    fi

    if [ "X$NodeNum" != "X0" ]; then
        return 0
    fi

    PairCount=`GetRepTargetNum "pair" "PairCount" 1`
    ret=$?
    if [ $ret -ne 0 ]; then
        echo "0 Fail MasterNode -"
        return 1
    fi

    echo "0 Succ MasterNode - $NodeNum $PairCount"

    return 1
}

#新增内容 复制任务运行状态（反复断建链，执行时间过长）建议优化
function CheckPairStatus
{
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"
    local tmpFile_date="${G_REMOTE_TMP_FILE}${FUNCNAME}_date"
    #新增内容
    local dbFile="/opt/huawei/snas/etc/cm_alarm_record.db"
    local cmd="select ID,ALMOCCURTIME from CM_ALARM_RECORD_T where ALMID=0x400E0015 or ALMID=0x400E0013;"
    sqlite3 ${dbFile} "${cmd}" | sort -rV | head -40 >${tmpFile}
    local nowTime=$(date +%s)
    local dayTime=$((60*60*24))
    >${tmpFile_date}
    while read line; do
        if [ "$line" = "" ]; then
            continue
        fi
        task_id=$(echo "$line"|awk -F "|" '{print $1}' )
        alarmOccurTime=$(echo "$line"|awk -F "|" '{print $2}')
        tmpOccurTime=$(date -d @${alarmOccurTime} "+%Y-%m-%d %H:%M:%S")
        echo "${task_id}| ${tmpOccurTime}" >>${tmpFile_date}
    done < ${tmpFile}
    for ((i=0;i<8;++i)); do
        tmpTime=$((nowTime - i*dayTime))
        tmpDay=$(date -d @${tmpTime} "+%Y-%m-%d %H:%M:%S" | awk '{print $1}')
        tmpCnt=$(cat ${tmpFile_date} | grep -w "${tmpDay}" | wc -l)
        if [ $tmpCnt -ge 5 ];then
            echo "0 Succ - CtrlNode Repeated link disconnection. ${tmpDay} Num $tmpCnt"
        fi
    done
    local rcl_db="/opt/huawei/etc/ccdb/rep_9000_rcl_db_0/rep_9000_rcl_db"
    local pair_cmd="select pair_id,sync_start_time,sync_stop_time from rep_9000_pair_table"
    sqlite3  ${rcl_db} "${pair_cmd}" | grep -w "\-1$" >${tmpFile}
    while read line; do
        if [ "$line" = "" ]; then
            continue
        fi
        pair_id=$(echo "$line"|awk -F "|" '{print $1}' )
        start_time=$(echo "$line"|awk -F "|" '{print $2}')
        tmpStartTime=$(date -d @${start_time} "+%Y-%m-%d %H:%M:%S" )
        days=$(( (nowTime-start_time) / dayTime))
        if [ $days -ge 60 ];then
            echo "0 Succ - CtrlNode The task execution time exceeds 60 days.PairId ${pair_id} StartTime $tmpStartTime"
        fi
    done < ${tmpFile}
    rm ${tmpFile}
    rm ${tmpFile_date}
    return 1
}

#nodeStatus isGetSuccess isCtrlNode PairId RunningStatus HealthStatus
function GetReplicationPairStatus
{
    local iRet=""
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"
    local version=0

    nodeStatus=$(egrep '[[]|^status=' $snasIniFile|tr -d '\n'|grep -Po '(?<=[[]NODE[]]status=)[0-9]+')
    if [ $nodeStatus -ne 0 ]; then
        echo "$nodeStatus"
        return 0
    fi
    IsReplicationMasterNode
    isMasterNode=$?
    if [ $isMasterNode -eq 1 ]; then
        GetMasterNodePairInfo
        isContinue=$?
        if [ $isContinue -eq 0 ]; then
            GetReplicationPairStatusInfo "MasterNode"
        fi
    elif [ $isMasterNode -eq 0 ]; then
        GetReplicationPairStatusInfo "NotMasterNode"
    else
        echo "0 Fail - -"
    fi

}

#add filed:NodeStatus isGetSuccess isMasterNode isCtrlNode
#add filed:  0    Fail/Empty/Succ    MasterNode/NotMasterNode    CtrlNode/NotCtrlNode/-
#original filed:NodeNum ChannelNum
function  GetReplicationZoneStatusInfo
{
    local isMasterNode="$1"
    IsReplicationCtrlNode
    isCtrlNode=$?

    if [ $isCtrlNode -eq 1 ]; then
        NodeNum=`GetRepTargetNum "node" "NodeNum" 2`
        ret=$?
        if [ $ret -ne 0 ]; then
            echo "0 Fail $isMasterNode CtrlNode"
            return 1
        fi

        ChannelNum=`GetRepTargetNum "channel" "ChannelNum" 2`
        ret=$?
        if [ $ret -ne 0 ]; then
            echo "0 Fail $isMasterNode CtrlNode"
            return 1
        fi
        echo "0 Succ $isMasterNode CtrlNode ${NodeNum} ${ChannelNum}" | sed 's/\r/\ /g'
    elif [ $isCtrlNode -eq 0 ]; then
        echo "0 Succ $isMasterNode NotCtrlNode"
    else
        LOG "[$FUNCNAME][$LINENO] Failed to exec ${G_EXEC_DIAGNOSE_CMD} $diagCmd."
        cat $tmpFile >> LOG_FILE 2>&1
        echo "0 Fail $isMasterNode -"
    fi
}

#add filed:NodeStatus isGetSuccess isMasterNode isCtrlNode
#original filed: NodeNum ChannelNum
function GetReplicationZoneStatus
{
    local iRet=""
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"

    nodeStatus=$(egrep '[[]|^status=' $snasIniFile|tr -d '\n'|grep -Po '(?<=[[]NODE[]]status=)[0-9]+')
    if [ $nodeStatus -ne 0 ]; then
        echo "$nodeStatus"
        return 0
    fi

    IsReplicationMasterNode
    isMasterNode=$?
    if [ $isMasterNode -eq 1 ]; then
        GetMasterNodeChannelInfo
        isContinue=$?
        if [ $isContinue -eq 0 ]; then
            GetReplicationZoneStatusInfo "MasterNode"
        fi
    elif [ $isMasterNode -eq 0 ]; then
        GetReplicationZoneStatusInfo "NotMasterNode"
    else
        echo "0 Fail - -"
    fi
}

#nodeStatus isGetSuccess LocalNodeId VersionInfo
function GetReplicationVersion
{
    local tmpFile="${G_REMOTE_TMP_FILE}$FUNCNAME"

    nodeStatus=`egrep '[[]|^'status'=' /opt/huawei/snas/etc/snas.ini | tr -d '\n' | grep -Po '(?<=[[]'NODE'[]]'status'=)[0-9]+'`
    if [ $nodeStatus -ne 0 ]; then
        echo "$nodeStatus - - -"
        return 0
    fi

    diagCmd="rnm showrepnodeversion"
    ${G_EXEC_DIAGNOSE_CMD}  "${diagCmd}" > ${tmpFile} 2>>${LOG_FILE}
    ret=$?
    if [ $ret -ne 0 ]; then
        LOG "[$FUNCNAME][$LINENO] Failed to exec ${G_EXEC_DIAGNOSE_CMD} $diagCmd."
        echo "0 Fail - -"
        return 1
    fi

    cmdOutPut=`DiagnoseOutPut "${diagCmd}" "${tmpFile}"`
    ret=$?
    if [ $ret -ne 0 ]; then
        LOG "[$FUNCNAME][$LINENO] Failed to DiagnoseOutPut diagCmd($diagCmd) tmpFile($tmpFile), ret($ret)."
        cat $tmpFile >> LOG_FILE 2>&1
        echo "0 Fail - -"
        return 1
    fi

    if [ `echo "$cmdOutPut" | grep -w VersionInfo | wc -l` -ne 1 ]; then
        LOG "[$FUNCNAME][$LINENO] parse ${G_EXEC_DIAGNOSE_CMD} $diagCmd output unexpect."
        cat $tmpFile >> LOG_FILE 2>&1
        echo "0 Fail - -"
        return 1
    fi

    info=`echo "$cmdOutPut" | grep -v VersionInfo  | grep -v "^$"`
    if [ `echo "$info" | wc -l` -ne 1 ]; then
        LOG "[$FUNCNAME][$LINENO] parse ${G_EXEC_DIAGNOSE_CMD} $diagCmd output unexpect."
        cat $tmpFile >> LOG_FILE 2>&1
        echo "0 Fail - -"
        return 1
    fi

    echo "0 Succ $info" | sed 's/\r/\ /g'

    return 0
}


case $1 in
    "Link")
        GetReplicationLink
        ;;
    "Process")
        GetReplicationProcess
        ;;
    "ZoneStatus")
        GetReplicationZoneStatus
        ;;
    "PairStatus")
        GetReplicationPairStatus
        ;;
    "Version")
        GetReplicationVersion
        ;;
    *)
        echo "No this type"
        ;;

esac

[ -f "${G_REMOTE_TMP_FILE}GetReplicationVersion" ] && rm -f "${G_REMOTE_TMP_FILE}GetReplicationVersion"
[ -f "${G_REMOTE_TMP_FILE}IsReplicationCtrlNode" ] && rm -f "${G_REMOTE_TMP_FILE}IsReplicationCtrlNode"
[ -f "${G_REMOTE_TMP_FILE}IsReplicationMasterNode" ] && rm -f "${G_REMOTE_TMP_FILE}IsReplicationMasterNode"

exit 0
