#!/bin/sh
set +x

G_MML_FILE_PATH="/opt/huawei/snas/script/inspect_mml/"
Remote_filename="MemPoolUsage$$"
ConfigFile="/opt/huawei/snas/etc/snas.ini"
#Ƶس
Surveillance="surveillance"
LOG_FILE="/var/log/inspect.log"
needtocheck=1
source $G_MML_FILE_PATH/CheckItems
source $G_MML_FILE_PATH/CommonFunc.sh
G_REMOTE_TMP_FILE="/tmp/tmpfile"
CurInspectNum="214"
CurInspectFun="$(GetInspectType $CurInspectNum)"
RESULTFILE="/tmp/tmpResult${CurInspectFun}"
>${RESULTFILE}

localip=`GetLocalIp`
SSDC_NEED_CHK=1

if [ -z "${localip}" ]; then
	LOG "[$LINENO]Error:Get localip failed."
	exit 0
fi

#жǷudsƷ
ProductType=$(cat "/opt/huawei/snas/etc/snas.ini"|grep productType|awk -F= '{print $2}')
#ȡڵ洢
node_service_type=$(egrep '[[]|^'node_service_type'=' /opt/huawei/snas/etc/snas.ini | tr -d '\n' | grep -Po '(?<=[[]'NODE'[]]'node_service_type'=)[0-9]+')
if [ "48" == "${ProductType}" ] || [ "91" == "${ProductType}" ];then
    needtocheck=0
fi

function LOG()
{
    time=$(date)
    echo [${time}][$$][$CurInspectFun]$@ >> $LOG_FILE
}

function getMemUsedHisRecFile()
{
    echo "/tmp/memUsedHisRecord"
}

function successiveMemInspect()
{
    local module=$1
    local hisInfo=""
    if [ -e $(getMemUsedHisRecFile) ]; then
        hisInfo=$(sed -n "/${module}\s/p" $(getMemUsedHisRecFile))
        echo "${hisInfo}" |
            awk 'BEGIN{ \
                isAlarm=0; \
                memInfo=""; \
            } \
            { \
                isAlarm=$2; \
                for (i=4; i<=NF; i++) { \
                    memInfo=sprintf("%s %-10s", memInfo, $i); \
                } \
            } \
            END{ \
                if(isAlarm == 1) { \
                    print "memSuccessiveArise=1,"memInfo; \
                } \
                else { \
                    print "memSuccessiveArise=0"; \
                } \
            }'
    else
        echo "memSuccessiveArise=0"
    fi
}

function CheckMemFragment()
{
    local modules=("MDS" "DS" "CA" "NOFS" "SSDC")
    local commands=("mem show stat 1" "mem show stat 2" "mem show stat 3" "mem show stat 15" "mem show stat 30")
    local ports=("988" "988" "988" "988" "988")
    for i in $(seq 0 `expr ${#modules[@]} - 1`);do
        module=${modules[$i]}
        command=${commands[$i]}
        port=${ports[$i]}
        if [ X"${module}" == X"SSDC" -a ${SSDC_NEED_CHK} -eq 0 ]; then
            continue
        fi
        
        $G_MML_FILE_PATH/ShowInspectMml $localip $port "${command}" >${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment 2>&1
        iRet=$?
        if [ 0 -eq  ${iRet} ];then
            dos2unix ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment >/dev/null 2>&1
            PooId="$(cat ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment | grep -Poa '(?<=PooId )[0-9]+' | tr -t '\n' ' ')"
            >${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment
            for pool in ${PooId};do
                sleep 1
                $G_MML_FILE_PATH/ShowInspectMml $localip $port "${command} ${pool}" >> ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment  2>&1
                iRet=$?
                if [ 0 -ne ${iRet} ];then
                    LOG "[$LINENO]${module}[ERR]${module} exec cmd failed, $port ${command} ${pool}"
                    continue
                fi
            done
            dos2unix ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment >/dev/null 2>&1
            cat ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment | grep Freearea | awk '{print $NF}' | grep -wvq 0 >/dev/null 2>&1
            if [ $? -ne 0 ];then
                echo "${module}[ERR]${module} all pools have no memory left!" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            fi
        else
            LOG "[$LINENO]${module}[ERR]${module} exec cmd failed,$port ${command}"
        fi
        sleep 1
    done
    local vmallocNum=$(cat /proc/vmallocinfo |wc -l)
    if [ ${vmallocNum:-0} -gt 100000 ];then
        echo "vmalloc[ERR]The number of vmallocinfo in the system exceeds 100000!" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    fi
    LOG "[$LINENO]vmalloc vmallocNum : ${vmallocNum:-0}"
    rm -f ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment
}


function CheckCompanionLogCount()
{
    local module="$1"
    local tmpinfo="$2"
    local result=$(echo "${tmpinfo}" |
        awk -F',' 'BEGIN{
                totalUsed=0;
            }
            {
                if ($4~/used: [ ]+[0-9]+/)
                {
                    #printf("%s\n", $4)
                    split($4, a, ":");
                    tmpused=a[2]
                    totalUsed=totalUsed + tmpused;
                }
            }
            END {
                printf("%d" ,totalUsed);
            }'
        )

    if [ $result -gt 80000 ];then
        echo "${module}[ERR]${module} associated log count($result) is greater than 80000!" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    fi
}
function GetModuleMem()
{
    local module=$1 
    local command=$2
    local port=$3   
    local succInspectResult=""
    echo "[${module}]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
    for i in $(seq 1 3);do
        iRet=""
        $G_MML_FILE_PATH/ShowInspectMml $localip $port "${command}" >${G_REMOTE_TMP_FILE}${Remote_filename}_${module} 2>&1 
        iRet=$?
        if [ 0 -eq  ${iRet} ];then
	    dos2unix ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} >/dev/null 2>&1

            #־
            if [ "${module}" == "TRNS_LOG" ];then
                tmpinfo=$(grep "Cache:Kmalloc, Size:      304" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module})
                CheckCompanionLogCount "${module}" "$tmpinfo"
            elif [ "${module}" == "DS" ] ;then
                tmpinfo=$(grep "Cache:S_Kmalloc2-4096" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module})
                CheckCompanionLogCount "${module}" "$tmpinfo"
            fi

            if [ "${module}" == "NOFS" ];then
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "nofs.6" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "${iRet}" -eq "3" ];then
                        echo "$module[ERR]${module} get date fialed,please inspect again!"
                        LOG "[$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            elif [ "${module}" == "DLM" ];then
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "base.0" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "${i}" -eq "3" ];then
                        echo "${module}[ERR]${module} get date fialed,please inspect again!"
                        LOG "[$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            elif [ "${module}" == "CA" ];then

                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "ca.0" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "${i}" -eq "3" ];then
                        echo "$module[ERR]$module get date fialed,please inspect again!"
                        LOG "[$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
		service_type=$( grep -w "^serviceType" "${ConfigFile}" |tr -d " "|awk -F"=" '{print $2}' )  
		#ֻƵسżPooId 3
		if [ "X${Surveillance}" == "X${service_type}" ];then
		    echo "[$module]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
		    grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "ca.3" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                    if [ $? -eq 0 ];then
                    	succInspectResult=$(successiveMemInspect "ca.3")
                    	echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                    else
                    	if [ "$i" -eq "3" ];then
                            echo "${module}[ERR]${module} get date fialed,please inspect again!"
                    	fi
                    	continue
                    fi
		fi
             elif [ "${module}" == "MDS" ];then
                MDS_LeftMemSize=$( grep "LeftMemSize" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} | awk -F"=" '{print $2}' |tr -d " " ) 
		MDS_ResvMemSize=$( grep "ResvMemSize" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} | awk -F"=" '{print $2}' |tr -d " " ) 
                echo "MDS_LeftMemSize=${MDS_LeftMemSize},MDS_ResvMemSize=${MDS_ResvMemSize}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
		break
            elif [ "${module}" == "BASE" ];then
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "base.1" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "${i}" -eq "3" ];then
                        echo "$module[ERR]$module get date fialed,please inspect again!"
                        LOG "[$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            else
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "${i}" -eq "3" ];then
                        echo "${module}[ERR]${module} get date fialed,please inspect again!"
                        LOG "[$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            fi
        fi
        sleep 1
    done
    if [ "3" == "${i}" -a 0 -ne $iRet ];then
        LOG "[$LINENO]Module ${module} get data failed"
        echo "$module[ERR]Module ${module} get data failed" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
    fi
}

function NASmemUsageSummary()
{
    local memSummaryInfo=$(nas_proto_pum -m nas_mem -c get_summary |grep "\["|grep "[0-9]")
    local module=$1
    echo "$memSummaryInfo"| while read line
    do
        echo "[$module]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
        poolid=$(echo ${line} | awk '{print $1}'|awk -F"[" '{print $1}')
        if [ "10" == "$poolid" ];then
            Pool=$(echo ${line}|awk -F"]" '{print $1}'|awk -F"[" '{print $2}')
            totalsize=$(echo ${line} | awk '{print $2}')
            freesize=$(echo ${line} |awk '{print $3}')
            usedsize=$(echo ${line} |awk '{print $4}')
        
        else
            Pool=$(echo ${line}|awk -F"]" '{print $1}'|awk -F"[" '{print $2}')
            totalsize=$(echo ${line} | awk '{print $3}')
            freesize=$(echo ${line} |awk '{print $4}')
            usedsize=$(echo ${line} |awk '{print $5}')
        fi
        echo "Pool[${Pool}]:total size:${totalsize},free size:${freesize},usedsize:${usedsize}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
        LOG "[$LINENO]Pool[${Pool}]:total size:${totalsize},free size:${freesize},usedsize:${usedsize}"
        
    done
}
function OSdlmUsageSummary()
{
    local memSummaryInfo=$(echo -n "showmp" >/proc/dlm_info;cat /proc/dlm_info)
    result=$(echo "${memSummaryInfo}" |
        awk 'BEGIN{ \
                result="OS_dlm_normal"; \
            } \
            { \
                if (($3~/^[0-9]+$/) && ($4~/^[0-9]+$/)) { \
                    if (($8~/plugin/) && ($9~/Pool/)) { \
                    } \
                    else { \
                        if (($4 + 0) > ($3 + 0)) { \
                            printf ("os_dlm[ERR]OS_dlm mem over id %s,num %s,used %s\n",$1,$3,$4); \
                            result="osdlm_alarm"; \
                        } \
                    } \
                } \
            } \
            END { \
                print result; \
            }' \
        )
    echo "${result}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    LOG "[$LINENO]result=${result}"
}

function main()
{
    local succInspectResult=""
    local Thrnasprocnum="10"
    local version=$(cat /opt/huawei/deploy/package/version |grep "ProductVersion"|awk -F= '{print $2}')
    #ȡҪִMMLĸģϢ
    MODULEMEM=("MON" "BJM_Server" "BJM_Worker" "CA" "MDS" "DAB" "DS" "OBS" "CM_Kernel" "CM_User" "NV" "TRNS" "TRNS_LOG" "MONC" "NOFS" "DLM" "BASE")
    modulecommand=("mem show stat 5" "mem show stat 21" "mem show stat 21" "mem show stat 3"  "mds show lru" "mem show stat 17" "mem show stat 2" "mem show stat 9" "mem show stat 8" "mem show stat 7" "nvmem show stat 5" "nvmem show stat 6" "nvmem show stat 7" "mem show stat 4" "mem show stat 15" "mem show stat 6" "mem show stat 6")
    moduleport=("4004" "4104" "4108" "988" "988" "988" "988" "988" "988" "4016" "988" "988" "988" "988" "988" "988" "988")
    for i in $(seq 0 `expr ${#MODULEMEM[@]} - 1`);do
        module=${MODULEMEM[$i]}
        command=${modulecommand[$i]}
        port=${moduleport[$i]}
        if [ "MON" == "$module" ];then
            GetModuleMem "$module" "$command" "$port"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "MON_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            MON_Used=$(ps -C snas_mon v | grep -v "COMMAND" | awk '{print $8}')
            echo "MON_Used=${MON_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$LINENO]succInspectResult=[${succInspectResult}] MON_Used=${MON_Used}"
            continue
        fi
        if [ "BJM_Server" == "${module}" ];then
            GetModuleMem "${module}" "${command}" "${port}"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "BJM_Server_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            BJM_Server_Used=$(ps -C snas_bjm_server v | grep -v "COMMAND" | awk '{print $8}')
            echo "BJM_Server_Used=${BJM_Server_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$LINENO]succInspectResult=[${succInspectResult}]  BJM_Server_Used=${BJM_Server_Used}" 
            continue
        fi
        if [ "BJM_Worker" == "${module}" ];then
            GetModuleMem "${module}" "${command}" "${port}"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "BJM_Worker_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            BJM_Worker_Used=$(ps -C snas_bjm_worker v | grep -v "COMMAND" | awk '{print $8}')
            echo "BJM_Worker_Used=${BJM_Worker_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$LINENO]succInspectResult=[${succInspectResult}]  BJM_Worker_Used=${BJM_Worker_Used}"
            continue
        fi
        if [ "CA" == "$module" ];then
            if [ 1 -eq $node_service_type -a 0 -eq $needtocheck ];then
                GetModuleMem "$module" "$command" "$port"
                succInspectResult=$(successiveMemInspect "ca.0")
                
                echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                LOG "[$LINENO]succInspectResult=[${succInspectResult}]"
                continue
            fi
            continue
        fi
        GetModuleMem "${module}" "${command}" "${port}"
        succInspectResult=$(successiveMemInspect "${module}")
        echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        LOG "[$LINENO]module=[${module}] port=[${port}],succInspectResult=[${succInspectResult}]"
    done
    #ȡNASģڴϢ
    if [ 1 -eq ${node_service_type} -a 0 -eq ${needtocheck} ];then
        NASmemUsageSummary "NAS"
        nasprocnum=$(ps -efww |grep watchdog_nas | grep -v grep |wc -l)
        if [ ${nasprocnum} -gt ${Thrnasprocnum} ];then
            echo "nas[ERR]module NAS process watchdog_nas over 10" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$LINENO]nas[ERR]module NAS process watchdog_nas over 10"
        fi
    fi
    #ȡBJM ManagerģڴϢ
    echo "[BJM_Manager]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    succInspectResult=$(successiveMemInspect "BJM_Manager")
    echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    local BJM_Manager_Used=$(ps -C snas_bjm_manager v | grep -v "COMMAND" | awk '{print $8}')
    echo "BJM_Manager_Used=${BJM_Manager_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    LOG "[$LINENO]succInspectResult=[${succInspectResult}] BJM_Manager_Used=${BJM_Manager_Used}"
    #ȡReplicationģڴϢֻ9000ҪԶ̸ģ
    if [ 1 -eq $node_service_type -a "48" -eq "${ProductType}" ];then
        echo "[Replication]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        local Replication_Used=$(ps -C snas_rep v | grep -v "COMMAND" | awk '{print $8}')
        echo "Replication_Used=${Replication_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        LOG "[$LINENO]Replication_Used=${Replication_Used}"
    fi
    #ȡRPCģڴϢ
    echo "[RPC]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    succInspectResult=$(successiveMemInspect "RPC")
    if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
        succInspectResult=$(successiveMemInspect "RPC_Page")
    fi
    echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    local RPC_memused=$(cat /proc/sys/lnet/memused | awk '{print $1}')
    local RPC_pageused=$(cat /proc/sys/lnet/pageused | awk '{print $1}')
    echo "RPC_memused=${RPC_memused},RPC_pageused=${RPC_pageused}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    LOG "[$LINENO]succInspectResult=[${succInspectResult}] RPC_memused=${RPC_memused},RPC_pageused=${RPC_pageused}" 

    #ȡSSDCڴϢ MmlBatch 988 "mem show stat 30" V5(V3R6֧)  ͳʣڴ棬50M
    local ssdc_memfree="--"
    local mempoolcnt=0
    $G_MML_FILE_PATH/ShowInspectMml $localip 988 "mem show stat 30" | grep -A11 -E "Pool\[ssdc.1\]:total size"|grep "^Freearea\[" >${G_REMOTE_TMP_FILE}${Remote_filename}_SSDC 2>&1
    if [ $? -eq 0 ];then
        ssdc_memfree=0
        # λK4k 8k---4M,ʾתM
        # ڵڴ汾ͲǲSSDCģMMLִвܶԴ0ԣͨMMLжSSDCǷ
        size=("4" "8" "16" "32" "64" "128" "256" "512" "1024" "2048" "4096")
        while read line; do
            mempoolcnt=$((mempoolcnt+1))
            modle=$(echo $line | grep -Po '(?<=Freearea\[)[0-9]+')
            freeNum=$(echo $line | grep -Po '(?<=:Free )[0-9]+')
            ssdc_memfree=$[ $ssdc_memfree + $freeNum * ${size[$modle]} ]
        done < ${G_REMOTE_TMP_FILE}${Remote_filename}_SSDC
    fi
    if [ ${mempoolcnt} -ne 0 ]; then
        echo "[SSDC]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        echo "SSDC_memfree=${ssdc_memfree}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        LOG "[$LINENO] SSDC_memfree=${ssdc_memfree} "
    else
        SSDC_NEED_CHK=0
        LOG "[$LINENO]node have no ssdc module,memory size is:$(free -m | grep Mem | awk '{print $2}')M"
    fi

    #OS_dlmģڴϢֻV3R6C00SPC200汾Ŵ,UDS֧OS_dlmѯ
    if [ "${ProductType}" != "80" ];then
        LOG "[$LINENO] check OS_dlm module"
        OSdlmUsageSummary
    fi
    CheckMemFragment
    echo "[END]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    cat ${G_REMOTE_TMP_FILE}${Remote_filename} >>${RESULTFILE} 2>&1
}

>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
main
sh ${G_MML_FILE_PATH}/dynamic_mem_inspect.sh "init" >/dev/null 2>&1
 if [ ! -d /var/log/mds_state_check/tmpdata ];then
    mkdir -p /var/log/mds_state_check/tmpdata >/dev/null 2>&1 
fi

rm -f ${G_REMOTE_TMP_FILE}${Remote_filename}*
