#!/bin/sh
set +x

G_MML_FILE_PATH="/opt/huawei/snas/script/inspect_mml/"
Remote_filename="MemPoolUsage$$"
ConfigFile="/opt/huawei/snas/etc/snas.ini"
#视频监控场景服务类型
Surveillance="surveillance"
LOG_FILE="/var/log/inspect.log"
needtocheck=1
source $G_MML_FILE_PATH/CheckItems
source $G_MML_FILE_PATH/CommonFunc.sh
G_REMOTE_TMP_FILE="${G_TMP_INSPECT_PATH}tmpfile"
CurInspectNum="214"
CurInspectFun="$(GetInspectType $CurInspectNum)"
RESULTFILE="${G_TMP_INSPECT_PATH}tmpResult${CurInspectFun}"
>${RESULTFILE}

localip=`GetLocalIp`
SSDC_NEED_CHK=1

if [ -z "${localip}" ]; then
	LOG "[$LINENO]Error:Get localip failed."
	exit 0
fi

#判断是否是uds产品
ProductType=$(cat "/opt/huawei/snas/etc/snas.ini"|grep productType|awk -F= '{print $2}')
#获取节点存储类型
node_service_type=$(egrep '[[]|^'node_service_type'=' /opt/huawei/snas/etc/snas.ini | tr -d '\n' | grep -Po '(?<=[[]'NODE'[]]'node_service_type'=)[0-9]+')
if [ "48" == "${ProductType}" ] || [ "91" == "${ProductType}" ];then
    needtocheck=0
fi

function LOG()
{
    time=$(date)
    echo [${time}][$$][$CurInspectFun]$@ >> $LOG_FILE
}

function getMemUsedHisRecFile()
{
    echo "/tmp/memUsedHisRecord"
}

function successiveMemInspect()
{
    local module=$1
    local hisInfo=""
    if [ -e $(getMemUsedHisRecFile) ]; then
        hisInfo=$(sed -n "/${module}\s/p" $(getMemUsedHisRecFile))
        echo "${hisInfo}" |
            awk 'BEGIN{ \
                isAlarm=0; \
                memInfo=""; \
            } \
            { \
                isAlarm=$2; \
                for (i=4; i<=NF; i++) { \
                    memInfo=sprintf("%s %-10s", memInfo, $i); \
                } \
            } \
            END{ \
                if(isAlarm == 1) { \
                    print "memSuccessiveArise=1,"memInfo; \
                } \
                else { \
                    print "memSuccessiveArise=0"; \
                } \
            }'
    else
        echo "memSuccessiveArise=0"
    fi
}

function GetModuleMemFormatLoaded()
{
    #Pool[nvcache.0]:total size : 6582 MB (6901964800 B), free size : 6071 MB (6366544768 B), SglNum : 5233, SglPageNum : 102445.
    #Pool[nvcache.0]:total size : 6804 MB (7135232000 B), free size : 6157 MB (6456391776 B), SglNum : 6297, SglPageNum : 132391.
    #NV TRNS TRNS_LOG 的数据按 ModuleId 4, PoolNum 1, PooId 0, Total 651165696, Free 635237296, Percent 2, AllocFailTimes 0
    local tmpinfo="$1"
    local result=$(echo "${tmpinfo}" |
        awk -F',' 'BEGIN{
                totalSize=0;
                totalFree=0;
            }
            {
                if (($1~/total size : [0-9]+/) && ($2~/free size : [0-9]+/))
                {
                    split($1, a, " ");
                    tmptotal=a[4]
                    totalSize=totalSize + tmptotal;

                    split($2, b, " ");
                    tmpfree=b[4]
                    totalFree=totalFree + tmpfree;
                }
            }

            END {
                if (totalSize != 0 )
                {
                    Percent=(totalSize-totalFree)*100/totalSize
                    totalSize=totalSize*1048576
                    totalFree=totalFree*1048576
                }
                else
                {
                    Percent=0;
                }

                printf("ModuleId 0, PoolNum 0, PooId 0, Total %d, Free %d, Percent %d, AllocFailTimes 0" ,totalSize, totalFree, Percent);
            }'
        )
    echo "$result"
}

function CheckMemFragment()
{
    local modules=("MDS" "DS" "CA" "NOFS" "SSDC")
    local commands=("mem show stat 1" "mem show stat 2" "mem show stat 3" "mem show stat 15" "mem show stat 30")
    local ports=("988" "988" "988" "988" "988")
    for i in $(seq 0 `expr ${#modules[@]} - 1`);do
        module=${modules[$i]}
        command=${commands[$i]}
        port=${ports[$i]}
        if [ X"${module}" == X"SSDC" -a ${SSDC_NEED_CHK} -eq 0 ]; then
            continue
        fi
        
        $G_MML_FILE_PATH/ShowInspectMml $localip $port "${command}" >${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment 2>&1
        iRet=$?
        if [ 0 -eq  ${iRet} ];then
            dos2unix ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment >/dev/null 2>&1
            PooId="$(cat ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment | grep -Poa '(?<=PooId )[0-9]+' | tr -t '\n' ' ')"
            >${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment
            for pool in ${PooId};do
                sleep 1
                $G_MML_FILE_PATH/ShowInspectMml $localip $port "${command} ${pool}" >> ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment  2>&1
                iRet=$?
                if [ 0 -ne ${iRet} ];then
                    LOG "[$LINENO]${module}[ERR]${module} exec cmd failed, $port ${command} ${pool}"
                    continue
                fi
            done
            dos2unix ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment >/dev/null 2>&1
            cat ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment | grep Freearea | awk '{print $NF}' | grep -wvq 0 >/dev/null 2>&1
            if [ $? -ne 0 ];then
                echo "${module}[ERR]${module} all pools have no memory left!" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            fi
        else
            LOG "[$LINENO]${module}[ERR]${module} exec cmd failed,$port ${command}"
        fi
        sleep 1
    done
    local vmallocNum=$(cat /proc/vmallocinfo |wc -l)
    if [ ${vmallocNum:-0} -gt 100000 ];then
        echo "vmalloc[ERR]The number of vmallocinfo in the system exceeds 100000!" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    fi
    LOG "[$LINENO]vmalloc vmallocNum : ${vmallocNum:-0}"
    rm -f ${G_REMOTE_TMP_FILE}${Remote_filename}_CheckMemFragment
}

function GetModuleMem()
{
    local module=$1
    local command=$2
    local port=$3
    local tmpinfo=""
    local filters="PooId 0"
    local succInspectResult=""

    echo "[${module}]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1

    if [ "${module}" == "NOFS" ];then
        filters="PooId 6"
    elif [[ "${module}" == "NV" ]] || [[  "${module}" == "TRNS" ]] || [[ "${module}" == "TRNS_LOG" ]];then
        filters="total size"
    elif [ "${module}" == "CA_PoolId3_submodule" ];then
        sed -i 's/CA_PoolId3_submodule/CA/g' ${G_REMOTE_TMP_FILE}${Remote_filename}
        filters="PooId 3"
    elif [ "${module}" == "BASE" ];then
        filters="PooId 1"
    fi

    for i in $(seq 1 3);do
        iRet=""
        $G_MML_FILE_PATH/ShowInspectMml $localip $port "${command}" >${G_REMOTE_TMP_FILE}${Remote_filename}_${module} 2>&1
        iRet=$?
        if [ 0 -eq  ${iRet} ];then
            dos2unix ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} >/dev/null 2>&1
            if [ "${module}" == "MDS" ];then
                MDS_LeftMemSize=$( grep "LeftMemSize" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} | awk -F"=" '{print $2}' |tr -d " " )
                MDS_ResvMemSize=$( grep "ResvMemSize" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} | awk -F"=" '{print $2}' |tr -d " " )
                echo "MDS_LeftMemSize=${MDS_LeftMemSize},MDS_ResvMemSize=${MDS_ResvMemSize}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                break
            elif [[ "${module}" == "NV" ]] || [[  "${module}" == "TRNS" ]] || [[ "${module}" == "TRNS_LOG" ]];then
                local totalnum=$(grep "${filters}" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} | wc -l )
                if [ $totalnum -eq 0 ];then
                    if [ "${i}" -eq "3" ];then
                        echo "${module}[ERR]${module} get date fialed,please inspect again!"
                        LOG "[$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                else
                    tmpinfo=$(grep "${filters}" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module})
                    tmpinfo=$(GetModuleMemFormatLoaded "${tmpinfo}")
                    echo "${tmpinfo}" >> ${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                    break
                fi
            else
                grep "${filters}" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "${i}" -eq "3" ];then
                        echo "${module}[ERR]${module} get date fialed,please inspect again!"
                        LOG "[$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            fi
        fi
        sleep 1
    done
    if [ "3" == "${i}" -a 0 -ne $iRet ];then
        LOG "[$LINENO]Module ${module} get data failed"
        echo "$module[ERR]Module ${module} get data failed" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    fi
}

function NASmemUsageSummary()
{
    local memSummaryInfo=$(nas_proto_pum -m nas_mem -c get_summary |grep "\["|grep "[0-9]")
    local module=$1
    echo "$memSummaryInfo"| while read line
    do
        echo "[$module]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        poolid=$(echo ${line} | awk '{print $1}'|awk -F"[" '{print $1}')
        if [ "10" == "$poolid" ];then
            Pool=$(echo ${line}|awk -F"]" '{print $1}'|awk -F"[" '{print $2}')
            totalsize=$(echo ${line} | awk '{print $2}')
            freesize=$(echo ${line} |awk '{print $3}')
            usedsize=$(echo ${line} |awk '{print $4}')

        else
            Pool=$(echo ${line}|awk -F"]" '{print $1}'|awk -F"[" '{print $2}')
            totalsize=$(echo ${line} | awk '{print $3}')
            freesize=$(echo ${line} |awk '{print $4}')
            usedsize=$(echo ${line} |awk '{print $5}')
        fi
        echo "Pool[${Pool}]:total size:${totalsize},free size:${freesize},usedsize:${usedsize}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        LOG "[$LINENO]Pool[${Pool}]:total size:${totalsize},free size:${freesize},usedsize:${usedsize}"

    done
}
function OSdlmUsageSummary()
{
    local memSummaryInfo=$(echo -n "showmp" >/proc/dlm_info;cat /proc/dlm_info)
    result=$(echo "${memSummaryInfo}" |
        awk 'BEGIN{ \
                result="OS_dlm_normal"; \
            } \
            { \
                if (($3~/^[0-9]+$/) && ($4~/^[0-9]+$/)) { \
                    if (($8~/plugin/) && ($9~/Pool/)) { \
                    } \
                    else { \
                        if (($4 + 0) > ($3 + 0)) { \
                            printf ("os_dlm[ERR]OS_dlm mem over id %s,num %s,used %s\n",$1,$3,$4); \
                            result="osdlm_alarm"; \
                        } \
                    } \
                } \
            } \
            END { \
                print result; \
            }' \
        )
    echo "${result}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    LOG "[$LINENO]result=${result}"
}

function main()
{
    local succInspectResult=""
    local Thrnasprocnum="10"
    local version=$(cat /opt/huawei/deploy/package/version |grep "ProductVersion"|awk -F= '{print $2}')
    #获取需要执行MML命令的各模块信息
    MODULEMEM=("MON" "BJM_Server" "BJM_Worker" "CA" "MDS" "DAB" "DS" "OBS" "CM_Kernel" "CM_User" "NV" "TRNS" "TRNS_LOG" "MONC" "NOFS" "DLM" "BASE")
    modulecommand=("mem show stat 5" "mem show stat 21" "mem show stat 21" "mem show stat 3"  "mds show lru" "mem show stat 17" "mem show stat 2" "mem show stat 9" "mem show stat 8" "mem show stat 7" "nvmem show stat 5" "nvmem show stat 6" "nvmem show stat 7" "mem show stat 4" "mem show stat 15" "mem show stat 6" "mem show stat 6")
    moduleport=("4004" "4104" "4108" "988" "988" "988" "988" "988" "988" "4016" "988" "988" "988" "988" "988" "988" "988")
    for i in $(seq 0 `expr ${#MODULEMEM[@]} - 1`);do
        module=${MODULEMEM[$i]}
        command=${modulecommand[$i]}
        port=${moduleport[$i]}
        if [ "MON" == "$module" ];then
            GetModuleMem "$module" "$command" "$port"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "MON_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            MON_Used=$(ps -C snas_mon v | grep -v "COMMAND" | awk '{print $8}')
            echo "MON_Used=${MON_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$LINENO]succInspectResult=[${succInspectResult}] MON_Used=${MON_Used}"
            continue
        fi
        if [ "BJM_Server" == "${module}" ];then
            GetModuleMem "${module}" "${command}" "${port}"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "BJM_Server_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            BJM_Server_Used=$(ps -C snas_bjm_server v | grep -v "COMMAND" | awk '{print $8}')
            echo "BJM_Server_Used=${BJM_Server_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$LINENO]succInspectResult=[${succInspectResult}]  BJM_Server_Used=${BJM_Server_Used}"
            continue
        fi
        if [ "BJM_Worker" == "${module}" ];then
            GetModuleMem "${module}" "${command}" "${port}"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "BJM_Worker_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            BJM_Worker_Used=$(ps -C snas_bjm_worker v | grep -v "COMMAND" | awk '{print $8}')
            echo "BJM_Worker_Used=${BJM_Worker_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$LINENO]succInspectResult=[${succInspectResult}]  BJM_Worker_Used=${BJM_Worker_Used}"
            continue
        fi
        if [ "CA" == "$module" ];then
            if [ 1 -eq $node_service_type -a 0 -eq $needtocheck ];then
                GetModuleMem "$module" "$command" "$port"
                succInspectResult=$(successiveMemInspect "ca.0")

                echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                LOG "[$LINENO]succInspectResult=[${succInspectResult}]"
                service_type=$( grep -w "^serviceType" "${ConfigFile}" |tr -d " "|awk -F"=" '{print $2}' )
                #只有视频监控场景才检查PooId 3
                if [ "X${Surveillance}" == "X${service_type}" ];then
                    GetModuleMem "CA_PoolId3_submodule" "$command" "$port"
                    succInspectResult=$(successiveMemInspect "ca.3")
                    echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                fi
            fi
            continue
        fi
        GetModuleMem "${module}" "${command}" "${port}"
        succInspectResult=$(successiveMemInspect "${module}")
        echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        LOG "[$LINENO]module=[${module}] port=[${port}],succInspectResult=[${succInspectResult}]"
    done
    #获取NAS模块的内存信息
    if [ 1 -eq ${node_service_type} -a 0 -eq ${needtocheck} ];then
        NASmemUsageSummary "NAS"
        nasprocnum=$(ps -efww |grep watchdog_nas | grep -v grep |wc -l)
        if [ ${nasprocnum} -gt ${Thrnasprocnum} ];then
            echo "nas[ERR]module NAS process watchdog_nas over 10" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$LINENO]nas[ERR]module NAS process watchdog_nas over 10"
        fi
    fi
    #获取BJM Manager模块内存信息
    echo "[BJM_Manager]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    succInspectResult=$(successiveMemInspect "BJM_Manager")
    echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    #arm版本对进程名的长度有限制
    local BJM_Manager_Used=$(ps -C snas_bjm_manage v | grep -v "COMMAND" | awk '{print $8}')
    echo "BJM_Manager_Used=${BJM_Manager_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    LOG "[$LINENO]succInspectResult=[${succInspectResult}] BJM_Manager_Used=${BJM_Manager_Used}"
    #获取Replication模块内存信息，只有9000场景需要检查远程复制模块
    if [ 1 -eq $node_service_type -a "48" -eq "${ProductType}" ];then
        echo "[Replication]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        local Replication_Used=$(ps -C snas_rep v | grep -v "COMMAND" | awk '{print $8}')
        echo "Replication_Used=${Replication_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        LOG "[$LINENO]Replication_Used=${Replication_Used}"
    fi
    #获取RPC模块内存信息
    echo "[RPC]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    succInspectResult=$(successiveMemInspect "RPC")
    if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
        succInspectResult=$(successiveMemInspect "RPC_Page")
    fi
    echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    local RPC_memused=$(cat /proc/sys/lnet/memused | awk '{print $1}')
    local RPC_pageused=$(cat /proc/sys/lnet/pageused | awk '{print $1}')
    echo "RPC_memused=${RPC_memused},RPC_pageused=${RPC_pageused}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    LOG "[$LINENO]succInspectResult=[${succInspectResult}] RPC_memused=${RPC_memused},RPC_pageused=${RPC_pageused}"

    #获取SSDC内存信息 MmlBatch 988 "mem show stat 30" 仅限V5；(V3R6不支持)  统计剩余内存，不低于50M
    local ssdc_memfree="--"
    local mempoolcnt=0
    $G_MML_FILE_PATH/ShowInspectMml $localip 988 "mem show stat 30" | grep -A11 -E "Pool\[ssdc.1\]:total size"|grep "^Freearea\[" >${G_REMOTE_TMP_FILE}${Remote_filename}_SSDC 2>&1
    if [ $? -eq 0 ];then
        ssdc_memfree=0
        # 单位K，4k 8k---4M,最后显示再转换成M
        # 如果节点内存本身就不够，是不会拉起SSDC二级缓存的，由于MML命令执行不管对错，都返回0，所以，通过MML的内容判定SSDC是否启动
        size=("4" "8" "16" "32" "64" "128" "256" "512" "1024" "2048" "4096")
        while read line; do
            mempoolcnt=$((mempoolcnt+1))
            modle=$(echo $line | grep -Po '(?<=Freearea\[)[0-9]+')
            freeNum=$(echo $line | grep -Po '(?<=:Free )[0-9]+')
            ssdc_memfree=$[ $ssdc_memfree + $freeNum * ${size[$modle]} ]
        done < ${G_REMOTE_TMP_FILE}${Remote_filename}_SSDC
    fi
    if [ ${mempoolcnt} -ne 0 ]; then
        echo "[SSDC]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        echo "SSDC_memfree=${ssdc_memfree}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        LOG "[$LINENO] SSDC_memfree=${ssdc_memfree} "
    else
        SSDC_NEED_CHK=0
        LOG "[$LINENO]node have no ssdc module,memory size is:$(free -m | grep Mem | awk '{print $2}')M"
    fi

    #OS_dlm模块内存信息只有在V3R6C00SPC200及后续版本才存在,UDS不支持OS_dlm查询
    if [ "${ProductType}" != "80" ];then
        LOG "[$LINENO] check OS_dlm module"
        OSdlmUsageSummary
    fi
    CheckMemFragment
    echo "[END]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    cat ${G_REMOTE_TMP_FILE}${Remote_filename} >>${RESULTFILE} 2>&1
}

>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
main
sh ${G_MML_FILE_PATH}/dynamic_mem_inspect.sh "init" >/dev/null 2>&1
 if [ ! -d /var/log/mds_state_check/tmpdata ];then
    mkdir -p /var/log/mds_state_check/tmpdata >/dev/null 2>&1
fi

rm -f ${G_REMOTE_TMP_FILE}${Remote_filename}*
