#!/bin/sh

G_REMOTE_TMP_FILE="/tmp/tmpfile"
G_MML_FILE_PATH="/opt/huawei/snas/script/inspect_mml/"
Remote_filename="MemPoolUsage"
ConfigFile="/opt/huawei/snas/etc/snas.ini"
#Ƶس
Surveillance="surveillance"
LOG_FILE="/var/log/inspect.log"
needtocheck=1

localip=$1   
if [ "$localip" == "" ];then
    echo "[GetMemPoolUsageInfo]: localip is NULL"
    exit 0
fi

#жǷudsƷ
ProductType=$(cat "/opt/huawei/snas/etc/snas.ini"|grep productType|awk -F= '{print $2}')
#ȡڵ洢
node_service_type=$(egrep '[[]|^'node_service_type'=' /opt/huawei/snas/etc/snas.ini | tr -d '\n' | grep -Po '(?<=[[]'NODE'[]]'node_service_type'=)[0-9]+')
if [ "48" == "${ProductType}" ] || [ "91" == "${ProductType}" ];then
    if [ "X1" == "X${node_service_type}" ];then
        needtocheck=0
    fi
fi

function LOG()
{
   time=$(date)
   echo [${time}][$$]$@ >> $LOG_FILE
}

function getMemUsedHisRecFile()
{
    echo "/tmp/memUsedHisRecord"
}

function successiveMemInspect()
{
    local module=$1
    local hisInfo=""
    if [ -e $(getMemUsedHisRecFile) ]; then
        hisInfo=$(sed -n "/${module}\s/p" $(getMemUsedHisRecFile))
        echo "${hisInfo}" |
            awk 'BEGIN{ \
                isAlarm=0; \
                memInfo=""; \
            } \
            { \
                isAlarm=$2; \
                for (i=4; i<=NF; i++) { \
                    memInfo=sprintf("%s %-10s", memInfo, $i); \
                } \
            } \
            END{ \
                if(isAlarm == 1) { \
                    print "memSuccessiveArise=1,"memInfo; \
                } \
                else { \
                    print "memSuccessiveArise=0"; \
                } \
            }'
    else
        echo "memSuccessiveArise=0"
    fi
}

function CheckCompanionLogCount()
{
    local module="$1"
    local tmpinfo="$2"
    local result=$(echo "${tmpinfo}" |
        awk -F',' 'BEGIN{
                totalUsed=0;
            }
            {
                if ($4~/used: [ ]+[0-9]+/)
                {
                    #printf("%s\n", $4)
                    split($4, a, ":");
                    tmpused=a[2]
                    totalUsed=totalUsed + tmpused;
                }
            }
            END {
                printf("%d" ,totalUsed);
            }'
        )

    if [ $result -gt 80000 ];then
        echo "${module}[ERR]${module} associated log count($result) is greater than 80000!" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    fi
}
function GetModuleMem()
{
    local module=$1 
    local command=$2
    local port=$3   
    local succInspectResult=""
    #CAģĬϳڴʹֵ50,ֵ70
    local CA_DefaultScenePer="50"
    local CA_OtherSceanPer="70"
    echo "[${module}]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
    for i in $(seq 1 3);do
        iRet=""
        $G_MML_FILE_PATH/ShowInspectMml $localip $port "${command}" >${G_REMOTE_TMP_FILE}${Remote_filename}_${module} 2>&1 
        iRet=$?
        if [ 0 -eq  $iRet ];then
	    dos2unix ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} | dos2unix >/dev/null 2>&1
            #־
            if [ "${module}" == "TRNS_LOG" ];then
                tmpinfo=$(grep "Cache:Kmalloc, Size:      304" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module})
                CheckCompanionLogCount "${module}" "$tmpinfo"
            elif [ "${module}" == "DS" ] ;then
                tmpinfo=$(grep "Cache:S_Kmalloc2-4096" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module})
                CheckCompanionLogCount "${module}" "$tmpinfo"
            fi
            if [ "$module" == "NOFS" ];then
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "nofs.6" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "${iRet}" -eq "3" ];then
                        echo "$module[ERR]${module} get date fialed,please inspect again!"
                        LOG "[$FUNCNAME][$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            elif [ "${module}" == "DLM" ];then
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "base.0" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "${i}" -eq "3" ];then
                        echo "${module}[ERR]${module} get date fialed,please inspect again!"
                        LOG "[$FUNCNAME][$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            elif [ "$module" == "CA" ];then
	        #S3ڵ㲻
	        if [ "1" == ${needtocheck} ];then
		    continue
		fi
	        service_type=$( grep -w "^serviceType" "${ConfigFile}" |tr -d " "|awk -F"=" '{print $2}' ) 
		if [ "other" == "${service_type}" ];then
		    DesPercentage="${CA_DefaultScenePer}"
		else
		    DesPercentage="${CA_OtherSceanPer}"
		fi
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "ca.0" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "${i}" -eq "3" ];then
                        echo "$module[ERR]$module get date fialed,please inspect again!"
                        LOG "[$FUNCNAME][$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi		 
		
		info=$( grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "ca.0" )
		totalsize=$( echo ${info} | awk -F"(" '{print $2}' | awk -F")" '{print $1}' | awk '{print $1}')
		freesize=$( echo ${info} | awk -F"(" '{print $3}' | awk -F")" '{print $1}' | awk '{print $1}' )
		usedSize=$(expr ${totalsize} \- ${freesize})
                usedSize_temp=$(expr ${usedSize} \* 100)
                percentage=$(expr ${usedSize_temp} \/ ${totalsize})
		if [ ${percentage} -le ${DesPercentage} ];then
		    echo "$module percentage ${percentage} over the threshould percentage ${Threpercentage}!" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
		fi
		#ֻƵسżPooId 3
		if [ "X${Surveillance}" == "X${service_type}" ];then
		    echo "[$module]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
		    grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "ca.3" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                    if [ $? -eq 0 ];then
                    	succInspectResult=$(successiveMemInspect "ca.3")
                    	echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                    else
                    	if [ "$i" -eq "3" ];then
                            echo "${module}[ERR]${module} get date fialed,please inspect again!"
                    	fi
                    	continue
                    fi
		    info=$( grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "ca.3" )
		    totalsize=$( echo ${info} | awk -F"(" '{print $2}' | awk -F")" '{print $1}' | awk '{print $1}')
		    freesize=$( echo ${info} | awk -F"(" '{print $3}' | awk -F")" '{print $1}' | awk '{print $1}' )
		    usedSize=$(expr ${totalsize} \- ${freesize})
                    usedSize_temp=$(expr ${usedSize} \* 100)
                    percentage=$(expr ${usedSize_temp} \/ ${totalsize})
		    if [ ${percentage} -le ${DesPercentage} ];then
		        echo "$module percentage ${percentage} over the threshould percentage ${Threpercentage}!" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
		    fi
		fi
                
                
             elif [ "$module" == "MDS" ];then
	     	#S3ڵ㲻
	        if [ "1" == ${needtocheck} ];then
		    continue
		fi
                MDS_LeftMemSize=$( grep "LeftMemSize" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} | awk -F"=" '{print $2}' |tr -d " " ) 
		MDS_ResvMemSize=$( grep "ResvMemSize" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} | awk -F"=" '{print $2}' |tr -d " " ) 
                echo "MDS_LeftMemSize=${MDS_LeftMemSize},MDS_ResvMemSize=${MDS_ResvMemSize}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
		break
            else
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "${i}" -eq "3" ];then
                        echo "${module}[ERR]${module} get date fialed,please inspect again!"
                        LOG "[$FUNCNAME][$LINENO]${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            fi
        fi
        sleep 1
    done
    if [ "3" == "${i}" -a 0 -ne $iRet ];then
        LOG "[${FUNCNAME}][$LINENO]Module ${module} get data failed"
        echo "$module[ERR]Module ${module} get data failed" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
    fi
}

function NASmemUsageSummary()
{
    local memSummaryInfo=$(nas_proto_pum -m nas_mem -c get_summary |grep "\["|grep "[0-9]")
    local module=$1
    echo "$memSummaryInfo"| while read line
    do
        echo "[$module]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
        poolid=$(echo ${line} | awk '{print $1}'|awk -F"[" '{print $1}')
        if [ "10" == "$poolid" ];then
            Pool=$(echo ${line}|awk -F"]" '{print $1}'|awk -F"[" '{print $2}')
            totalsize=$(echo ${line} | awk '{print $2}')
            freesize=$(echo ${line} |awk '{print $3}')
            usedsize=$(echo ${line} |awk '{print $4}')
        
        else
            Pool=$(echo ${line}|awk -F"]" '{print $1}'|awk -F"[" '{print $2}')
            totalsize=$(echo ${line} | awk '{print $3}')
            freesize=$(echo ${line} |awk '{print $4}')
            usedsize=$(echo ${line} |awk '{print $5}')
        fi
        echo "Pool[${Pool}]:total size:${totalsize},free size:${freesize},usedsize:${usedsize}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
        LOG "[$FUNCNAME][$LINENO]Pool[${Pool}]:total size:${totalsize},free size:${freesize},usedsize:${usedsize}"
        
    done
}
function OSdlmUsageSummary()
{
    local memSummaryInfo=$(echo -n "showmp" >/proc/dlm_info;cat /proc/dlm_info)
    result=$(echo "${memSummaryInfo}" |
        awk 'BEGIN{ \
                result="OS_dlm_normal"; \
            } \
            { \
                if (($3~/^[0-9]+$/) && ($4~/^[0-9]+$/)) { \
                    if (($8~/plugin/) && ($9~/Pool/)) { \
                    } \
                    else { \
                        if (($4 + 0) > ($3 + 0)) { \
                            printf ("os_dlm[ERR]OS_dlm mem over id %s,num %s,used %s\n",$1,$3,$4); \
                            result="osdlm_alarm"; \
                        } \
                    } \
                } \
            } \
            END { \
                print result; \
            }' \
        )
    echo "${result}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    LOG "[$FUNCNAME][$LINENO]result=${result}"
}

function main()
{
    local succInspectResult=""
    local Thrnasprocnum="10"
    local version=$(cat /opt/huawei/deploy/package/version |grep "ProductVersion"|awk -F= '{print $2}')
    #ȡҪִMMLĸģϢ
    MODULEMEM=("MON" "BJM_Server" "BJM_Worker" "CA" "MDS" "DAB" "DS" "OBS" "CM_Kernel" "CM_User" "NV" "TRNS" "TRNS_LOG" "MONC" "NOFS" "DLM") 
    modulecommand=("mem show stat 5" "mem show stat 21" "mem show stat 21" "mem show stat 3"  "mds show lru" "mem show stat 17" "mem show stat 2" "mem show stat 9" "mem show stat 8" "mem show stat 7" "nvmem show stat 5" "nvmem show stat 6" "nvmem show stat 7" "mem show stat 4" "mem show stat 15" "mem show stat 6")
    moduleport=("4004" "4104" "4108" "988" "988" "988" "988" "988" "988" "4016" "988" "988" "988" "988" "988" "988")
    for i in $(seq 0 `expr ${#MODULEMEM[@]} - 1`);do
        module=${MODULEMEM[$i]}
        command=${modulecommand[$i]}
        port=${moduleport[$i]}
        if [ "MON" == "$module" ];then
            GetModuleMem "$module" "$command" "$port"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "MON_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            MON_Used=$(ps -C snas_mon v | grep -v "COMMAND" | awk '{print $8}')
            echo "MON_Used=${MON_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$FUNCNAME][$LINENO]succInspectResult=[${succInspectResult}] MON_Used=${MON_Used}"
            continue
        fi
        if [ "BJM_Server" == "${module}" ];then
            GetModuleMem "${module}" "${command}" "${port}"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "BJM_Server_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            BJM_Server_Used=$(ps -C snas_bjm_server v | grep -v "COMMAND" | awk '{print $8}')
            echo "BJM_Server_Used=${BJM_Server_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$FUNCNAME][$LINENO]succInspectResult=[${succInspectResult}]  BJM_Server_Used=${BJM_Server_Used}" 
            continue
        fi
        if [ "BJM_Worker" == "${module}" ];then
            GetModuleMem "${module}" "${command}" "${port}"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "BJM_Worker_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            BJM_Worker_Used=$(ps -C snas_bjm_worker v | grep -v "COMMAND" | awk '{print $8}')
            echo "BJM_Worker_Used=${BJM_Worker_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$FUNCNAME][$LINENO]succInspectResult=[${succInspectResult}]  BJM_Worker_Used=${BJM_Worker_Used}"
            continue
        fi
        if [ "CA" == "$module" ];then
            if [ 1 -eq $node_service_type -a 0 -eq $needtocheck ];then
                GetModuleMem "$module" "$command" "$port"
                #succInspectResult=$(successiveMemInspect "ca.0")
                
                #echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                continue
            fi
            continue
        fi
        GetModuleMem "${module}" "${command}" "${port}"
        succInspectResult=$(successiveMemInspect "${module}")
        echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        LOG "[$FUNCNAME][$LINENO]module=[${module}] port=[${port}],succInspectResult=[${succInspectResult}]"
    done
    #ȡNASģڴϢ
    if [ 1 -eq ${node_service_type} -a 0 -eq ${needtocheck} ];then
        NASmemUsageSummary "NAS"
        nasprocnum=$(ps -efww |grep watchdog_nas | grep -v grep |wc -l)
        if [ ${nasprocnum} -gt ${Thrnasprocnum} ];then
            echo "nas[ERR]module NAS process watchdog_nas over 10" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            LOG "[$FUNCNAME][$LINENO]nas[ERR]module NAS process watchdog_nas over 10"
        fi
    fi
    #ȡBJM ManagerģڴϢ
    echo "[BJM_Manager]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    succInspectResult=$(successiveMemInspect "BJM_Manager")
    echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    local BJM_Manager_Used=$(ps -C snas_bjm_manager v | grep -v "COMMAND" | awk '{print $8}')
    echo "BJM_Manager_Used=${BJM_Manager_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    LOG "[$FUNCNAME][$LINENO]succInspectResult=[${succInspectResult}] BJM_Manager_Used=${BJM_Manager_Used}"
    #ȡReplicationģڴϢֻ9000ҪԶ̸ģ
    if [ 1 -eq $node_service_type -a "48" -eq "${ProductType}" ];then
        echo "[Replication]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        local Replication_Used=$(ps -C snas_rep v | grep -v "COMMAND" | awk '{print $8}')
        echo "Replication_Used=${Replication_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        LOG "[$FUNCNAME][$LINENO]Replication_Used=${Replication_Used}"
    fi
    #ȡRPCģڴϢ
    echo "[RPC]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    succInspectResult=$(successiveMemInspect "RPC")
    if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
        succInspectResult=$(successiveMemInspect "RPC_Page")
    fi
    echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    local RPC_memused=$(cat /proc/sys/lnet/memused | awk '{print $1}')
    local RPC_pageused=$(cat /proc/sys/lnet/pageused | awk '{print $1}')
    echo "RPC_memused=${RPC_memused},RPC_pageused=${RPC_pageused}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    LOG "[$FUNCNAME][$LINENO]succInspectResult=[${succInspectResult}] RPC_memused=${RPC_memused},RPC_pageused=${RPC_pageused}" 
    #OS_dlmģڴϢֻV3R6C00SPC200汾Ŵ,UDS֧OS_dlmѯ
    if [ "${ProductType}" != "80" ];then
        LOG "[$FUNCNAME][$LINENO] check OS_dlm module"
        OSdlmUsageSummary
    fi
   
    echo "[END]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
}

>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
main
sh ${G_MML_FILE_PATH}/dynamic_mem_inspect.sh "init" >/dev/null 2>&1
 if [ ! -d /var/log/mds_state_check/tmpdata ];then
    mkdir -p /var/log/mds_state_check/tmpdata >/dev/null 2>&1 
fi
rm -f ${G_REMOTE_TMP_FILE}${Remote_filename}_* > /dev/null 2>&1
exit 0

