#!/bin/sh

G_REMOTE_TMP_FILE="/tmp/tmpfile"
G_MML_FILE_PATH="/opt/huawei/snas/script/inspect_mml/"
Remote_filename="MemPoolUsage"
ConfigFile="/opt/huawei/snas/etc/snas.ini"
#Ƶس
Surveillance="surveillance"
LOG_FILE="/var/log/inspect.log"
needtocheck=1

localip=$1   
if [ "$localip" == "" ];then
    echo "[GetMemPoolUsageInfo]: localip is NULL"
    exit 0
fi

#жǷudsƷ
ProductType=$(cat "/opt/huawei/snas/etc/snas.ini"|grep productType|awk -F= '{print $2}')
#ȡڵ洢
node_service_type=$(egrep '[[]|^'node_service_type'=' /opt/huawei/snas/etc/snas.ini | tr -d '\n' | grep -Po '(?<=[[]'NODE'[]]'node_service_type'=)[0-9]+')
if [ "48" == "${ProductType}" ] || [ "91" == "${ProductType}" ];then
    needtocheck=0
fi

function LOG()
{
    time=$(date)
   echo [${time}][$!]$@ >> $LOG_FILE
}

function getMemUsedHisRecFile()
{
    echo "/tmp/memUsedHisRecord"
}

function successiveMemInspect()
{
    local module=$1
    local hisInfo=""
    if [ -e $(getMemUsedHisRecFile) ]; then
        hisInfo=$(sed -n "/${module}\s/p" $(getMemUsedHisRecFile))
        echo "${hisInfo}" |
            awk 'BEGIN{ \
                isAlarm=0; \
                memInfo=""; \
            } \
            { \
                isAlarm=$2; \
                for (i=4; i<=NF; i++) { \
                    memInfo=sprintf("%s %-10s", memInfo, $i); \
                } \
            } \
            END{ \
                if(isAlarm == 1) { \
                    print "memSuccessiveArise=1,"memInfo; \
                } \
                else { \
                    print "memSuccessiveArise=0"; \
                } \
            }'
    else
        echo "memSuccessiveArise=0"
    fi
}

function GetModuleMem()
{
    local module=$1 
    local command=$2
    local port=$3   
    local succInspectResult=""
    echo "[${module}]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
    for i in $(seq 1 3);do
        iRet=""
        $G_MML_FILE_PATH/ShowInspectMml $localip $port "$command" >${G_REMOTE_TMP_FILE}${Remote_filename}_${module} 2>&1 
        iRet=$?
        if [ 0 -eq  $iRet ];then
	    dos2unix ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} >/dev/null 2>&1
            if [ "$module" == "NOFS" ];then
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "nofs.6" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "$i" -eq "3" ];then
                        echo "${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            elif [ "$module" == "DLM" ];then
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "base.0" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "$i" -eq "3" ];then
                        echo "${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            elif [ "$module" == "CA" ];then
	    	
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "ca.0" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "$i" -eq "3" ];then
                        echo "${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
		service_type=$( grep -w "^serviceType" "${ConfigFile}" |tr -d " "|awk -F"=" '{print $2}' )  
		#ֻƵسżPooId 3
		if [ "X${Surveillance}" == "X${service_type}" ];then
		    echo "[$module]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
		    grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} |grep "ca.3" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                    if [ $? -eq 0 ];then
                    	succInspectResult=$(successiveMemInspect "ca.3")
                    	echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                    else
                    	if [ "$i" -eq "3" ];then
                            echo "${module}[ERR]${module} get date fialed,please inspect again!"
                    	fi
                    	continue
                    fi
		fi
                
                
             elif [ "$module" == "MDS" ];then
	     	
                MDS_LeftMemSize=$( grep "LeftMemSize" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} | awk -F"=" '{print $2}' |tr -d " " ) 
		MDS_ResvMemSize=$( grep "ResvMemSize" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} | awk -F"=" '{print $2}' |tr -d " " ) 
                echo "MDS_LeftMemSize=${MDS_LeftMemSize},MDS_ResvMemSize=${MDS_ResvMemSize}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
		break
            else
                grep "total size" ${G_REMOTE_TMP_FILE}${Remote_filename}_${module} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                if [ $? -eq 0 ];then
                    break
                else
                    if [ "$i" -eq "3" ];then
                        echo "${module}[ERR]${module} get date fialed,please inspect again!"
                    fi
                    continue
                fi
            fi
        fi
        sleep 1
    done
    if [ "3" == "$i" -a 0 -ne $iRet ];then
        LOG "[${FUNCNAME}]Module ${module} get data failed"
        echo "${module}[ERR]Module ${module} get data failed" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
    fi
}

function NASmemUsageSummary()
{
    local memSummaryInfo=$(nas_proto_pum -m nas_mem -c get_summary |grep "\["|grep "[0-9]")
    local module=$1
    echo "$memSummaryInfo"| while read line
    do
        echo "[$module]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
        poolid=$(echo $line | awk '{print $1}'|awk -F"[" '{print $1}')
        if [ "10" == "$poolid" ];then
            Pool=$(echo $line|awk -F"]" '{print $1}'|awk -F"[" '{print $2}')
            totalsize=$(echo $line | awk '{print $2}')
            freesize=$(echo $line |awk '{print $3}')
            usedsize=$(echo $line |awk '{print $4}')
        
        else
            Pool=$(echo $line|awk -F"]" '{print $1}'|awk -F"[" '{print $2}')
            totalsize=$(echo $line | awk '{print $3}')
            freesize=$(echo $line |awk '{print $4}')
            usedsize=$(echo $line |awk '{print $5}')
        fi
        echo "Pool[$Pool]:total size:$totalsize,free size:$freesize,usedsize:$usedsize" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1 
    done
}
function OSdlmUsageSummary()
{
    local memSummaryInfo=$(echo -n "showmp" >/proc/dlm_info;cat /proc/dlm_info)
    result=$(echo "${memSummaryInfo}" |
        awk 'BEGIN{ \
                result="OS_dlm_normal"; \
            } \
            { \
                if (($3~/^[0-9]+$/) && ($4~/^[0-9]+$/)) { \
                    if (($8~/plugin/) && ($9~/Pool/)) { \
                    } \
                    else { \
                        if (($4 + 0) > ($3 + 0)) { \
                            printf ("os_dlm[ERR]OS_dlm mem over id %s,num %s,used %s\n",$1,$3,$4); \
                            result="osdlm_alarm"; \
                        } \
                    } \
                } \
            } \
            END { \
                print result; \
            }' \
        )
    echo "$result" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
}

function main()
{
    local succInspectResult=""
    local Thrnasprocnum="10"
    local version=$(cat /opt/huawei/deploy/package/version |grep "ProductVersion"|awk -F= '{print $2}')
    #ȡҪִMMLĸģϢ
    MODULEMEM=("MON" "BJM_Server" "BJM_Worker" "CA" "MDS" "DAB" "DS" "OBS" "CM_Kernel" "CM_User" "NV" "TRNS" "TRNS_LOG" "MONC" "NOFS" "DLM") 
    modulecommand=("mem show stat 5" "mem show stat 21" "mem show stat 21" "mem show stat 3"  "mds show lru" "mem show stat 17" "mem show stat 2" "mem show stat 9" "mem show stat 8" "mem show stat 7" "nvmem show stat 5" "nvmem show stat 6" "nvmem show stat 7" "mem show stat 4" "mem show stat 15" "mem show stat 6")
    moduleport=("4004" "4104" "4108" "988" "988" "988" "988" "988" "988" "4016" "988" "988" "988" "988" "988" "988")
    for i in $(seq 0 `expr ${#MODULEMEM[@]} - 1`);do
        module=${MODULEMEM[$i]}
        command=${modulecommand[$i]}
        port=${moduleport[$i]}
        if [ "MON" == "$module" ];then
            GetModuleMem "$module" "$command" "$port"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "MON_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            MON_Used=$(ps -C snas_mon v | grep -v "COMMAND" | awk '{print $8}')
            echo "MON_Used=${MON_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            continue
        fi
        if [ "BJM_Server" == "$module" ];then
            GetModuleMem "$module" "$command" "$port"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "BJM_Server_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            BJM_Server_Used=$(ps -C snas_bjm_server v | grep -v "COMMAND" | awk '{print $8}')
            echo "BJM_Server_Used=${BJM_Server_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            continue
        fi
        if [ "BJM_Worker" == "$module" ];then
            GetModuleMem "$module" "$command" "$port"
            succInspectResult=$(successiveMemInspect "${module}")
            if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
                succInspectResult=$(successiveMemInspect "BJM_Worker_pid")
            fi
            echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            BJM_Worker_Used=$(ps -C snas_bjm_worker v | grep -v "COMMAND" | awk '{print $8}')
            echo "BJM_Worker_Used=${BJM_Worker_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
            continue
        fi
        if [ "CA" == "$module" ];then
            if [ 1 -eq $node_service_type -a 0 -eq $needtocheck ];then
                GetModuleMem "$module" "$command" "$port"
                succInspectResult=$(successiveMemInspect "ca.0")
                
                echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
                continue
            fi
            continue
        fi
        GetModuleMem "$module" "$command" "$port"
        succInspectResult=$(successiveMemInspect "${module}")
        echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    done
    #ȡNASģڴϢ
    if [ 1 -eq $node_service_type -a 0 -eq $needtocheck ];then
        NASmemUsageSummary "NAS"
        nasprocnum=$(ps -ef |grep watchdog_nas | grep -v grep |wc -l)
        if [ $nasprocnum -gt $Thrnasprocnum ];then
            echo "nas[ERR]module NAS process watchdog_nas over 10" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        fi
    fi
    #ȡBJM ManagerģڴϢ
    echo "[BJM_Manager]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    succInspectResult=$(successiveMemInspect "BJM_Manager")
    echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    local BJM_Manager_Used=$(ps -C snas_bjm_manager v | grep -v "COMMAND" | awk '{print $8}')
    echo "BJM_Manager_Used=${BJM_Manager_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    
    #ȡReplicationģڴϢ
    if [ 1 -eq $node_service_type -a 0 -eq $needtocheck ];then
        echo "[Replication]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
        local Replication_Used=$(ps -C snas_rep v | grep -v "COMMAND" | awk '{print $8}')
        echo "Replication_Used=${Replication_Used}" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    fi
    #ȡRPCģڴϢ
    echo "[RPC]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    succInspectResult=$(successiveMemInspect "RPC")
    if [ "${succInspectResult}" == "memSuccessiveArise=0" ]; then
        succInspectResult=$(successiveMemInspect "RPC_Page")
    fi
    echo ${succInspectResult} >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    local RPC_memused=$(cat /proc/sys/lnet/memused | awk '{print $1}')
    local RPC_pageused=$(cat /proc/sys/lnet/pageused | awk '{print $1}')
    echo "RPC_memused=$RPC_memused,RPC_pageused=$RPC_pageused" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
    #OS_dlmģڴϢֻV3R6C00SPC200汾Ŵ
    if [ "V300R006C00SPC200" == "$version" ];then
        OSdlmUsageSummary
    fi
   
    echo "[END]" >>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
}

>${G_REMOTE_TMP_FILE}${Remote_filename} 2>&1
main
sh ${G_MML_FILE_PATH}/dynamic_mem_inspect.sh "init" >/dev/null 2>&1
 if [ ! -d /var/log/mds_state_check/tmpdata ];then
    mkdir -p /var/log/mds_state_check/tmpdata >/dev/null 2>&1 
fi
