#!/bin/bash
set +x

G_INSPECT_MMLPATH="/opt/huawei/snas/script/inspect_mml"
source $G_INSPECT_MMLPATH/CheckItems
CurInspectNum="337"
CurInspectFun="$(GetInspectType $CurInspectNum)"
RESULTFILE="/tmp/tmpResult${CurInspectFun}"
>${RESULTFILE}

CHECK_PASSED=0
CHECK_FAILED=1
isPass=${CHECK_PASSED}

LOGFILE="/var/log/inspect.log"
NOFS_MOS_OBJECT_BEGIN=0
NOFS_MOS_OBJECT_END=7
SECOND_COLUMN_NAME="(ObjType ChunkCount Sum Percent):"

poolSatisfiedCount=0

#SMR盘相关阈值
LEFT_ZONE=(7000 6500 6000 5500 5000 4500 4000 3500 3000 2500 2000 1500 1000 0)
#除以1000为实际利用率
TSRate_INTERRUPT=(898 859 820 781 742 703 664 625 586 547 547 547 547 547)
TSRate_FC_NORMAL=(820 781 742 703 664 625 586 547 508 469 469 469 469 469)
TSRate_FC_FAULT=(664 575 540 505 470 435 400 365 330 300 300 300 300 300)
TSRate_ALARM=(742 703 664 625 586 547 508 469 430 0 0 0 0 0)

InterruptPools=""
InterruptPoolNum=0
FcPools=""
FcPoolNum=0
AlarmPools=""
AlarmPoolNum=0

function LOG()
{
    time=$(date)
    echo [${time}][$$][$CurInspectFun]$@ >> $LOGFILE
}

# 检查非SMR盘，mos空间占比情况
function checkNofsMosObjectUsage()
{
    local poolname=""
    local listmemberinfo_f=""
    local result=""
    local loop=0
    local mosObject_prefix=/var/tmp/disk_nofs_mos_objext_
    local mos_object_ind=""
    local tmpfile=""
    local str=""
    local str_piect=""
    local disk_nofs_usage=0
    local freechuntcnt=0
    local isSmrFlag=0
    local zoneleft_f="/var/tmp/zoneleft"

    # 判断当前节点是SMR节点
    /opt/huawei/snas/sbin/nofs-snas dumpzoneleft |grep -v "NAME" |grep -w "^P.*" > ${zoneleft_f}
    if [ $? -eq 0 ]; then
        isSmrFlag=1
    fi
    
    rm -f /var/tmp/nofs_pool_listmemberinfo*
    for pool in $( /opt/huawei/snas/sbin/nofs-snas listpool | awk '{print $1}' | grep P )
    do
        disk_nofs_usage=0
        freechuntcnt=0
        str="poolName:$pool||$SECOND_COLUMN_NAME"
        result=$( /opt/huawei/snas/sbin/nofs-snas listdiskmemberinfo ${pool} 2>/dev/null )
        if [ $? -ne 0 ]; then
            LOG "[$LINENO]failed to excute command:/opt/huawei/snas/sbin/nofs-snas listdiskmemberinfo ${pool}"
            continue
        fi

        listmemberinfo_f=$( echo "${result}" | awk -F 'file' '{print $2}' | sed -e "s/ *//g" -e "s/ *$//g" )
        if [ ! -f "${listmemberinfo_f}" -o ! -s "${listmemberinfo_f}" ]; then
            LOG "[$LINENO]failed to excute command:/opt/huawei/snas/sbin/nofs-snas listdiskmemberinfo ${pool}."
            continue
        fi

        cat "${listmemberinfo_f}" | grep -a index >${mosObject_prefix}_${pool}
        if [ ! -f "${mosObject_prefix}_${pool}" -o ! -s "${mosObject_prefix}_${pool}" ]; then
            LOG "[$LINENO]there is no index contents about P${pool}."
            continue
        fi

        RandomZoneTotalAlloc=0
        for ((loop=${NOFS_MOS_OBJECT_BEGIN};loop<=${NOFS_MOS_OBJECT_END};loop++))
        do
            str_piect=`awk '{if($3=='"$loop"')print}' ${mosObject_prefix}_${pool} | \
                awk '{sum+=$4}END{if(NR>0){printf "%d %d %d %1.2f\n", '$loop',NR,sum*4/1024,sum/NR/16384}else{printf "%d %d %d %1.2f\n", '$loop',0,0,0}}'`
            str="$str ($str_piect)"

            # type类型为4/5/6的对象是随机区对象 +type0
            if [ ${loop} -eq 4 -o ${loop} -eq 5 -o ${loop} -eq 6 -o ${loop} -eq 0 ]; then
                allocAll=$( echo "${str_piect}" | awk '{print $3}' )
                RandomZoneTotalAlloc=$((RandomZoneTotalAlloc+allocAll))
            fi

            if [ ${loop} -eq 6 ]; then
                disk_nofs_usage=$( echo "${str_piect}" | awk '{print $4}' | sed -e "s/^ *//g" -e "s/ *$//g" )
            fi

            if [ ${loop} -eq 7 ]; then
                freechuntcnt7=$( echo "${str_piect}" | awk '{print $2}' | sed -e "s/^ *//g" -e "s/ *$//g" )
            fi

            if [ ${loop} -eq 0 ]; then
                freechuntcnt0=$( echo "${str_piect}" | awk '{print $2}' | sed -e "s/^ *//g" -e "s/ *$//g" )
            fi
        done

        datacnt=$( cat ${mosObject_prefix}_${pool} | awk '{if($3==1||$3==2||$3==3)print}' | wc -l )
        metacnt=$( cat ${mosObject_prefix}_${pool} | awk '{if($3==4||$3==5||$3==6)print}' | wc -l )
        mixfree=$( cat ${mosObject_prefix}_${pool} | awk '{if($3==0||$3==7)print}' | wc -l )
        freechuntcnt=$((freechuntcnt7+freechuntcnt0))

        totalcnt=$[${datacnt}+${metacnt}+${mixfree}]
        datalimit=$[${totalcnt}*93/100+1]
        metalimit=$[${totalcnt}*8/100+1]

        str="$str||dataCount(type1-3):$datacnt||dataLimit:$datalimit||metaCount(type4-6):$metacnt||metalimit:$metalimit||totalCount(type07):$totalcnt||RandomZoneTotalAlloc:${RandomZoneTotalAlloc}"

        # 判断本节点的mos巡检项通过，且 超过1/2的盘满足"free+mixchunk的个数超过1000"的条件，则预埋MosReserve配置到128
        # free+mixchunk的个数超过1000
        if [ ${mixfree} -gt 1000 ];then
            let poolSatisfiedCount++
        fi

        # 检查SMR盘随机区占用率 大于70%不通过，[60%-70)建议优化，小于60%则通过
        if [ ${isSmrFlag} -eq 1 ]; then
            TotalAlloc=$[128 * 1024]
            Rate=$(echo ""|awk -v x=${RandomZoneTotalAlloc} -v y=${TotalAlloc} '{printf("%0.2f",x*100/y)}')
            if [ $(echo "$Rate >= 70" | bc) -eq 1 ];then
                isPass=$(check_pass ${isPass} 1)
                echo "[ERR]poolName:${pool}||${SECOND_COLUMN_NAME}random zone usage more than 70% " >>${RESULTFILE}
            elif [ $(echo "$Rate >= 60" | bc) -eq 1 ];then
                isPass=$(check_pass ${isPass} 4)
                echo "[ERR]poolName:${pool}||${SECOND_COLUMN_NAME}random zone usage between 60%-70% " >>${RESULTFILE}
            fi

            leftzone=$( cat ${zoneleft_f} |grep -aw "$pool" |awk '{print $2}' )

            str="$str||ZoneLeft:$leftzone"

            TSRate=$( echo ""|awk -v x=$Rate '{printf("%d",x*10)}' )
            checkSMRLeftzoneAndTSRate ${leftzone} ${TSRate} ${pool}
        fi

        disk_nofs_use_digit=$( echo "${disk_nofs_usage} * 100" | bc | awk -F. '{print $1}' )
        # type为6的利用率≥0.80(这个使用率实际是磁盘的nofs文件系统元数据的使用率)，且type为7和type0的FreeChunk的个数低于32个，巡检不通过
        if [ X"${disk_nofs_use_digit}" != X"" -a ${disk_nofs_use_digit} -ge 80 -a ${freechuntcnt} -lt 32 ]; then
            isPass=$(check_pass ${isPass} 1)
            echo "[ERR]${str}" >> ${RESULTFILE} 2>&1

        # type为6的利用率≥0.75(这个使用率实际是磁盘的nofs文件系统元数据的使用率)，且type为7和type0的FreeChunk的个数低于32个，巡检建议优化
        elif [ X"${disk_nofs_use_digit}" != X"" -a ${disk_nofs_use_digit} -ge 75 -a ${freechuntcnt} -lt 32 ]; then
            isPass=$(check_pass ${isPass} 4)
            echo "[ERR]${str}" >> ${RESULTFILE} 2>&1
        else
            echo "${str}||MosReserve:--" >> ${RESULTFILE} 2>&1
        fi

        rm -f ${mosObject_prefix}_${pool}
        rm -f ${listmemberinfo_f}
    done

    rm -f /var/tmp/nofs_pool_listmemberinfo*
    [ -f ${zoneleft_f} ] && rm -f ${zoneleft_f}

    if [ "${InterruptPools}X" != "X" ];then
        isPass=$( check_pass ${isPass} 1 )
        echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}${InterruptPoolNum} disks meet the interruption criteria:$InterruptPools" >> ${RESULTFILE} 2>&1
    fi
    if [ "${FcPools}X" != "X" ];then
        isPass=$( check_pass ${isPass} 1 )
        echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}${FcPoolNum} disks meet the flow control criteria:$FcPools" >> ${RESULTFILE} 2>&1
    fi
    if [ "${AlarmPools}X" != "X" ];then
        isPass=$( check_pass ${isPass} 1 )
        echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}${AlarmPoolNum} disks meet the alarm criteria:$AlarmPools" >> ${RESULTFILE} 2>&1
    fi

    return
}

# 输入：单盘leftzone和随机区占用率TSRate
# 根据单盘leftzone和随机区占用率TSRate判断 哪些盘达到流控、告警、及业务中断的标准
function checkSMRLeftzoneAndTSRate()
{
    leftzone=$1
    TSRate=$2
    pool=$3
    Cnt=$( echo ${#LEFT_ZONE[*]} )

    if [[ "${leftzone}X" == "X" || "${TSRate}X" == "X" ]];then
        LOG "[$LINENO] leftzone or TSRate is empty, leftzone:${leftzone}, TSRate:${TSRate}"
        return
    fi

    local i=0
    for((i=0;i<${Cnt};i++))
    do
        if [[ ${leftzone} -ge ${LEFT_ZONE[$i]} ]];then
            break
        fi
    done
    if [[ ${i} -ge ${Cnt} ]]; then
        LOG "[$LINENO] leftzone abnormal,i:${i},Cnt:${Cnt}"
        return
    fi

    # 判断是否满足业务中断条件
    if [[ ${TSRate} -ge ${TSRate_INTERRUPT[$i]} ]];then
        InterruptPools="$InterruptPools $pool"
        let InterruptPoolNum++
    fi

    TSRate_FC=(${TSRate_FC_NORMAL[@]})
    checkDsFaultState
    # 故障场景使用故障流控阈值
    if [[ $? -ne 0 ]];then
        TSRate_FC=(${TSRate_FC_FAULT[@]})
    fi
    # 判断是否满足业务流控条件
    if [[ ${TSRate} -ge ${TSRate_FC[$i]} ]];then
        FcPools="$FcPools $pool"
        let FcPoolNum++
    fi

    # 判断是否满足告警上报条件
    if [[ ${TSRate} -ge ${TSRate_ALARM[$i]} ]];then
        AlarmPools="$AlarmPools $pool"
        let AlarmPoolNum++
    fi

    return
}

function checkDsFaultState()
{
    DS_MAP_PATH="/var/tmp/checkdsmap"
    cat /proc/monc_dsmap > ${DS_MAP_PATH}

    if [[ ! -f ${DS_MAP_PATH} ]];then
        LOG "[$LINENO]ERR:Get dsmap failed"
        return 0
    fi

    ret=$( cat ${DS_MAP_PATH} | grep -e DS -e NodePool )
    if [[ "${ret}X" == "X" ]];then
        LOG "[$LINENO]ERR:Get dsmap fault"
        return 0        
    fi

    ret=$( cat ${DS_MAP_PATH} | grep -a DS | grep -a "Abnormal(1)" )
    if [[ "${ret}X" != "X" ]];then
        LOG "[$LINENO]ERR: dsmap is fault,state(1)"
        return 1
    fi

    ret=$( cat ${DS_MAP_PATH} | grep -a "Abnormal(3)" )
    if [[ "${ret}X" != "X" ]];then
        LOG "[$LINENO]ERR: dsmap is fault,state(3)"
        return 1
    fi

    return 0
}

# 预留freechunk，只针对针对7.1.1和7.1.1.SPC1版本
function checkMosReserve()
{
    local areaKey=$(GetAreaKey)
    local nofsini="/opt/huawei/snas/etc/nofs.ini"

    str="poolName:--||${SECOND_COLUMN_NAME}--||dataCount(type1-3):--||dataLimit:--||metaCount(type4-6):--||metalimit:--||totalCount(type07):--||RandomZoneTotalAlloc:--"
    poolTotalCount=$( /opt/huawei/snas/sbin/nofs-snas listpool | awk '{print $1}' | grep P | wc -l )
    if [ ${poolTotalCount} -eq 0 ];then
        LOG "[$LINENO]Result is empty, cmd:/opt/huawei/snas/sbin/nofs-snas listpool"
    else
        doubleSatisfiedCount=$((poolSatisfiedCount*2))
        LOG "[$LINENO] poolTotalCount:${poolTotalCount}, poolSatisfiedCount:${poolSatisfiedCount}."

        val=$( sed -n '/^ *\[Defrag\]/,/^ *\[/p' ${nofsini} | grep -aw "MosReserve" | awk '/^ *MosReserve/{print}' | sed 's/ //g' | awk -F= '{print $2}' )
        if [ -z "${val}" ];then
            # 判断本节点的mos巡检项通过 且 超过1/2的盘满足"free+mixchunk的个数超过1000"的条件，则预埋MosReserve配置到128
            if [ ${isPass} -eq 0 -a ${doubleSatisfiedCount} -ge ${poolTotalCount} -a "${areaKey}" == "true" ];then
                # 判断[Defrag]是否存在
                grep -aw "^ *\[Defrag\]" ${nofsini} >/dev/null 2>&1
                if [ $? -ne 0 ];then
                    # [Defrag]不存在，添加[Defrag] MosReserve=128
                    sed -i '/MosReserve *=/d' ${nofsini}
                    sed -i '/^\[END\]$/i\\[Defrag\]' ${nofsini}
                    sed -i '/^\[END\]$/i\MosReserve=128' ${nofsini}
                else
                    # [Defrag]存在，添加MosReserve=128
                    sed -i 's/^ *\[Defrag\]/\[Defrag\]/g' ${nofsini}
                    sed -i "/\[Defrag\]/a MosReserve=128" ${nofsini}
                fi

                # 更新配置
                /opt/huawei/snas/sbin/nofs-snas updateconfig
                if [ $? -ne 0 ];then
                    sleep 1
                    /opt/huawei/snas/sbin/nofs-snas updateconfig
                    if [ $? -ne 0 ];then
                        LOG "[$LINENO]failed to excute command:/opt/huawei/snas/sbin/nofs-snas updateconfig"
                        echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}updateconfig failed." >> ${RESULTFILE} 2>&1
                        isPass=1
                        return
                    fi
                fi
            fi
        fi
    fi

    # 配置完重新获取一遍
    val=$( sed -n '/^ *\[Defrag\]/,/^ *\[/p' ${nofsini} | grep -aw "MosReserve" | awk '/^ *MosReserve/{print}' | sed 's/ //g' | awk -F= '{print $2}' )
    if [ -z "${val}" ];then
        val="-"
    else
        # 查询当前内存中使用的MosReserve值
        val=$( /opt/huawei/snas/sbin/nofs-snas printnofsconfigure | grep -aw "MosReserve" | awk '{print $3}' )
        if [ $? -ne 0 ];then
            val="null"
            LOG "[$FUNCNAME][$LINENO]failed to excute command:/opt/huawei/snas/sbin/nofs-snas printnofsconfigure"
        fi
    fi

    echo "${str}||MosReserve:${val}" >> ${RESULTFILE} 2>&1
    return
}

checkNofsMosObjectUsage

checkMosReserve

echo "${CurInspectFun}_Pass ${isPass}" >> ${RESULTFILE} 2>&1
exit 0
