#!/bin/bash
set +x

G_INSPECT_MMLPATH="/opt/huawei/snas/script/inspect_mml"
source $G_INSPECT_MMLPATH/CheckItems
CurInspectNum="337"
CurInspectFun="$(GetInspectType $CurInspectNum)"
RESULTFILE="${G_TMP_INSPECT_PATH}tmpResult${CurInspectFun}"
>${RESULTFILE}

CHECK_PASSED=0
CHECK_FAILED=1
isPass=${CHECK_PASSED}

LOGFILE="/var/log/inspect.log"
NOFS_MOS_OBJECT_BEGIN=0
NOFS_MOS_OBJECT_END=7

SECOND_COLUMN_NAME="(ObjType ChunkCount Sum Percent):"
isSmrFlag=0
poolSatisfiedCount=0
areaKey="$(GetAreaKey)"

#SMR盘相关阈值
LEFT_ZONE=(7000 6500 6000 5500 5000 4500 4000 3500 3000 2500 2000 1500 1000 0)
#除以1000为实际利用率
TSRate_INTERRUPT=(898 859 820 781 742 703 664 625 586 547 547 547 547 547)
TSRate_FC_NORMAL=(820 781 742 703 664 625 586 547 508 469 469 469 469 469)
TSRate_FC_FAULT=(664 575 540 505 470 435 400 365 330 300 300 300 300 300)
TSRate_ALARM=(742 703 664 625 586 547 508 469 430 0 0 0 0 0)

InterruptPools=""
InterruptPoolNum=0
FcPools=""
FcPoolNum=0
AlarmPools=""
AlarmPoolNum=0

function LOG()
{
    time=$(date)
    echo [${time}][$$][$CurInspectFun]$@ >> $LOGFILE
}

# 检查非SMR盘，mos空间占比情况
function checkNofsMosObjectUsage()
{
    local poolname=""
    local listmemberinfo_f=""
    local result=""
    local loop=0
    local mosObject_prefix=/var/tmp/disk_nofs_mos_objext_
    local mos_object_ind=""
    local tmpfile=""
    local str=""
    local str_piect=""
    local disk_nofs_usage=0
    local freechuntcnt=0
    local zoneleft_f="/var/tmp/zoneleft"

    # 判断当前节点是SMR节点
    /opt/huawei/snas/sbin/nofs-snas dumpzoneleft |grep -v "NAME" |grep -w "^P.*" > ${zoneleft_f}
    if [ $? -eq 0 ]; then
        isSmrFlag=1
    fi
    
    rm -f /var/tmp/nofs_pool_listmemberinfo*
    for pool in $( /opt/huawei/snas/sbin/nofs-snas listpool | awk '{print $1}' | grep P )
    do
        disk_nofs_usage=0
        freechuntcnt=0
        str="poolName:$pool||$SECOND_COLUMN_NAME"
        result=$( /opt/huawei/snas/sbin/nofs-snas listdiskmemberinfo ${pool} 2>/dev/null )
        if [ $? -ne 0 ]; then
            LOG "[$LINENO]failed to excute command:/opt/huawei/snas/sbin/nofs-snas listdiskmemberinfo ${pool}"
            continue
        fi

        listmemberinfo_f=$( echo "${result}" | awk -F 'file' '{print $2}' | sed -e "s/ *//g" -e "s/ *$//g" )
        if [ ! -f "${listmemberinfo_f}" -o ! -s "${listmemberinfo_f}" ]; then
            LOG "[$LINENO]failed to excute command:/opt/huawei/snas/sbin/nofs-snas listdiskmemberinfo ${pool}."
            continue
        fi

        cat "${listmemberinfo_f}" | grep -a index >${mosObject_prefix}_${pool}
        if [ ! -f "${mosObject_prefix}_${pool}" -o ! -s "${mosObject_prefix}_${pool}" ]; then
            LOG "[$LINENO]there is no index contents about P${pool}."
            continue
        fi

        RandomZoneTotalAlloc=0
        for ((loop=${NOFS_MOS_OBJECT_BEGIN};loop<=${NOFS_MOS_OBJECT_END};loop++))
        do
            str_piect=`awk '{if($3=='"$loop"')print}' ${mosObject_prefix}_${pool} | \
                awk '{sum+=$4}END{if(NR>0){printf "%d %d %d %1.2f\n", '$loop',NR,sum*4/1024,sum/NR/16384}else{printf "%d %d %d %1.2f\n", '$loop',0,0,0}}'`
            str="$str ($str_piect)"

            # type类型为4/5/6的对象是随机区对象 +type0
            if [ ${loop} -eq 4 -o ${loop} -eq 5 -o ${loop} -eq 6 -o ${loop} -eq 0 ]; then
                allocAll=$( echo "${str_piect}" | awk '{print $3}' )
                RandomZoneTotalAlloc=$((RandomZoneTotalAlloc+allocAll))
            fi

            if [ ${loop} -eq 6 ]; then
                disk_nofs_usage=$( echo "${str_piect}" | awk '{print $4}' | sed -e "s/^ *//g" -e "s/ *$//g" )
            fi

            if [ ${loop} -eq 7 ]; then
                freechuntcnt7=$( echo "${str_piect}" | awk '{print $2}' | sed -e "s/^ *//g" -e "s/ *$//g" )
            fi

            if [ ${loop} -eq 0 ]; then
                freechuntcnt0=$( echo "${str_piect}" | awk '{print $2}' | sed -e "s/^ *//g" -e "s/ *$//g" )
            fi
        done

        datacnt=$( cat ${mosObject_prefix}_${pool} | awk '{if($3==1||$3==2||$3==3)print}' | wc -l )
        metacnt=$( cat ${mosObject_prefix}_${pool} | awk '{if($3==4||$3==5||$3==6)print}' | wc -l )
        mixfree=$( cat ${mosObject_prefix}_${pool} | awk '{if($3==0||$3==7)print}' | wc -l )
        freechuntcnt=$((freechuntcnt7+freechuntcnt0))

        totalcnt=$[${datacnt}+${metacnt}+${mixfree}]
        datalimit=$[${totalcnt}*93/100+1]
        metalimit=$[${totalcnt}*8/100+1]

        str="$str||dataCount(type1-3):$datacnt||dataLimit:$datalimit||metaCount(type4-6):$metacnt||metalimit:$metalimit||totalCount(type07):$totalcnt||RandomZoneTotalAlloc:${RandomZoneTotalAlloc}"

        # 判断该盘是否满足配置定时任务的条件 1）或者2）满足，认为该盘满足条件
        # 1）datacnt < datalimit && metacnt < metalimit && free+mixchunk > 32 
        # 2）free+mixchunk > 2000时
        if [[ ${datacnt} -lt ${datalimit} ]] && [[ ${metacnt} -lt ${metalimit} ]] && [[ ${mixfree} -gt 32 ]] || [[ ${mixfree} -gt 2000 ]];then
            let poolSatisfiedCount++
        fi

        # 检查SMR盘随机区占用率 大于70%不通过，[60%-70)建议优化，小于60%则通过
        if [ ${isSmrFlag} -eq 1 ]; then
            TotalAlloc=$[128 * 1024]
            Rate=$(echo ""|awk -v x=${RandomZoneTotalAlloc} -v y=${TotalAlloc} '{printf("%0.2f",x*100/y)}')
            if [ $(echo "$Rate >= 70" | bc) -eq 1 ];then
                isPass=$(check_pass ${isPass} 1)
                echo "[ERR]poolName:${pool}||${SECOND_COLUMN_NAME}random zone usage more than 70% " >>${RESULTFILE}
            elif [ $(echo "$Rate >= 60" | bc) -eq 1 ];then
                isPass=$(check_pass ${isPass} 4)
                echo "[ERR]poolName:${pool}||${SECOND_COLUMN_NAME}random zone usage between 60%-70% " >>${RESULTFILE}
            fi

            leftzone=$( cat ${zoneleft_f} |grep -aw "$pool" |awk '{print $2}' )

            str="$str||ZoneLeft:$leftzone"

            TSRate=$( echo ""|awk -v x=$Rate '{printf("%d",x*10)}' )
            checkSMRLeftzoneAndTSRate ${leftzone} ${TSRate} ${pool}
        fi

        # 获取磁盘空间分配倍数
        result=$( /opt/huawei/snas/sbin/nofs-snas bigblockdump ${pool} 2>/dev/null )
        if [ $? -ne 0 ]; then
            LOG "[$LINENO]failed to excute command:/opt/huawei/snas/sbin/nofs-snas bigblockdump ${pool}"
        fi

        bigblockdump_f=$( echo "${result}" | awk -F 'file' '{print $2}' | sed -e "s/ *//g" -e "s/ *$//g" )
        if [ ! -f "${bigblockdump_f}" -o ! -s "${bigblockdump_f}" ]; then
            LOG "[$LINENO]failed to excute command:/opt/huawei/snas/sbin/nofs-snas bigblockdump ${pool},result file is empty."
            bigblockdump="- - - - - - -"
        else
            bigblockdump=$( grep -v "^$" ${bigblockdump_f} | head -1 )
        fi

        str="$str||bigblockdump:(${bigblockdump})"

        disk_nofs_use_digit=$( echo "${disk_nofs_usage} * 100" | bc | awk -F. '{print $1}' )
        # type为6的利用率≥0.80(这个使用率实际是磁盘的nofs文件系统元数据的使用率)，且type为7和type0的FreeChunk的个数低于32个，巡检不通过
        if [ X"${disk_nofs_use_digit}" != X"" -a ${disk_nofs_use_digit} -ge 80 -a ${freechuntcnt} -lt 32 ]; then
            isPass=$(check_pass ${isPass} 1)
            echo "[ERR]${str}" >> ${RESULTFILE} 2>&1

        # type为6的利用率≥0.75(这个使用率实际是磁盘的nofs文件系统元数据的使用率)，且type为7和type0的FreeChunk的个数低于32个，巡检建议优化
        elif [ X"${disk_nofs_use_digit}" != X"" -a ${disk_nofs_use_digit} -ge 75 -a ${freechuntcnt} -lt 32 ]; then
            isPass=$(check_pass ${isPass} 4)
            echo "[ERR]${str}" >> ${RESULTFILE} 2>&1

        else
            echo "${str}||ScheduledTask:--" >> ${RESULTFILE} 2>&1
        fi

        rm -f ${mosObject_prefix}_${pool}
        rm -f ${listmemberinfo_f}
        rm -f ${bigblockdump_f}
    done

    rm -f /var/tmp/nofs_pool_listmemberinfo*
    [ -f ${zoneleft_f} ] && rm -f ${zoneleft_f}

    if [ "${InterruptPools}X" != "X" ];then
        isPass=$( check_pass ${isPass} 1 )
        echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}${InterruptPoolNum} disks meet the interruption criteria:$InterruptPools" >> ${RESULTFILE} 2>&1
    fi
    if [ "${FcPools}X" != "X" ];then
        isPass=$( check_pass ${isPass} 1 )
        echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}${FcPoolNum} disks meet the flow control criteria:$FcPools" >> ${RESULTFILE} 2>&1
    fi
    if [ "${AlarmPools}X" != "X" ];then
        isPass=$( check_pass ${isPass} 1 )
        echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}${AlarmPoolNum} disks meet the alarm criteria:$AlarmPools" >> ${RESULTFILE} 2>&1
    fi

    return
}

# 输入：单盘leftzone和随机区占用率TSRate
# 根据单盘leftzone和随机区占用率TSRate判断 哪些盘达到流控、告警、及业务中断的标准
function checkSMRLeftzoneAndTSRate()
{
    leftzone=$1
    TSRate=$2
    pool=$3
    Cnt=$( echo ${#LEFT_ZONE[*]} )

    if [[ "${leftzone}X" == "X" || "${TSRate}X" == "X" ]];then
        LOG "[$LINENO] leftzone or TSRate is empty, leftzone:${leftzone}, TSRate:${TSRate}"
        return
    fi

    local i=0
    for((i=0;i<${Cnt};i++))
    do
        if [[ ${leftzone} -ge ${LEFT_ZONE[$i]} ]];then
            break
        fi
    done
    if [[ ${i} -ge ${Cnt} ]]; then
        LOG "[$LINENO] leftzone abnormal,i:${i},Cnt:${Cnt}"
        return
    fi

    # 判断是否满足业务中断条件
    if [[ ${TSRate} -ge ${TSRate_INTERRUPT[$i]} ]];then
        InterruptPools="$InterruptPools $pool"
        let InterruptPoolNum++
    fi

    TSRate_FC=(${TSRate_FC_NORMAL[@]})
    checkDsFaultState
    # 故障场景使用故障流控阈值
    if [[ $? -ne 0 ]];then
        TSRate_FC=(${TSRate_FC_FAULT[@]})
    fi
    # 判断是否满足业务流控条件
    if [[ ${TSRate} -ge ${TSRate_FC[$i]} ]];then
        FcPools="$FcPools $pool"
        let FcPoolNum++
    fi

    # 判断是否满足告警上报条件
    if [[ ${TSRate} -ge ${TSRate_ALARM[$i]} ]];then
        AlarmPools="$AlarmPools $pool"
        let AlarmPoolNum++
    fi

    return
}

function checkDsFaultState()
{
    DS_MAP_PATH="/var/tmp/checkdsmap"
    cat /proc/monc_dsmap > ${DS_MAP_PATH}

    if [[ ! -f ${DS_MAP_PATH} ]];then
        LOG "[$LINENO]ERR:Get dsmap failed"
        return 0
    fi

    ret=$( cat ${DS_MAP_PATH} | grep -e DS -e NodePool )
    if [[ "${ret}X" == "X" ]];then
        LOG "[$LINENO]ERR:Get dsmap fault"
        return 0        
    fi

    ret=$( cat ${DS_MAP_PATH} | grep -a DS | grep -a "Abnormal(1)" )
    if [[ "${ret}X" != "X" ]];then
        LOG "[$LINENO]ERR: dsmap is fault,state(1)"
        return 1
    fi

    ret=$( cat ${DS_MAP_PATH} | grep -a "Abnormal(3)" )
    if [[ "${ret}X" != "X" ]];then
        LOG "[$LINENO]ERR: dsmap is fault,state(3)"
        return 1
    fi

    return 0
}

# SPC1版本通过巡检工具修改了MosReserve，升级到SPC600版本需要适配处理
function checkMosReserve()
{
    local nofsini="/opt/huawei/snas/etc/nofs.ini"
    local nofsConfig="/tmp/nofsConfig"
    /opt/huawei/snas/sbin/nofs-snas printnofsconfigure > ${nofsConfig}
    DataBottom=$( cat ${nofsConfig} | grep -aw "DataBottom" | awk '{print $3}' )
    MosReserve=$( cat ${nofsConfig} | grep -aw "MosReserve" | awk '{print $3}' )
    MosReserveBottom=$( cat ${nofsConfig} | grep -aw "MosReserveBottom" | awk '{print $3}' )

    # DataBottom ≥ MosReserve ≥ MosReserveBottom ，不需要修改当前值，退出
    if [ ${DataBottom} -ge ${MosReserve} -a ${MosReserve} -ge ${MosReserveBottom} ] || [ "$areaKey" == "false" ];then
        return
    fi

    grep -aw "^\[Defrag\]" ${nofsini} >/dev/null 2>&1
    if [ $? -ne 0 ];then
        sed -i '/^\[END\]$/i\\[Defrag\]' ${nofsini}
    fi

    if [ ${DataBottom} -lt ${MosReserveBottom} ];then
        DataBottom=$((MosReserveBottom+16))
        sed -i '/DataBottom=/d' ${nofsini}
        sed -i "/\[Defrag\]/a DataBottom=${DataBottom}" ${nofsini}
    fi

    MosReserve=$(((DataBottom+MosReserveBottom)/2))
    sed -i '/MosReserve=/d' ${nofsini}
    sed -i "/\[Defrag\]/a MosReserve=${MosReserve}" ${nofsini}

    # 更新配置
    /opt/huawei/snas/sbin/nofs-snas updateconfig
    if [ $? -ne 0 ];then
        sleep 1
        /opt/huawei/snas/sbin/nofs-snas updateconfig
        if [ $? -ne 0 ];then
            LOG "[$LINENO]failed to excute command:/opt/huawei/snas/sbin/nofs-snas updateconfig"
            echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}updateconfig failed." >> ${RESULTFILE} 2>&1
            isPass=$( check_pass ${isPass} 1 )
            return
        fi
    fi
    return
}

# 针对SPH603以及之后的版本需要进行如下处理
function checkScheduledTask()
{
    # 判断是否SMR节点，SMR节点直接退出
    if [ ${isSmrFlag} -eq 1 ]; then
        return
    fi
    # 判断是否配置过定时任务，已经配置则退出
    cat /etc/crontab |grep "modifychunk.sh"
    if [ $? -eq 0 ]; then
        if [ -f "/var/log/9000.ws/modifychunk.sh" ]; then
            echo "${str}||ScheduledTask:Configured" >> ${RESULTFILE} 2>&1
            return
        elif [ "${areaKey}" == "true" ];then
            sed -i "/modifychunk.sh/d" /etc/crontab
        fi
    fi

    poolTotalCount=$( /opt/huawei/snas/sbin/nofs-snas listpool | awk '{print $1}' | grep P | wc -l )
    if [ ${poolTotalCount} -eq 0 ]; then
        LOG "[$LINENO]Result is empty, cmd:/opt/huawei/snas/sbin/nofs-snas listpool"
        echo "${str}||ScheduledTask:Not Configured" >> ${RESULTFILE} 2>&1
        return
    fi

    doubleSatisfiedCount=$((poolSatisfiedCount*2))
    LOG "[$LINENO] poolTotalCount:${poolTotalCount}, poolSatisfiedCount:${poolSatisfiedCount}."

    olderVersion="false"
    sphVersion=$( ls /var/huawei/patch/cur/sph | grep -e "OceanStor*" -e "FusionStorage*" | awk -F_ '{print $NF}' | awk -F 'SPH' '{print $2}' | grep -o "[0-9]*" )
    grep -q V100R006C30SPC970 /opt/huawei/deploy/package/version 2>/dev/null
    if [ $? -eq 0 ]; then
        if [ "X${sphVersion}" == "X" ] || [ ${sphVersion} -lt 971 ]; then
            olderVersion="true"
        fi
    else
        if [ "X${sphVersion}" == "X" ] || [ ${sphVersion} -lt 603 ]; then
            olderVersion="true"
        fi
    fi

    str="poolName:--||${SECOND_COLUMN_NAME}--||dataCount(type1-3):--||dataLimit:--||metaCount(type4-6):--||metalimit:--||totalCount(type07):--||RandomZoneTotalAlloc:--||bigblockdump:--"

    # 判断当前是否可配置定时任务（满足条件的盘达到总数的一半）
    if [ ${doubleSatisfiedCount} -ge ${poolTotalCount} ]; then
        # 满足
        if [ "$olderVersion" == "true" ]; then
            # 低版本 建议优化 打最新补丁
            echo "${str}||ScheduledTask:Not Configured" >> ${RESULTFILE} 2>&1
            isPass=$( check_pass ${isPass} 4 )
            echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}Task can be configured. Please install the latest hot patch." >> ${RESULTFILE} 2>&1
            return

        elif [ "${areaKey}" == "true" ];then
            # 高版本 国内局点直接配置
            mkdir -p /var/log/9000.ws
            cp /opt/huawei/snas/script/inspect_mml/modifychunk.sh /var/log/9000.ws/modifychunk.sh
            echo "10 2 * * * root sh /var/log/9000.ws/modifychunk.sh >/dev/null 2>&1" >> /etc/crontab
            sleep 1
            service crond restart >/dev/null 2>&1

            echo "${str}||ScheduledTask:Configured" >> ${RESULTFILE} 2>&1
            return

        else
            echo "${str}||ScheduledTask:Not Configured" >> ${RESULTFILE} 2>&1
            isPass=$( check_pass ${isPass} 4 )
            echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}Task can be configured." >> ${RESULTFILE} 2>&1
        fi
    else
        # 不满足
        echo "${str}||ScheduledTask:Not Configured" >> ${RESULTFILE} 2>&1
        isPass=$( check_pass ${isPass} 4 )
        echo "[ERR]poolName:--||${SECOND_COLUMN_NAME}Task can not be configured." >> ${RESULTFILE} 2>&1
    fi
}

checkNofsMosObjectUsage

checkMosReserve

checkScheduledTask

echo "${CurInspectFun}_Pass ${isPass}" >> ${RESULTFILE} 2>&1
exit 0
