#!/bin/bash
set +x
G_MML_PATH="/opt/huawei/snas/script/inspect_mml"
LOG_FILE="/var/log/inspect.log"
source ${G_MML_PATH}/CommonFunc.sh
#报告输出的分隔符号
G_SEPARATOR="-----------------------------------------------------------------------------"
GetInspectType()
{
    case $1 in
        100)
            #1.Check Disk Usage
            echo "checkDiskUsage"
            ;;
        101)
            #2.Check CPU Usage
            echo "checkCPUUsage"
            ;;
        103)
            #4.Check Disk Status
            echo "checkDiskStatus_MML"
            ;;
        106)
            #7.Check Netport Status
            echo "ShowNIC"
            ;;
        109)
            #10.Check disk verify error
            echo "checkDiskVerify"
            ;;
        110)
            #节点(进出风口)温度
            echo "checkCardTemp"
            ;;
        111)
            #内存配置检查
            echo "checkMemCfg"
            ;;
        113)
            #check SSD
            echo "GetSSDInfo"
            ;;
        114)
            #check RAID
            echo "Checksysdisk"
            ;;
        117)
            #check ARP解析错误
            echo "checkArpErr_MML"
            ;;
        130)
            #检测PHY
            echo "CheckDiskPhy"
            ;;
        133)
            #检查物理内存
            echo "CheckPhyMem"
            ;;
        134)
            echo "checkNodeNCQ"
            ;;
        136)
            #检查设备节点是否有时钟源异常风险
            echo "checkClockSourceRisk"
            ;;
        139)
            #检查 rmbg任务并发数
            echo "checkRMGB"
            ;;
        140)
            #虚拟机信息检查
            echo "checkVmstatus"
            ;;
        143)
            #S3数据库分区有效空间检查
            echo "checkDBAvailableSpace"
            ;;
        144)
            #网卡PCIE速率协商
            echo "CheckPcieSpeed"
            ;;
        145)
            #检查SES固件版本号
            echo "GetSesFwVersion"
            ;;
        146)
            #磁盘写缓存检查
            echo "checkDiskWriteCache"
            ;;
        147)
            #SAS固件版本检查
            echo "checkSASVersion"
            ;;
        148)
            #SMR磁盘运行状态检查
            echo "CheckSMRdiskinfo"
            ;;
        149)
            #检查各节点网口MTU是否一致、误码与巨帧
            echo "checkNetportBitErrandJumboframe"
            ;;
        150)
            #踢盘开关检查
            echo "checkKicksSwitch"
            ;;
        151)
            #网卡固件版本检查
            echo "checkNetAdapterVersion"
            ;;
        157)
            #节点电子标签检查
            echo "CheckElecLabel"
            ;;
        171)
            echo "CheckHssdV6DiskLifeRisk"
            ;;
        202)
            #4.Check Node IO Perform
            echo "ShowNodeIO"
            ;;
        203)
            #4.Check Watchdog Status
            echo "checkWatchdogStatus_Async"
            ;;
        204)
            #4.MON connection Status with each subsystem
            echo "MonConStatuswithEachSubsys"
            ;;
        205)
            #6.Check NTP Service
            echo "checkNTPService"
            ;;
        206)
            #7.Check CM master
            echo "checkCMMaster"
            ;;
        209)
            #10.Check File operation常见的文件操作
            echo "ShowFileOperation"
            ;;
        210)
            #11.Check NAS client
            echo "checkALLClient"
            ;;
        211)
            #12.Check NAS Cluster Status
            echo "checkNASClusterStatus"
            ;;
        213)
            #14.Check Communication Link (节点间故障的通讯链路)
            echo "checkCommuLink"
            ;;
        214)
            #14.Check memory pool usage
            echo "checkMemPoolUsage_Async"
            ;;
        215)
            #Check data consistent数据一致性
            echo "checkDataConsistent_Async"
            ;;
        218)#linux系统打开的文件句柄数
            #Check system file opened
            echo "checkSystemFileOpened"
            ;;
        221)
            #Check LDAP 域控服务器的状态
            echo "ShowLdapServer"
            ;;
        222)
            #Check IPMI地址是否配置
            echo "checkIPMI"
            ;;
        225)
            #业务主节点状态  mon mds pip
            echo "checkMasterStatus"
            ;;
        226)
            #MON Server节点
            echo "checkMonServer"
            ;;
        227)
            #Nvdimm内存池的使用情况
            echo "checkNvdimmPool"
            ;;
        231)
            #检查MON是否在频繁选举
            echo "checkMonHealth"
            ;;
        233)
            #是否有core文件
            echo "checkCoreFile"
            ;;
        234)
            #ntp链接状态
            echo "checkNtpConn"
            ;;
        236)
            #检测写NOFS是否有超时
            echo "CheckWrtNofsTimeout"
            ;;
        242)
            #检查一致性扫描导致的数据重构失败
            echo "CheckRecovFailForDSC"
            ;;
        244)
            #检查删除任务
            echo "checkDelTask"
            ;;
        245)
            #1.Check Replication Link
            echo "checkReplicationLink"
            ;;
        246)
            #1.Check Replication Process
            echo "checkReplicationProcess"
            ;;
        247)
            #1.Check ReplicationZone Status
            echo "checkReplicationZoneStatus"
            ;;
        248)
            #1.Check Replication Pair Status
            echo "checkReplicationPairStatus"
            ;;
        249)
            #1.Check ReplicationZone Version
            echo "checkReplicationVersion"
            ;;
        253)
            #check default log print level
            echo "CheckLogPrintLvl"
            ;;
        254)
            #对象存储状态
            echo "checkObjsServiceStat"
            ;;
        260)
            #check ad connection
            echo "nas_checkADdomainControllerStatus"
            ;;
        261)
            #检查延时导入磁盘
            echo "checkDelayReimportDisk"
            ;;
        264)
            #check s3 s3fs service
            echo "checkS3FSService"
            ;;
        265)
            #check ufp database backup service
            echo "checkUPFDBBackup"
            ;;
        266)
            #check s3 key service
            echo "checkS3KeySerStatus"
            ;;
        268)
            #check s3 dns service
            echo "checkS3DNSservice"
            ;;
        269)
            #check bill service
            echo "checkBillService"
            ;;
        270)
            echo "CheckNodeState"
            ;;
        274)
            echo "CheckCertificate"
            ;;
        276)
            echo "accountCheck"
            ;;
        278)
            echo "check_gateway"
            ;;
        280)
            echo "checkdomainname"
            ;;
        282)
            #检查热补丁状态
            echo "checkHPatchstatus"
            ;;
        285)
            #获取vfs层性能统计
            echo "checkVfsPerformance"
            ;;
        286)
            #关键文件权限检查
            echo "checkKeyFileAuthority"
            ;;
        288)
            #集群ID一致性校验
            echo "checkClusterId"
            ;;
        289)
            #DLM集群状态检查
            echo "checkDLMstatus"
            ;;
        290)
            #check ds report preslow info
            echo "CheckDsReportPreSlow"
            ;;
        291)
            #check nofs report preslow info
            echo "CheckNofsReportPreSlow"
            ;;
        292)
            #omuser密码同步检查
            echo "checkOSUserInfoSync"
            ;;
        294)
            echo "checkDiskNokickStatus"
            ;;

        302)
            #操作系统环境变量检查
            echo "CheckEnvironVarPath"
            ;;
        305)
            #集群SMR配置一致性检查检查
            echo "checkNodeSMRConfig"
            ;;
        306)
            #viid server状态检查
            echo "checkViidServerStatus"
            ;;
        307)
            #viid 摄像头状态检查
            echo "checkViidIpcStatus"
            ;;
        309)
            #viid wal开关状态一致性检查
            echo "checkNodeWALSwitch"
            ;;
        310)
            #检查是否存在NFS连接静态ip的客户端
            echo "nascheck_static_mount"
            ;;
        311)
            #检查密钥备份服务器连通性
            echo "CheckAKBackServer"
            ;;
        312)
            echo "CheckBirthTime"
            ;;
        313)
            echo "CheckDepXML"
            ;;
        314)
            #内存碎片化程度检查
            echo "CheckMemoryFragmentation"
            ;;
        316)
            #协议IO状态检查
            echo "CheckNasIOStatus"
            ;;
        318)
            #主节点特殊配置检查
            echo "CheckMasterSpecialConfig"
            ;;
        319)
            #增值特性开关检查
            echo "CheckValueaddedfeatures"
            ;;
        320)
            #特殊配置检查
            echo "CheckALLNodeSpecialConf"
            ;;
        321)
            echo "DeepCheckHardDriveFirmware"
            ;;
        322)
            echo "CheckZoneMount"
            ;;
        323)
            echo "CheckBMCDataAcquisitionSwitch"
            ;;
        326)
            echo "CheckRouteConfigured"
            ;;
        327)
            echo "CheckCCDBDBConsistency"
            ;;
        328)
            echo "checkDiskModePage"
            ;;
        331)
            echo "CheckStripeCfg"
            ;;
        334)
            echo "CheckStaticCfg"
            ;;
        335)
            echo "CheckUpdFile"
            ;;
        338)
            echo "CheckNofsWriteMagnify"
            ;;
        339)
            echo "CheckConcomitantLogs"
            ;;
        340)
            echo "CheckV5NvdimmFirmware"
            ;;
        341)
            echo "CheckIfUpgradeInterfaceCardAndChipCard"
            ;;
        342)
            echo "CheckDiskNofsSpaceContinuity"
            ;;
        346)
            echo "CheckALLNodeSpecialConfPreUpgrade"
            ;;
        347)
            echo "CheckALLNodeSpecialConfExpand"
            ;;
        348)
            echo "CheckALLNodeSpecialConfPatch"
            ;;
        350)
            echo "CheckStorageVersion"
            ;;
        351)
            echo "CheckHardWareVersion"
            ;;
        352)
            echo "CheckFrontAndBackPort"
            ;;
        353)
            echo "CheckFrontIPNumber"
            ;;
        354)
            echo "CheckBackNetworkIPaddr"
            ;;
        355)
            echo "CheckClusterRedundancyRatio"
            ;;
        356)
            echo "CheckClusterNodeNumber"
            ;;
        357)
            echo "CheckNodePoolDiskConsistency"
            ;;
        358)
            echo "CheckManagerNet"
            ;;
        359)
            echo "CheckCMiniFile"
            ;;
        360)
            echo "CheckCtnrGcConfigured"
            ;;
        361)
            echo "CheckLDBLoadMessage"
            ;;
        407)
            echo "CheckConfigurationCompatibility"
            ;;
        501)
            echo "certUpdate"
            ;;
        502)
            echo "ConfigurationCompatibilityRepair"
            ;;
        504)
            echo "HPatchStatusRepair"
            ;;
        505)
            echo "EnvironVarPathRepair"
            ;;
        *)
            echo ""
            ;;
    esac
}

function set_ifs
{
    IFS_OLD=$IFS
    IFS=$'\n'
}

function restore_ifs
{
    IFS=$IFS_OLD
}

function check_pass()
{
    local list=(["2"]=0 ["1"]=1 ["4"]=2 ["0"]=3 ["3"]=4) #未检查(2)>不通过(1)>建议优化(4)>通过(0)>不涉及(3)
    local isPass=$1   #集群结果
    local tmpIsPass=$2 #当前检查结果

    if [ ${list[$tmpIsPass]} -lt ${list[$isPass]} ];then
        isPass=$tmpIsPass
    fi
    echo $isPass
}

function get_node_backnet_ip
{
    local nid=$1
    local temp=1
    local iRet=0
    sqlite3 /opt/huawei/snas/etc/cm_conf.db "select IP_ADDR from CM_NODE_IP_T where NID=$nid and SUBNET_ID=1 order by NID limit 1"
    iRet=$?
    while [[ ${iRet} -ne 0 ]]
    do
        sleep 2
        LOG "[$FUNCNAME]Failed to get back subnet IP of $nid from db"
        temp=$((temp + 1))
        if [ $temp -gt 5 ];then
            break
        fi
        sqlite3 /opt/huawei/snas/etc/cm_conf.db "select IP_ADDR from CM_NODE_IP_T where NID=$nid and SUBNET_ID=1 order by NID limit 1"
        iRet=$?
    done
    return $iRet
}

function LOG
{
    echo [`date`][$0][$$]$@ >> $LOG_FILE
}

function LOG_Check
{
    # delete log file bigger than 20M
    FILE_SIZE=`ls -l $LOG_FILE | cut -d' ' -f 5`
    if [ ! -z "$FILE_SIZE" ]; then
        # delete file when file bigger than 10M
        if [ $FILE_SIZE -gt 20971520 ]; then
            [ -f /var/log/inspect_1.log ] && mv /var/log/inspect_1.log /var/log/inspect_2.log >/dev/null 2>&1
            [ -f /var/log/inspect.log   ] && mv /var/log/inspect.log /var/log/inspect_1.log >/dev/null 2>&1
            touch /var/log/inspect.log >/dev/null 2>&1
        fi
    fi
}

function ChangeProperty()
{
    local change_type=$1
    local property=""
    LOG "TYPE:${change_type},FILE:${Paras}"


    for changepath in ${Paras}
    do
        if [ -d ${changepath} ];then
            path_type=0
            property=$( lsattr -a ${changepath} | grep "/.$" |awk '{print $1}' )
        else
            path_type=1
            property=$( lsattr -a ${changepath} | awk '{print $1}' )
        fi
        if [ "X${property}" == "X" ];then
            continue
        fi
        if [ "${change_type}" == "add" ];then
            if [[ ${property} =~ "i" ]];then
                continue
            fi
            if [ "${path_type}" == "0" ];then
                chattr +i ${changepath} >/dev/null 2>&1
            else
                chattr -f +i ${changepath} >/dev/null 2>&1
            fi
        elif [ "${change_type}" == "del" ];then
            if [[ ${property} =~ "i" ]];then
                if [ "${path_type}" == "0" ];then
                    chattr -i ${changepath} >/dev/null 2>&1
                else
                    chattr -f -i ${changepath} >/dev/null 2>&1
                fi
            fi
        else
            continue
        fi
    done
    return
}

Add_i_attr="add"
Del_i_attr="del"
G_MML_PATH="/opt/huawei/snas/script/inspect_mml"
Paras=${@:2}
Params=$2
taskids=$1

#更改入口脚本的i属性
if [ "${taskids}" == "${Add_i_attr}" -o "${taskids}" == "${Del_i_attr}" ];then
    ChangeProperty "${taskids}"
    exit 0
fi

function ExecFun()
{
    local id="$1"
    InspectFun=`GetInspectType $id`
    LOG "[${FUNCNAME[@]}][$LINENO]Enter inspect item ${taskid}"
    if [ "${InspectFun}" != "" ]; then
        #获取此脚本的运行参数
        CurParam=""
        ss=${InspectFun}-
        for Param in $Params
        do
        {
            tmpStr=$(echo $Param |grep $InspectFun)
            if [ "$tmpStr" != "" ];then
                CurParam=$(echo $Param |awk -F "$ss" '{printf $2}'|sed 's/-/ /g')
                break;
            fi
        }
        done

        InspectBeginTime=$(date +%s)
        $G_MML_PATH/$InspectFun "$CurParam"
        InspectEndTime=$(date +%s)
        difftime=$(($InspectEndTime - $InspectBeginTime))

        #判断巡检项是否通过且时间允许，若不通过不重新检查，内部错误才重新检查
        PassCode=$(cat /tmp/tmpResult${InspectFun} |grep ${InspectFun}_Pass |awk '{printf $2}')
        if [ "$PassCode" = "2" -a "X$PassCode" != "X" ] && [ $difftime -lt 250 ];then # 巡检主节点重试300s超时
            LOG "[${FUNCNAME[@]}][$LINENO]Retry inspect item ${taskid}"
            $G_MML_PATH/$InspectFun "$CurParam"
            InspectEndTime=$(date +%s)
            difftime=$(($InspectEndTime - $InspectBeginTime))
        fi
    fi
    LOG "[${FUNCNAME[@]}][$LINENO]Exit inspect item ${taskid}, elapsed time ${difftime}"
} #通过localexec后无需多线程并发，并发和非并发时间一致，且并发消耗更多CPU

function main()
{
    local flag=0
    isMaster
    flag=$?
for taskid in $taskids
do
        if [ ${flag} -eq 0 ];then
            #主节点不需要并行：1.可以减轻主节点的CPU压力。 2.主节点需要等待其他节点的结果，时间不会有影响
            ExecFun "${taskid}"
        else
{
               ExecFun "${taskid}"
            }&
        fi
    done
    wait
    for taskid in ${taskids}
    do
        InspectFun=`GetInspectType ${taskid}`

    if [ "${InspectFun}" != "" ]; then
        echo  ${InspectFun}
            cat "/tmp/tmpResult${InspectFun}" 2>/dev/null
        mkdir -p /var/log/inspect_his/
            mv -f "/tmp/tmpResult${InspectFun}" /var/log/inspect_his/ 2>/dev/null
        echo  ${InspectFun}_End
    fi
    done
}

main
##非主节点日志处理
LOG_Check