# coding=utf-8
# Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved.
import datetime
import os
import shutil
import pathlib
import random
import string
import threading
import time
import re
import subprocess
import shlex
import secrets
import concurrent.futures

from IPy import IP

from utils.common import log as logger
from utils.common.fic_base import ToolClient
from utils.client.webUI_client import g_webui_client_ctrl
from utils.business.param_util import ParamUtil
from utils.common.exception import HCCIException
from utils.DBAdapter.DBConnector import BaseOps
from utils.common.ssh_util import Ssh as ssh
from utils.Driver.CloudDC.OpenStack.baremetal.resource_pool import ResourcePool
import plugins.DistributedStorage.utils.common.crypt_api as cryptApi
from plugins.DistributedStorage.utils.client.cps_web_client import CPSWebRest
from plugins.DistributedStorage.logic.vm_operate import VMOperate
from plugins.DistributedStorage.utils.common.deploy_constant import DeployConstant
from plugins.DistributedStorage.utils.interface.DistributedStorage import DistributedStorageTool


class InstallOperate(object):
    def __init__(self, project_id=None, pod_id=None, fs_args=None):
        self.client = None
        self.body = None
        self.is_pass = None
        self.default_client = ToolClient.DSWARE_CLIENT
        self.project_id = project_id
        self.pod_id = pod_id
        self.fs_args = fs_args
        self.db = BaseOps()
        self.param_util = ParamUtil()

    @classmethod
    def general_ha_and_account_config(cls, args, ha_items_cmd):
        ssh_client = ssh.ssh_create_client(args.fsm_ip, DeployConstant.ROOT_USER, args.fsm_root_pwd)
        try:
            cmd_res = ssh.ssh_send_command(ssh_client, ha_items_cmd, "#", 60, 3)
            logger.info("Make the HAInfoFromInstallTool.properties on host:{}, result:{}".format(args.fsm_ip, cmd_res))

            cmd_res = ssh.ssh_exec_command_return(ssh_client, args.account_set_cmd, 60, 3)
            cmd_res = str(cmd_res)
            if cmd_res.find('last_result=0') == -1:
                err_msg = "Add account settings failed"
                logger.error(err_msg)
                raise Exception(err_msg)
            logger.info("Add account settings success")
        finally:
            ssh.ssh_close(ssh_client)

    @classmethod
    def del_mark_file(cls, host_ip, user_name, passwd, root_pwd):
        ssh_client = cls.create_ssh_root_client(host_ip, user_name, passwd, root_pwd)
        mark_file = "/opt/dsware/manager/openstack/iscpsconfig"
        cmd = 'if [ -f  {0} ];then rm -f {0};fi;echo "last_result=$?"'.format(mark_file)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd, 20)
        logger.info("cmd:{}, result:{}".format(cmd, cmd_res))
        if not str(cmd_res).__contains__("last_result=0"):
            err_msg = "Failed to delete the file {}, host_ip:{}".format(mark_file, host_ip)
            logger.error(err_msg)
            raise Exception(err_msg)

    @staticmethod
    def create_ssh_root_client(host_ip, user, passwd, root_pwd):
        def su_root():
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("exe cmd[su root] result:%s" % (str(cmd_res)))

            cmd_res = ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            logger.info("exe cmd[#] result:%s" % (str(cmd_res)))

            cmd_res = ssh.ssh_send_command(ssh_client, 'TMOUT=0', '#', 20)
            logger.info("exe cmd[TMOUT=0] result:%s" % (str(cmd_res)))
        try:
            ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        except Exception as e:
            create_err_msg = "Failed to connect host using user %s, omip:%s, err:%s" % (user, host_ip, str(e))
            logger.error(create_err_msg)
            raise Exception(create_err_msg) from e
        try:
            su_root()
        except Exception as e:
            switch_err_msg = "Failed to switch to root user, omip:%s, err:%s" % (host_ip, str(e))
            logger.error(switch_err_msg)
            if ssh_client:
                ssh.ssh_close(ssh_client)
            raise Exception(switch_err_msg) from e
        return ssh_client

    @staticmethod
    def get_certificate_path(cert_pattern, dir_path=None, all_match=False):
        if not os.path.exists(dir_path):
            logger.error("Directory path [%s] does not exist. " % dir_path)
            raise HCCIException(626367, dir_path)
        items = os.listdir(dir_path)
        cert_name = None
        for names in items:
            if len(re.findall(cert_pattern, names)) > 0:
                cert_name = names
                if all_match:
                    continue
                break
        if not cert_name:
            logger.error("The certificate of %s is not exist." % cert_pattern)
            raise HCCIException(626368, cert_pattern, dir_path)
        cert_path = '{}/{}'.format(dir_path, cert_name)
        logger.info("Certificate pattern:%s, Find certificate:%s" % (cert_pattern, cert_path))
        return cert_path

    @staticmethod
    def host_ping(host_ip):
        try:
            IP(host_ip)
        except Exception:
            return False
        if IP(host_ip).version() == 6:
            ping_act = "ping6"
        else:
            ping_act = "ping"
        cmd = "%s %s -c 2" % (ping_act, host_ip)
        cmd = shlex.split(cmd)
        p = subprocess.Popen(args=cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        (std_output, err_output) = p.communicate(timeout=None)
        std_output = std_output.decode('utf-8') if isinstance(std_output, bytes) else std_output
        if std_output.find('ttl=') >= 0:
            return True
        return False

    @staticmethod
    def reboot_host_list(host_data):
        host_ip_list = [host.get('om_ip') for host in host_data]
        logger.info('Rebooting hosts:%s' % host_ip_list)
        cmd_reboot = 'reboot'
        for host in host_data:
            user = host.get('user')
            passwd = host.get('passwd')
            root_pwd = host.get('root_pwd')
            host_ip = host.get('om_ip')
            ssh_client = InstallOperate.create_ssh_root_client(host_ip, user, passwd, root_pwd)
            cmd_ret = ssh.ssh_exec_command_return(ssh_client, cmd_reboot)
            logger.info("Rebooting host[%s], result: %s" % (host_ip, str(cmd_ret)))
            if ssh_client:
                ssh.ssh_close(ssh_client)
        time.sleep(120)
        timeout = 900
        while timeout > 0:
            for host in host_ip_list:
                if InstallOperate.host_ping(host):
                    logger.info('The host[%s] is avaible' % host)
                    host_ip_list.remove(host)
            if len(host_ip_list) == 0:
                break
            logger.info('The remaining hosts%s are still unreachable. Try to ping it after 30 second.' % host_ip_list)
            time.sleep(30)
            timeout -= 30
        if len(host_ip_list) > 0:
            err_msg = "The host%s is still not restarted. Please check it by manually." % host_ip_list
            logger.error(err_msg)
            raise Exception(err_msg)

    @staticmethod
    def re_find_ans(res_ssh, mark):
        res_ssh = ''.join(res_ssh)
        pattern = '%s=(\d+\.\d+\.\d+\.\d+)' % mark
        res = re.findall(pattern, res_ssh)
        return res

    @staticmethod
    def generate_random_character():
        # 字母类型
        english_char = ['q', 'w', 'e', 'r', 't', 'y', 'u', 'i', 'o', 'p', 'l',
                        'k', 'j', 'h', 'g', 'f', 'd', 's', 'a', 'z', 'x',
                        'c', 'v', 'b', 'n', 'm', 'Q', 'W', 'E', 'R', 'T', 'Y',
                        'U', 'I', 'O', 'P', 'A', 'S', 'D', 'F', 'G', 'H', 'J',
                        'K', 'L', 'Z', 'X', 'C', 'V', 'B', 'N', 'M']
        # 数字类型
        number_char = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0']
        # 符号类型
        symbol_char = ['@', '_', '-']
        all_char = english_char.copy() + number_char.copy() + symbol_char.copy()
        # 重新打乱数组
        random.shuffle(all_char)
        password_length = 16
        while True:
            password = ''
            # 循环生成密码
            secrets_generator = secrets.SystemRandom()
            for _ in range(password_length):
                index = len(all_char) - 1
                password = password + all_char[secrets_generator.randint(0, index)]
            if not re.search(r"[@_-]", password):
                continue
            if not re.search(r"[0-9]", password):
                continue
            if not re.search(r"[a-z]", password):
                continue
            if re.search(r"[A-Z]", password):
                break
        return password

    @staticmethod
    def multi_thread_exec_cmd(host_list, handler, cmd, timeout=300):
        failed_info = {}
        with concurrent.futures.ThreadPoolExecutor() as executor:
            future_list = [executor.submit(handler, cmd, host_info, timeout=timeout) for host_info in host_list]
            for cur_fu in concurrent.futures.as_completed(future_list):
                res_code, res_node, res_details = cur_fu.result()
                if not res_code:
                    failed_info[res_node] = res_details
                    logger.error("[{}]Execute result: {}".format(res_node, res_details))
                else:
                    logger.info("[{}]Execute result: {}".format(res_node, res_details))
        return failed_info

    @staticmethod
    def check_vbs_node_fusionstorage_agent_rpm(vbs_list):
        logger.info('Run the command [rpm -qa|grep FusionStorage-Agent] to '
                    'check whether the RPM package [FusionStorage-Agent-1.0.0-1.noarch] is installed on the VBS nodes')
        cmd = "rpm -qa | grep FusionStorage[_-]Agent;echo result=$?"
        res = InstallOperate.multi_thread_exec_cmd(vbs_list, InstallOperate.check_fusionstorage_agent_handler, cmd)
        if res:
            err_msg = "The RPM package[FusionStorage-Agent-1.0.0-1.noarch] is not installed on some nodes"
            logger.error(err_msg + ",details:{}".format(res))
            raise Exception(err_msg + ":{}".format(list(res.keys())))
        logger.info('The RPM[FusionStorage-Agent-1.0.0-1.noarch] is found on all nodes')

    @staticmethod
    def check_fusionstorage_agent_handler(cmd, host_info: dict, timeout=300):
        def execute_check_command():
            start_time = datetime.datetime.now(tz=datetime.timezone.utc)
            ret = []
            while (datetime.datetime.now(tz=datetime.timezone.utc) - start_time).seconds < timeout:
                try:
                    ret = ssh.ssh_exec_command_return(ssh_client, cmd, retry_times=0)
                except Exception as err1:
                    return False, str(err1)
                if str(ret).__contains__("result=0"):
                    return True, str(ret)
                time.sleep(10)
            if ssh_client:
                ssh.ssh_close(ssh_client)
            return False, str(ret)

        host_ip = host_info.get("om_ip")
        try:
            ssh_client = InstallOperate.create_ssh_root_client(
                host_ip, host_info.get('user'), host_info.get('passwd'), host_info.get('root_pwd'))
        except Exception as err:
            return False, host_ip, str(err)

        res_code, res_details = execute_check_command()
        if not res_code:
            return False, host_ip, res_details
        return True, host_ip, res_details

    @staticmethod
    def check_fusionstorage_agent_rpm(host_list):
        timeout = 300 + 30 * len(host_list)
        timeout = min(timeout, 600)
        sleep_time = 30
        wait_time = 0
        check_hosts = host_list
        while wait_time <= timeout:
            remain_hosts = check_hosts
            for host in check_hosts:
                host_ip = host.get('om_ip')
                logger.info('Checking if the nodes[%s] install the FusionStorage-Agent RPM package' % host_ip)
                if InstallOperate.check_fusionstorage_agent(host_ip, host.get('user'), host.get('passwd'),
                                                            host.get('root_pwd')):
                    logger.info("Find role rpm package on node[%s]" % host_ip)
                    remain_hosts.remove(host)
            check_hosts = remain_hosts
            if len(check_hosts) == 0:
                break
            check_host_ips = [node.get('om_ip') for node in check_hosts]
            try:
                logger.info('The rpm is not installed on hosts[%s], check %sth times after %ss'
                            % (check_host_ips, wait_time / sleep_time, sleep_time))
            except ZeroDivisionError as e:
                err_msg = 'ZeroDivisionError, Detail:%s' % str(e)
                logger.error(err_msg)
                raise e
            time.sleep(sleep_time)
            wait_time += sleep_time
        if len(check_hosts) > 0:
            check_host_ips = [node.get('om_ip') for node in check_hosts]
            err_msg = 'The RPM[FusionStorage-Agent-1.0.0-1.noarch] is not found on ' \
                      'some nodes%s after waitting for %s seconds times out' \
                      % (check_host_ips, timeout)
            raise Exception(err_msg)
        else:
            logger.info('The RPM[FusionStorage-Agent-1.0.0-1.noarch] is found on all nodes')
            return

    @staticmethod
    def check_fusionstorage_agent(host_ip, user, passwd, root_pwd):
        cmd_chk = "rpm -qa | grep FusionStorage[_-]Agent;echo result=$?"
        ssh_client = InstallOperate.create_ssh_root_client(host_ip, user, passwd, root_pwd)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_chk, 1200, 0)
        logger.info("Run check cmd[%s], result:%s" % (cmd_chk, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        ret_str = ','.join(cmd_res).replace('\n', '')
        test_ret = re.findall("result=\d+", ret_str)
        test_ret = test_ret[0].split('=')[1]
        test_ret = int(test_ret)
        if test_ret != 0:
            err_msg = "Failed to install role to node[%s], Detail:%s" % (host_ip, str(ret_str))
            logger.error(err_msg)
            return False
        logger.info("The FusionStorage-Agent rpm package is installed on node[%s]" % host_ip)
        return True

    @staticmethod
    def get_role_on_host(cps_client, host_id, target_role='fusionstorage-block'):
        resp_data = cps_client.do_host_role_list(host_id)
        role_data_list = resp_data.get('roles')
        role_list = list()
        for role_data in role_data_list:
            role = role_data.get('name')
            if role.find(target_role) == 0:
                role_list.append(role)
        len_role = len(role_list)
        if len_role > 1:
            err_msg = "A node cannot have multiple fusionstorage-blockXXX roles."
            logger.info(err_msg)
            raise Exception(err_msg)
        elif len_role == 1:
            role_name = role_list[-1]
        else:
            role_name = None
        return role_name

    @staticmethod
    def remove_fusionstorage_block_role(cps_client, hosts_id_list):
        logger.info("Targeting hosts ids%s" % str(hosts_id_list))
        for host_id in hosts_id_list:
            role = InstallOperate.get_role_on_host(cps_client, host_id)
            if not role:
                continue

            resp = cps_client.do_role_host_list(role)
            host_list = resp.get('hosts')
            host_id_list = list(set(host_list).intersection(set(hosts_id_list)))
            logger.info("Removing role[%s] on hosts%s" % (role, host_id_list))
            cps_client.do_role_host_delete(role, host_id_list)
            cps_client.do_commit()
            time.sleep(30)

            resp = cps_client.do_role_host_list(role)
            last_host_list = resp.get('hosts')
            if len(last_host_list) == 0:
                logger.info("Deleting role[%s]" % role)
                cps_client.do_role_delete(role)
                cps_client.do_commit()
        logger.info("Finish to remove role on nodes%s" % hosts_id_list)

    @staticmethod
    def check_fusionstorage_block_on_host(cps_client, host_id_list):
        exist_role_list = list()
        for host_id in host_id_list:
            role = InstallOperate.get_role_on_host(cps_client, host_id)
            if role:
                logger.info("There is exist fusionstorage-blockXXX%s role on host[%s]" % (role, host_id))
                exist_role_list.append({host_id: role})
        if len(exist_role_list) > 0:
            host_list = [host.keys() for host in exist_role_list]
            role_list = [role.values() for role in exist_role_list]
            role_list = list(set(role_list))
            err_msg = "There is exist role fusionstorage-blockXXX%s on host%s, " \
                      "Please remove cmds[step 1.cps role-host-delete --host " \
                      "[host_id1,host_id2...] fusionstorage-blockXXX; " \
                      "step2.cps commit] on one cps node. Host and role detail:%s" \
                      % (role_list, host_list, str(exist_role_list))
            raise Exception(err_msg)
        else:
            logger.info("There is no fusionstorage-blockXXX role on host%s" % host_id_list)

    @staticmethod
    def check_fusionstorage_block_before_install_role(cps_client, host_id_list):
        exist_role_list = list()
        for host_id in host_id_list:
            role = InstallOperate.get_role_on_host(cps_client, host_id)
            if role:
                logger.info("There is exist fusionstorage-blockXXX%s role on host[%s]" % (role, host_id))
                exist_role_list.append({host_id: role})
        if len(exist_role_list) == len(host_id_list):
            logger.info("All hosts has install fusionstorage-blockXXX role, no need install again.")
            return True
        else:
            logger.info("Not all hosts installed fusionstorage-blockXXX role, should install continue." % host_id_list)
            return False

    @staticmethod
    def restart_fusionstorage_block_template(cps_client, host_id_list):
        operated_host_list = list()
        logger.info("Stop and start fusionstorage-blockXXX tempalate on host%s" % host_id_list)
        for host_id in host_id_list:
            if host_id in operated_host_list:
                continue

            role = InstallOperate.get_role_on_host(cps_client, host_id)
            if not role:
                logger.info("There is no fusionstorage-blockXXX role on host[%s]" % host_id)
                continue

            role_data = cps_client.do_role_show(role)
            template_list = role_data.get('template')
            template = template_list[0]
            service_name = template.split('.')[0]
            template_name = template.split('.')[1]

            resp = cps_client.do_role_host_list(role)
            host_list = resp.get('hosts')
            host_list = list(set(host_list).intersection(set(host_id_list)))

            logger.info("Stop role[%s] on hosts%s" % (role, host_list))
            rsp_data = cps_client.do_host_template_instance_operate(service_name, template_name, 'stop', host_list)
            InstallOperate.check_operate_status(rsp_data, host_list, template, 'stop')

            time.sleep(5)
            logger.info("Start role[%s] on hosts%s" % (role, host_list))
            rsp_data = cps_client.do_host_template_instance_operate(service_name, template_name, 'start', host_list)
            InstallOperate.check_operate_status(rsp_data, host_list, template, 'start')

            operated_host_list.extend(host_list)
            logger.info("Stopped and started hosts%s" % operated_host_list)
            logger.info("Remaining hosts%s" % list(set(host_id_list).difference(set(operated_host_list))))
        logger.info("Finish to stop and start hosts%s" % operated_host_list)

    @staticmethod
    def stop_fusionstorage_block_template(cps_client, host_id_list):
        operated_host_list = list()
        logger.info("Stop and start fusionstorage-blockXXX tempalate on host%s" % host_id_list)
        for host_id in host_id_list:
            if host_id in operated_host_list:
                continue

            role = InstallOperate.get_role_on_host(cps_client, host_id)
            if not role:
                logger.info("There is no fusionstorage-blockXXX role on host[%s]" % host_id)
                continue

            role_data = cps_client.do_role_show(role)
            template_list = role_data.get('template')
            template = template_list[0]
            service_name = template.split('.')[0]
            template_name = template.split('.')[1]

            resp = cps_client.do_role_host_list(role)
            host_list = resp.get('hosts')
            host_list = list(set(host_list).intersection(set(host_id_list)))

            logger.info("Stop role[%s] on hosts%s" % (role, host_list))
            rsp_data = cps_client.do_host_template_instance_operate(service_name, template_name, 'stop', host_list)
            InstallOperate.check_operate_status(rsp_data, host_list, template, 'stop')
            time.sleep(5)

            operated_host_list.extend(host_list)
            logger.info("Stopped hosts%s" % operated_host_list)
            logger.info("Remaining hosts%s" % list(set(host_id_list).difference(set(operated_host_list))))
        logger.info("Finish to stop hosts%s" % operated_host_list)

    @staticmethod
    def check_operate_status(rsp_value, host_id_list, template, operate):
        rsp_hosts = rsp_value.get('hosts')
        fail_host_list = list()
        for host_id in host_id_list:
            if rsp_hosts.get(host_id).get(template).get('code') != 200:
                fail_host_list.append(host_id)
        if len(fail_host_list) > 0:
            service_name = template.split('.')[0]
            template_name = template.split('.')[1]
            err_msg = "Failed to %s FusionStorage-Agent on host%s. Please try it on one of openstack node by cmd" \
                      "[cps host-template-instance-operate --service %s %s --action %s --host %s]," \
                      " Detail:%s" % (operate, fail_host_list, service_name,
                                      template_name, operate, fail_host_list, rsp_value)
            logger.error(err_msg)
            raise Exception(err_msg)
        else:
            logger.info("Succeed start to %s FusionStorage-Agent. Detail:%s" % (operate, rsp_value))

    @staticmethod
    def config_license_switch(host_ip, user_name, passwd, root_pwd, delete=False):
        # 配置创建第一个存储池的参数：支持添加和删除，delete=True代表删除功能，默认False代表添加配置
        ssh_client = None
        try:
            ssh_client = ssh.ssh_create_client(host_ip, user_name, passwd)
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("Exec cmd[su root] result:%s" % (str(cmd_res)))
            ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            # 无论添加还是删除，先删除一次license.switch配置
            cmd_delete_config = 'su - omm -s /bin/bash -c \'sed -i "/^license.first.pool.switch/d" /opt/omm/oms/' \
                                'workspace/webapps/dsware/WEB-INF/manager.properties; echo last_code=$?\''
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_delete_config, 20)
            logger.info("Exec cmd delete switch license result: %s" % cmd_res)
            # 如果是删除逻辑，此处就返回
            if delete:
                return
            cmd_config = 'su - omm -s /bin/bash -c \'echo "license.first.pool.switch=close" >> /opt/omm/oms/' \
                         'workspace/webapps/dsware/WEB-INF/manager.properties; echo last_code=$?\''
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_config, 20)
            logger.info("Exec cmd switch license result: %s" % cmd_res)
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)

    @staticmethod
    def check_user_accessible(host_ip, user, password, timeout=900):
        def _check_fsadmin_accessible():
            ssh_client = ssh.ssh_create_client(host_ip, user, password)
            cmd_ret = ssh.ssh_send_command(ssh_client, check_id_cmd, 'code=', 10)
            cmd_ret = str(cmd_ret)
            logger.info("Run cmd[%s], result: %s" % (check_id_cmd, cmd_ret))
            ssh.ssh_close(ssh_client)
            return cmd_ret
        accessible_flag = False
        exp_err = None
        runtime = 0
        check_id_cmd = 'id fsadmin; echo code=$?'
        while runtime < timeout:
            runtime += 60
            time.sleep(60)
            try:
                cmd_ret = _check_fsadmin_accessible()
            except Exception as e:
                exp_err = str(e)
                err_msg = "Failed to connect host[%s] by user[%s], Detail:%s, Runtime:%s, Try again after %ss" \
                          % (host_ip, user, exp_err, runtime, 60)
                logger.error(err_msg)
                continue
            if 0 <= cmd_ret.find('code=0'):
                logger.info("Succeed to login host[%s]" % host_ip)
                accessible_flag = True
                break
        if not accessible_flag:
            err_msg = "Failed to connect host[%s] by user[%s] after %ss seconds of time-out wait, Detail:%s" \
                      % (host_ip, user, timeout, exp_err)
            raise Exception(err_msg)
        logger.info("Succeed to connect host[%s] by user[%s]" % (host_ip, user))

    @staticmethod
    def get_cabinet_no(node, cablinet_no, prefix_string, cablinet_list):
        """
        用户填写机柜场景： 检查机柜编号格式和机柜分组是否满足要求
        默认生成机柜场景： 每24个节点分为一组，storage_group1， storage_group2, compute_group1, compute_group2 ...
        """
        max_server_one_cabinet = 24
        rack_start_num = 1
        if cablinet_no:
            if re.search('[^\u4e00-\u9fa5a-zA-Z0-9_-]', cablinet_no):
                logger.info("The cabinet name[%s] is not standard." % cablinet_no)
                raise HCCIException(626113, node)
            if cablinet_list.count(cablinet_no) > max_server_one_cabinet:
                logger.info("More than 24 nodes use the same cabinet name[%s]." % cablinet_no)
                raise HCCIException(626114, cablinet_no)
        else:
            node_num = len(cablinet_list)
            try:
                suffix_num = rack_start_num + int(node_num / max_server_one_cabinet)
            except ZeroDivisionError as e:
                err_msg = 'ZeroDivisionError, Detail:%s' % str(e)
                logger.error(err_msg)
                raise e
            cablinet_no = "{}{}".format(prefix_string, suffix_num)
        cablinet_list.append(cablinet_no)
        return cablinet_no

    @staticmethod
    def get_current_disk_type(all_disk_info, slot):
        current_disk_type = None
        for disk_info in all_disk_info:
            if disk_info.get('devSlot') == slot:
                current_disk_type = disk_info.get('devType')
                break
        return current_disk_type

    @staticmethod
    def check_threading_is_alive(all_ssh_threads, os_ip):
        if all_ssh_threads.get(os_ip) is None:
            return
        if 'is_alive' not in dir(all_ssh_threads.get(os_ip)):
            logger.info('Threading of %s is not starting.' % os_ip)
            return
        if all_ssh_threads.get(os_ip).is_alive():
            time.sleep(1)
            if int(time.time()) // 5 == 0:
                logger.info('Threading is not finished.')
        else:
            logger.info('Threading of %s is finished.' % os_ip)
            all_ssh_threads.pop(os_ip)

    @staticmethod
    def upload_certificates(key, rp, value):
        if key != "password":
            file_name = value.split("/")[-1]
            tiny_data = '{"file":"%s","module":"HWDBS_KVM"}' % file_name
            body = {'tinyFormDatas': tiny_data}
            files = {"tiFile": (file_name, open(value, "rb"))}
            rp.upload_certificates(body, files)

    @staticmethod
    def _get_resource_data(role, cpu, memory):
        data = {"role": role,
                "cpu": {"core": cpu},
                "mem": {"size": (memory * 1024)}}
        return data

    def open_sys_root(self, host_ip, user, passwd, root_pwd):
        cmd_items = ["sed -i \"s/PermitRootLogin.*/PermitRootLogin yes/g\" /etc/ssh/sshd_config",
                     "grep -q -w \"AllowUsers.*root\" /etc/ssh/sshd_config || "
                     "sed -i 's/AllowUsers.*/& root/' /etc/ssh/sshd_config",
                     "grep -q -w \"AllowGroups.*root\" /etc/ssh/sshd_config || "
                     "sed -i 's/AllowGroups.*/& root/' /etc/ssh/sshd_config",
                     "service sshd restart"]
        open_cmds = ";".join(cmd_items)
        ssh_client = None
        cmd_res = None
        try:
            ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        except Exception as e:
            logger.error("Failed to connect host using user fsp, omip:%s, err:%s" % (host_ip, str(e)))
        cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
        logger.info("exe cmd[su root] result:%s" % (str(cmd_res)))
        try:
            cmd_res = ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
        except Exception as e:
            logger.error("Failed to switch to root user, omip:%s, err:%s" % (host_ip, str(e)))
            if ssh_client:
                ssh.ssh_close(ssh_client)
        cmd_res = ssh.ssh_send_command(ssh_client, 'TMOUT=0', '#', 20)
        logger.info("exe cmd[TMOUT=0] result:%s" % (str(cmd_res)))
        try:
            cmd_res = ssh.ssh_send_command(ssh_client, open_cmds, "#", 60, 3)
        except Exception as e:
            logger.error("Failed to open root privilege, omip:%s, err:%s, cmd_result:%s"
                         % (host_ip, str(e), str(cmd_res)))
            if ssh_client:
                ssh.ssh_close(ssh_client)
        logger.info("exe cmd[%s] result:%s" % (open_cmds, str(cmd_res)))
        logger.info('Success to to open root privilege, omip:%s.' % (host_ip))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return True

    def close_sys_root(self, sys_params, in_fsm=False):
        if in_fsm:
            cmd_items = ["sed -i \"s/PermitRootLogin.*/PermitRootLogin no/g\" /etc/ssh/sshd_config",
                         "grep -q -w \"AllowUsers.*root\" /etc/ssh/sshd_config && sed -i '"
                         "s/\\(AllowUsers.*\\)root\\(.*\\)/\\1\\2/' /etc/ssh/sshd_config",
                         "service sshd restart"]
        else:
            cmd_items = ["sed -i \"s/PermitRootLogin.*/PermitRootLogin no/g\" /etc/ssh/sshd_config",
                         "grep -q -w \"AllowUsers.*root\" /etc/ssh/sshd_config && sed -i '"
                         "s/\\(AllowUsers.*\\)root\\(.*\\)/\\1\\2/' /etc/ssh/sshd_config",
                         "grep -q -w \"AllowGroups.*root\" /etc/ssh/sshd_config && sed -i "
                         "'s/\\(AllowGroups.*\\)root\\(.*\\)/\\1\\2/' /etc/ssh/sshd_config",
                         "service sshd restart"]
        open_cmds = ";".join(cmd_items)
        ssh_client = None
        host_ip = sys_params.get('fsm_ip')
        user = sys_params.get('fsm_user')
        passwd = sys_params.get('fsm_passwd')
        root_pwd = sys_params.get('fsm_root_pwd')
        try:
            ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        except Exception as e:
            logger.error("Failed to connect host using user fsp, omip:%s, err:%s" % (host_ip, str(e)))
        try:
            logger.info("Open root privilege, omip:%s" % host_ip)
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("exe cmd[su root] result:%s" % (str(cmd_res)))
            ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            cmd_res = ssh.ssh_send_command(ssh_client, 'TMOUT=0', '#', 20)
            logger.info("exe cmd[TMOUT=0] result:%s" % (str(cmd_res)))
            cmd_res = ssh.ssh_send_command(ssh_client, open_cmds, "#", 60, 3)
            logger.info("exe cmd[%s] result:%s" % (open_cmds, str(cmd_res)))
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)
        logger.info('Success to to close root privilege, omip:%s.' % (host_ip))
        return True

    def add_sshd_host_key(self, node_list):
        # 多线程使用ssh登陆节点执行命令
        logger.info("Start check and add the ssh configuration.")
        self.parallel_run(self._add_sshd_for_one_node, node_list)
        logger.info("Checking and adding the ssh configuration is complete.")

    def check_host_list_ping(self, host_ip_list):
        no_ping_list = list()
        for ip in host_ip_list:
            if not self.host_ping(ip):
                logger.error("Failed to ping host[%s]" % ip)
                no_ping_list.append(ip)
            logger.info("Succeed to ping host[%s]" % ip)
        if len(no_ping_list) > 0:
            logger.error("Failed to ping hosts%s" % no_ping_list)
            return False, no_ping_list
        return True, no_ping_list

    def get_cpu_arch(self, ssh_client):
        cmd_arch = "uname -p"
        ret_cmd = ssh.ssh_exec_command_return(ssh_client, cmd_arch)
        logger.info("Run query cmd[%s], result: %s" % (ret_cmd, str(ret_cmd)))
        ret_cmd = ''.join(ret_cmd)
        ret_arch_list = re.findall('x86_64|aarch64', ret_cmd)
        if len(ret_arch_list) == 1:
            ret_arch = ret_arch_list[0]
        else:
            err_msg = "Failed query cpu arch, cmd result:%s find:%s" % (ret_cmd, ret_arch_list)
            logger.error(err_msg)
            raise Exception(err_msg)
        if ret_arch in ['x86_64', 'aarch64']:
            serv_cpu_arch = ret_arch
        else:
            serv_cpu_arch = 'Unkown'
            err_msg = "Failed get processor type, cmd result:%s find:%s" % (ret_cmd, serv_cpu_arch)
            logger.error(err_msg)
            raise Exception(err_msg)
        logger.info("Processor type:%s" % serv_cpu_arch)
        return serv_cpu_arch

    def query_hosts_info_from_webui(self, host_and_tool=None):
        """
        :param host_and_tool:
        :return:{manageip管理ip:id}
        """
        if host_and_tool:
            self.client = g_webui_client_ctrl.apply_client(host_and_tool, self)
        uri = "api/v1/hosts"
        method = "get"
        body = None
        cmd = {"method": method, "uri": uri, "body": body}
        rsp_status, rsp_header, rsp_body = self.client.exec_cmd(cmd)
        host_info = {}
        if int(rsp_status) >= 200 and int(rsp_status) <= 299:
            hosts = rsp_body.get("hosts", [])
            for host in hosts:
                host_info[host.get("omip")] = host.get("id")
        return host_info

    def encryp_password(self, password, key):
        def_key_a = "_fMZ88MzCLH_QdRvDRwu3w=="
        skey = cryptApi.sxor(def_key_a, key)
        skey = skey.decode() if isinstance(skey, bytes) else skey
        return cryptApi.pwencrypt(skey, password)

    def upload_packet_to_host(self, host_ip, user, passwd):
        pkg_path = self.fs_args.get("fs_install_path")
        if not os.path.isfile(pkg_path):
            logger.error("The installation package[%s] is not exist" % pkg_path)
            raise Exception("The installation package[%s] is not exist" % pkg_path)

        remote_fs_path = "/opt/fusionstorage"
        remote_repo_path = str(pathlib.Path(pathlib.Path(remote_fs_path), "repository"))
        remote_soft_path = str(pathlib.Path(pathlib.Path(remote_repo_path), "deploymanager_pkg/DSwareSoft"))
        ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        cmd_items = ['test -d %s && rm -rf %s' % (remote_fs_path, remote_soft_path),
                     'mkdir -p %s' % remote_soft_path,
                     'chmod 755 %s' % remote_fs_path,
                     'chmod -R 750 %s' % remote_repo_path]
        cmd_dir = ";".join(cmd_items)
        cmd_res1 = ssh.ssh_send_command(ssh_client, cmd_dir, "#", 60, 3)
        logger.info("Make an directory on host, result: %s" % str(cmd_res1))
        logger.info("Start to upload package[%s] to host[%s]" % (pkg_path, host_ip))
        ssh.put_file(host_ip, user, passwd, pkg_path, destination=remote_soft_path)
        logger.info("Upload package complete")

        fs_pkg = os.path.basename(pkg_path)
        fs_name = fs_pkg.strip('.tar.gz')
        cmd_items = ["tar -zxf %s/'%s' -C %s" % (remote_soft_path, fs_pkg, remote_soft_path),
                     "mv %s/'%s'/deploymanager/* %s" % (remote_soft_path, fs_name, remote_soft_path),
                     "chmod -R 700 %s" % remote_soft_path,
                     "chmod 600 %s/*.xlsx; ls -1 --color=none %s " % (remote_soft_path, remote_soft_path)]
        cmd_untar = ";".join(cmd_items)
        cmd_res2 = ssh.execute_command(ssh_client, cmd_untar)
        logger.info("Extract package[%s.tar.gz] complete, result: %s" % (fs_pkg, str(cmd_res2)))

        if ssh_client:
            ssh.ssh_close(ssh_client)
        return

    def get_netport_of_ip(self, ip, user, passwd):
        prefix = "HCC" + "".join(random.choices(string.ascii_letters + string.digits, k=10))
        ssh_client = ssh.ssh_create_client(ip, user, passwd)
        cmd_eth = "for eth in $(ip addr | grep \"^[1-9]+*\" | awk -F: \'{print $2}\' | " \
                  "awk -F@ \'{print $1}\'); do ip addr show $eth | grep -q \'%s\'; " \
                  "[ $? -eq 0 ] && echo network port %s${eth}%s && break; done" % (ip, prefix, prefix)
        res_eth = ssh.ssh_exec_command_return(ssh_client, cmd_eth)
        logger.info("Get network port by cmd[%s], result: %s" % (cmd_eth, str(res_eth)))
        for res in res_eth:
            ret = re.search('{0}.*{0}'.format(prefix), res)
            logger.info("match result:{}".format(ret))
            if ret:
                return ret.group(0)[len(prefix):-len(prefix)]
        err_msg = 'Failed to query the network port, result:{}'.format(res_eth)
        logger.error(err_msg)
        raise Exception(err_msg)

    def get_gateway_and_netmask_of_ip(self, ip, user, passwd, eth_port):
        netmask, gateway = None, None
        for _retry_time in range(3):
            if not netmask:
                ssh_client = ssh.ssh_create_client(ip, user, passwd)
                cmd_mask = "echo mask=$(ifconfig %s | grep %s | awk '{print $4}')" % (eth_port, ip)
                res_eth = ssh.ssh_exec_command_return(ssh_client, cmd_mask)
                logger.info("Get network port by cmd[%s], result: %s" % (cmd_mask, str(res_eth)))
                netmask = self.re_find_ans(res_eth, 'mask')
            if not gateway:
                cmd_gateway = "echo gw=$(route -n | grep '^0.0.0.0' | grep 'UG' | grep %s | " \
                              "head -n 1|awk '{print $2}')" % eth_port
                res_gateway = ssh.ssh_exec_command_return(ssh_client, cmd_gateway)
                logger.info("Get network port by cmd[%s], result: %s" % (cmd_gateway, str(res_gateway)))
                gateway = self.re_find_ans(res_gateway, 'gw')
            if netmask and gateway:
                return netmask[0], gateway[0]
        else:
            err = []
            if not netmask:
                err.append("netmask")
            if not gateway:
                err.append("gateway")
            err_msg = 'Failed to get ' + 'and'.join(err) + "of the host[%s]" % ip
            logger.error(err_msg)
            raise Exception(err_msg)

    def get_master_ha_items(self, args):
        file_path = DeployConstant.HA_FILE
        master_info, standby_info = args.fsm_info.get('master_fsm'), args.fsm_info.get('standby_fsm')
        login_pwd = self.encryp_password(master_info.get('root_pwd'), args.auto_key_b)
        items = ["echo 'product_series=atlantic' > %s" % file_path,
                 "echo 'ha_role=primary' >> %s" % file_path,
                 "echo 'init_role=primary' >> %s" % file_path,
                 "echo 'local_sn=' >> %s" % file_path,
                 "echo 'remote_sn=' >> %s" % file_path,
                 "echo 'local_cabinet=' >> %s" % file_path,
                 "echo 'remote_cabinet=' >> %s" % file_path,
                 "echo 'ha_mode=double'>> %s" % file_path,
                 "echo 'net_mode=single' >> %s" % file_path,
                 "echo 'service_float_ip=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
                 "echo 'service_gateway=%s' >> %s" % (master_info['gateway'], file_path),
                 "echo 'service_mask=%s' >> %s" % (master_info['netmask'], file_path),
                 "echo 'service_local_ip=%s' >> %s" % (master_info['om_ip'], file_path),
                 "echo 'service_local_port=%s' >> %s" % (master_info['netport'], file_path),
                 "echo 'service_remote_ip=%s' >> %s" % (standby_info['om_ip'], file_path),
                 "echo 'service_remote_port=%s' >> %s" % (standby_info['netport'], file_path),
                 "echo 'manager_float_ip=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
                 "echo 'float_ip_for_ha=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
                 "echo 'manager_gateway=%s' >> %s" % (master_info['gateway'], file_path),
                 "echo 'manager_mask=%s' >> %s" % (master_info['netmask'], file_path),
                 "echo 'manager_local_ip=%s' >> %s" % (master_info['om_ip'], file_path),
                 "echo 'manager_local_port=%s' >> %s" % (master_info['netport'], file_path),
                 "echo 'manager_remote_ip=%s' >> %s" % (standby_info['om_ip'], file_path),
                 "echo 'manager_remote_port=%s' >> %s" % (standby_info['netport'], file_path),
                 "echo 'local_host_name=' >> %s" % file_path,
                 "echo 'remote_host_name=' >> %s" % file_path,
                 "echo 'pk_b=%s' >> %s" % (args.auto_key_b, file_path),
                 "echo 'login_user_name=' >> %s" % file_path,
                 "echo 'login_user_pwd=' >> %s" % file_path,
                 "echo 'login_pwd=%s' >> %s" % (login_pwd, file_path),
                 "echo 'remote_login_user_name=' >> %s" % file_path,
                 "echo 'remote_login_user_pwd=' >> %s" % file_path,
                 "echo 'remote_login_pwd=%s' >> %s" % (login_pwd, file_path),
                 "echo 'active_ip=%s' >> %s" % (master_info['om_ip'], file_path),
                 "echo 'standby_ip=%s' >> %s" % (standby_info['om_ip'], file_path),
                 "echo 'install_ha=true' >> %s" % file_path,
                 "echo 'service_ip_list=%s,%s' >> %s" % (master_info['om_ip'], standby_info['om_ip'], file_path),
                 "echo 'internal_ethname=%s' >> %s" % (master_info['netport'], file_path),
                 "echo 'external_ethname=' >> %s" % file_path,
                 "echo 'external_service_gateway=' >> %s" % file_path,
                 "echo 'external_service_mask=' >> %s" % file_path,
                 "echo 'external_service_float_ip=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
                 "echo 'forbid_switch=true' >> %s" % file_path,
                 "echo 'auto_enable_sandbox=1' >> %s" % file_path,
                 "echo 'support_disable_sandbox=0' >> %s" % file_path,
                 "chmod 644 %s;" % file_path]
        return "; ".join(items)

    def get_standby_ha_items(self, args):
        master_info = args.fsm_info.get('master_fsm')
        standby_info = args.fsm_info.get('standby_fsm')
        file_path = DeployConstant.HA_FILE
        items = [
            "echo 'product_series=atlantic' > %s" % file_path,
            "echo 'ha_role=standby' >> %s" % file_path,
            "echo 'init_role=standby' >> %s" % file_path,
            "echo 'ha_mode=double'>> %s" % file_path,
            "echo 'net_mode=single' >> %s" % file_path,
            "echo 'service_float_ip=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
            "echo 'service_gateway=%s' >> %s" % (master_info['gateway'], file_path),
            "echo 'service_mask=%s' >> %s" % (master_info['netmask'], file_path),
            "echo 'service_local_ip=%s' >> %s" % (master_info['om_ip'], file_path),
            "echo 'service_local_port=%s' >> %s" % (master_info['netport'], file_path),
            "echo 'service_remote_ip=%s' >> %s" % (standby_info['om_ip'], file_path),
            "echo 'service_remote_port=%s' >> %s" % (standby_info['netport'], file_path),
            "echo 'manager_float_ip=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
            "echo 'float_ip_for_ha=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
            "echo 'manager_gateway=%s' >> %s" % (master_info['gateway'], file_path),
            "echo 'manager_mask=%s' >> %s" % (master_info['netmask'], file_path),
            "echo 'manager_local_ip=%s' >> %s" % (master_info['om_ip'], file_path),
            "echo 'manager_local_port=%s' >> %s" % (master_info['netport'], file_path),
            "echo 'manager_remote_ip=%s' >> %s" % (standby_info['om_ip'], file_path),
            "echo 'manager_remote_port=%s' >> %s" % (standby_info['netport'], file_path),
            "echo 'local_host_name=' >> %s" % file_path,
            "echo 'remote_host_name=' >> %s" % file_path,
            "echo 'auto_enable_sandbox=1' >> %s" % file_path,
            "echo 'support_disable_sandbox=0' >> %s" % file_path,
            "echo 'pk_b=%s' >> %s" % (args.auto_key_b, file_path),
        ]
        cmd_rw = "; ".join(items)
        return cmd_rw

    def install_fsm(self, host_ip, user, password):
        cmd_int = "sh /opt/fusionstorage/repository/deploymanager_pkg/DSwareSoft/install.sh;echo last_cmd=$?"
        ssh_client = ssh.ssh_create_client(host_ip, "root", password)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_int, 1200, 0)
        cmd_result = str(cmd_res).replace('password', '******')
        logger.info("Run install cmd[%s], result: %s" % (cmd_int, str(cmd_result)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    def clear_fsm(self, host_ip, user, password):
        cmd_items = ["if [ -f /opt/fusionstorage/deploymanager/clouda/clear_all_for_fs_node.sh ]",
                     "then echo file_exist=$?",
                     "elif [ -f /opt/fusionstorage/repository/deploymanager_pkg/DSwareSoft/clear_all_for_fs_node.sh ]",
                     "then echo file_exist=$?",
                     "else echo file_exist=1",
                     "fi",
                     "echo last_cmd=$?"]
        cmd_clr = ";".join(cmd_items)
        ssh_client = ssh.ssh_create_client(host_ip, "root", password)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_clr, 62, 0)
        cmd_res = ''.join(cmd_res)
        if cmd_res.find('file_exist=0') == -1:
            logger.info('Fsm not installed, no need to clear.')
            return cmd_res
        ds_path = "/opt/fusionstorage/repository/deploymanager_pkg/DSwareSoft"
        cmd_items = ["if [ -f /opt/fusionstorage/deploymanager/clouda/clear_all_for_fs_node.sh ]",
                     "then bash /opt/fusionstorage/deploymanager/clouda/clear_all_for_fs_node.sh all",
                     "elif [ -f {}/clear_all_for_fs_node.sh ]".format(ds_path),
                     "then bash {}/clear_all_for_fs_node.sh all".format(ds_path),
                     "fi",
                     "echo last_cmd=$?"]
        cmd_clr = ";".join(cmd_items)
        echo_str = "yes/no"
        ssh.ssh_send_command(ssh_client, cmd_clr, echo_str, 62, 0)
        echo_str = ']#'
        cmd_res = ssh.ssh_send_command(ssh_client, 'yes', echo_str, 1200, 0)
        cmd_res = ''.join(cmd_res[-60:])
        logger.info("Run clear cmd[%s], result: %s" % (cmd_clr, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    def clear_fsa(self, host_ip, user, passwd, root_pwd):
        cmd_items = ["if [ -f /opt/fusionstorage/deploymanager/clouda/clear_all_for_fs_node.sh ]",
                     "then echo file_exist=$?",
                     "else echo file_exist=1",
                     "fi",
                     "echo last_cmd=$?"]
        cmd_clr = ';'.join(cmd_items)
        ssh_client = self.create_ssh_root_client(host_ip, user, passwd, root_pwd)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_clr, 62, 0)
        cmd_res = ''.join(cmd_res)
        if cmd_res.find('file_exist=0') == -1:
            logger.info('Fsa not installed, no need to clear.')
            return 0
        cmd_clr = "bash /opt/fusionstorage/deploymanager/clouda/clear_all_for_fs_node.sh all;echo last_cmd=$?"
        echo_str = "yes/no"
        ssh.ssh_send_command(ssh_client, cmd_clr, echo_str, 62, 0)
        echo_str = '#'
        cmd_res = ssh.ssh_send_command(ssh_client, 'yes', echo_str, 1200, 0)
        cmd_res = ''.join(cmd_res[-60:])
        logger.info("Run clear cmd[%s], result: %s" % (cmd_clr, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        if 0 > str(cmd_res).find('last_cmd=0'):
            return 1
        else:
            return 0

    def get_account_settings_cmd(self, auto_key_b):
        pwd_setting = self.get_pwd_setting()
        file_path = DeployConstant.ACCOUNT_SET_FILE
        items = [
            "echo '[fsm]' > %s" % file_path,
            "echo 'cmdadmin=%s' >> %s" % (self.encryp_password(pwd_setting.get('fsm_cmdadmin'), auto_key_b), file_path),
            "echo 'metadata_backup=%s' >> %s" % (self.encryp_password(pwd_setting.get('fsm_metadata_backup'),
                                                                      auto_key_b), file_path),
            "echo '[database]' >> %s" % file_path,
            "echo 'omm=%s' >> %s" % (self.encryp_password(pwd_setting.get('db_omm'), auto_key_b), file_path),
            "echo 'ommdba=%s' >> %s" % (self.encryp_password(pwd_setting.get('db_ommdba'), auto_key_b), file_path),
            "echo 'nodemanager=%s' >> %s" % (self.encryp_password(pwd_setting.get('db_nodemanager'), auto_key_b),
                                             file_path),
            "echo 'admin=%s' >> %s" % (self.encryp_password(pwd_setting.get('db_admin'), auto_key_b), file_path),
            "echo 'dmdbadmin=%s' >> %s" % (self.encryp_password(pwd_setting.get('db_dmdbadmin'), auto_key_b),
                                           file_path),
            "echo '[oam-u]' >> %s" % file_path,
            "echo 'ibc_os_hs=%s' >> %s" % (self.encryp_password(pwd_setting.get('oam-u_ibc_os_hs'), auto_key_b),
                                           file_path),
            "echo 'nodemanager=%s' >> %s" % (self.encryp_password(pwd_setting.get('oam-u_nodemanager'), auto_key_b),
                                             file_path),
            "echo '[dm_cli]' >> %s" % file_path,
            "echo 'admin=%s' >> %s" % (self.encryp_password(pwd_setting.get('dm_cli_admin'), auto_key_b), file_path),
            "chmod 644 %s" % file_path,
            "echo last_result=$?"
        ]
        cmd_rw = "; ".join(items)
        return cmd_rw

    def get_pwd_setting(self):
        pwd_setting = {
            'fsm_cmdadmin': self.db.get_value_from_cloudparam(self.pod_id, "DistributedStorage", "FSMCmdadminPassword"),
            'fsm_metadata_backup': self.generate_random_character(),
            'db_omm': self.generate_random_character(),
            'db_ommdba': self.generate_random_character(),
            'db_nodemanager': self.generate_random_character(),
            'db_admin': self.generate_random_character(),
            'db_dmdbadmin': self.generate_random_character(),
            'oam-u_ibc_os_hs': self.generate_random_character(),
            'oam-u_nodemanager': self.generate_random_character(),
            'dm_cli_admin': self.db.get_value_from_cloudparam(self.pod_id, "DistributedStorage", "FSMPortalPassword")
        }
        return pwd_setting

    def disable_ntp_service(self, host_ip, user, passwd, root_pwd):
        cmd_ntp = "/opt/dfv/oam/oam-u/nma/scripts/nma_ctl.sh stopNtpService;systemctl " \
                  "stop ntpd; systemctl disable ntpd;echo result:$?"
        ssh_client = self.create_ssh_root_client(host_ip, user, passwd, root_pwd)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_ntp, 1200, 0)
        logger.info("Run clear cmd[%s], result: %s" % (cmd_ntp, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        if 0 > str(cmd_res).find('result:0'):
            err_msg = 'Stop ntpd failed.'
            logger.error(err_msg)

    def config_and_deploy_role(self, host_id_list, cpu, memory):
        cps_web_client = CPSWebRest(self.project_id, self.pod_id, self.db)
        logger.info("Step1 clone fusionstorage-block role")
        resp_role = cps_web_client.fusionstorage_block_clone()
        logger.info("Finish to clone role fusionstoarge-block. Detail:%s" % resp_role)
        role = resp_role.get('role_name')

        logger.info("Step2 config %s's resource" % role)
        resource_data = self._get_resource_data(role, cpu, memory)
        resp_res = cps_web_client.fusionstorage_block_resource(resource_data)
        logger.info("Finish to config resource for role[%s]. Detail:%s" % (role, resp_res))

        logger.info("Step3 deploy %s for nodes%s" % (role, host_id_list))
        cps_web_client.deploy_hosts_roles(host_id_list, role)
        logger.info("Step4 checking whether the role[%s] is deployed on nodes%s" % (role, host_id_list))
        cps_web_client.check_hosts_role_deploy(host_id_list, role)

    def config_encryption_license_switch(self, pod_id, float_ip, check_encryption=True):
        user_name = DeployConstant.FSM_USER
        passwd = self.db.get_value_from_cloudparam(pod_id, "DistributedStorage", "FSMfsdminPassword")
        root_pwd = self.db.get_value_from_cloudparam(pod_id, "DistributedStorage", "FSMrootPassword")
        ssh_client = None
        try:
            ssh_client = ssh.ssh_create_client(float_ip, user_name, passwd)
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("Exec cmd[su root] result:%s" % (str(cmd_res)))
            ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            # 检查文件中是否存在配置项
            cmd_items = ["if grep 'CheckEncryptLicense' /opt/dfv/oam/oam-s/manager/apache_tomcat/webapps/ROOT/WEB-INF/"
                         "classes/serviceLayerBackend.properties",
                         "then echo item_exist=$?",
                         "else echo item_exist=1",
                         "fi",
                         "echo last_cmd=$?"]
            cmd_clr = ";".join(cmd_items)
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_clr, 20)
            cmd_res = ''.join(cmd_res)
            if cmd_res.find('item_exist=0') == -1:
                err_msg = 'Check encryption item does not exist in license.'
                logger.error(err_msg)
                raise Exception(err_msg)
            if check_encryption:
                cmd_config = 'sed -i "s/.*CheckEncryptLicense.*$/CheckEncryptLicense=true/g" /opt/dfv/oam/oam-s/' \
                             'manager/apache_tomcat/webapps/ROOT/WEB-INF/classes/serviceLayerBackend.properties; ' \
                             'echo last_code=$?'
                cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_config, 20)
                logger.info("Exec cmd switch license result: %s" % cmd_res)
            else:
                cmd_config = 'sed -i "s/.*CheckEncryptLicense.*$/CheckEncryptLicense=false/g" /opt/dfv/oam/oam-s/' \
                             'manager/apache_tomcat/webapps/ROOT/WEB-INF/classes/serviceLayerBackend.properties; ' \
                             'echo last_code=$?'
                cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_config, 20)
                logger.info("Exec cmd switch license result: %s" % cmd_res)
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)

    def restart_oams(self, pod_id, float_ip):
        user_name = DeployConstant.FSM_USER
        passwd = self.db.get_value_from_cloudparam(pod_id, "DistributedStorage", "FSMfsdminPassword")
        root_pwd = self.db.get_value_from_cloudparam(pod_id, "DistributedStorage", "FSMrootPassword")
        ssh_client = None
        ssh_client = ssh.ssh_create_client(float_ip, user_name, passwd)
        cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
        logger.info("Exec cmd[su root] result:%s" % (str(cmd_res)))
        ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
        cmd_restart = 'sh /opt/dfv/oam/oam-s/manager/action/restart.sh; echo last_cmd=$?'
        try:
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_restart, 1200, 0)
        except Exception as e:
            err_msg = "Failed to restart oam-s, details: %s" % str(e)
            logger.error(err_msg)
            raise Exception(e) from e
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)
        logger.info("Exec restart oam-s cmd[%s] result: %s" % (cmd_restart, cmd_res))

    def config_create_pool_license_switch(self, pod_id, float_ip, delete=False):
        # 配置创建第一个存储池的参数：支持添加和删除，delete=True代表删除功能，默认False代表添加配置
        vm_operate = VMOperate(self.project_id, self.pod_id, None)
        vm_master, vm_standby = vm_operate.get_vm_data(pod_id, float_ip)
        host_ip = vm_master.get('om_ip')
        user_name = vm_master.get('user')
        passwd = vm_master.get('passwd')
        root_pwd = vm_master.get('root_pwd')
        ssh_client = None
        logger.info('Start to config license')
        try:
            ssh_client = ssh.ssh_create_client(host_ip, user_name, passwd)
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("Exec cmd[su root] result:%s" % (str(cmd_res)))
            try:
                ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            except HCCIException as error:
                err_msg = f'Switch user to root failed. Info：{str(error)}'
                logger.error(err_msg)
                raise HCCIException(626310, err_msg)
            # 无论添加还是删除，先删除一次license.switch配置
            cmd_delete_config = 'su - omm -s /bin/bash -c "sed -i \'/^license.first.pool.switch/d\' /opt/omm/oms/' \
                                'workspace/webapps/dsware/WEB-INF/manager.properties; echo last_code=$?"'
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_delete_config, 20)
            logger.info("Exec cmd delete switch license result: %s" % cmd_res)
            # 如果是删除逻辑，此处就返回
            if delete:
                return
            cmd_config = 'su - omm -s /bin/bash -c \'echo "license.first.pool.switch=close" >> /opt/omm/oms/' \
                         'workspace/webapps/dsware/WEB-INF/manager.properties; echo last_code=$?\''
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_config, 20)
            logger.info("Exec cmd switch license result: %s" % cmd_res)
            time.sleep(10)
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)

    def config_disk_type_switch(self, support=False):
        logger.info('Turn on the switch to support cache disks from different manufacturers')
        if not self.fs_args.get('open_disk_vendor_switch'):
            return
        float_ip = self.fs_args.get('float_ip')
        fsadmin_pwd = self.fs_args.get('fsadmin_pwd')
        root_pwd = self.fs_args.get('fsm_root_pwd')
        ssh_client = None
        file_name = '/opt/omm/oms/workspace/webapps/dsware/WEB-INF/manager.properties'
        pub_cmd = 'su - omm -s /bin/bash -c \'sed -i "s/^disk.vendor.in.pool.check.switch=.' \
                  '*/disk.vendor.in.pool.check.switch=%s/g" %s;echo last_code=$?\''
        try:
            ssh_client = self.create_ssh_root_client(float_ip, 'fsadmin', fsadmin_pwd, root_pwd)
            if support:
                close_cmd = pub_cmd % ('close', file_name)
                logger.info("Run the command [%s] to turn on the switch to support different manufacturers" % close_cmd)
                cmd_res = ssh.ssh_exec_command_return(ssh_client, close_cmd, 20)
                logger.info("Result: %s" % cmd_res)
            else:
                open_cmd = pub_cmd % ('open', file_name)
                logger.info("Execute the command [%s] to turn off the switch" % open_cmd)
                cmd_res = ssh.ssh_exec_command_return(ssh_client, open_cmd, 20)
                logger.info("Result: %s" % cmd_res)
            ssh.ssh_close(ssh_client)
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)

    def enable_fsm_sandbox(self, opr):
        logger.info('Enable fsm sandbox')
        fsm_ip = self.query_fsm_ip(opr)
        data = opr.enable_os_sandbox(fsm_ip)
        failed_nodes_list = []
        for node in data:
            if node.get('code') != 0:
                failed_nodes_list.append(node)
        if failed_nodes_list:
            err_msg = 'Failed enable sandbox nodes list: %s' % failed_nodes_list
            logger.error(err_msg)
            raise HCCIException(626389, failed_nodes_list)

    def disable_fsm_sandbox(self, opr, root_pwd, admin_pwd):
        logger.info("Disable fsm sandbox")
        fsm_ip = self.query_fsm_ip(opr)
        disable_sandbox_fsm = {'node_ips': fsm_ip,
                               'root_password': root_pwd,
                               'admin_password': admin_pwd}
        data = opr.disable_os_sandbox(disable_sandbox_fsm)
        failed_nodes_list = []
        for node in data:
            if node.get('code') != 0:
                failed_nodes_list.append(node)
        if failed_nodes_list:
            err_msg = 'Failed disable sandbox nodes list: %s' % failed_nodes_list
            logger.error(err_msg)
            raise HCCIException(626386, failed_nodes_list)

    def query_fsm_ip(self, opr):
        rsp_obj = opr.query_cluster_servers()
        rsp_code, rsp_result, rsp_data = rsp_obj.get_rsp_data()
        err_code = rsp_result.get('code')
        if rsp_code != 0 or err_code != 0:
            error_des = rsp_result.get('description')
            error_sgt = rsp_result.get('suggestion')
            err_msg = "Failed to query cluster nodes info, Detail:[%s]%s.Suggestion:%s" \
                      % (err_code, error_des, error_sgt)
            logger.error(err_msg)
            raise Exception(err_msg)
        fsm_ip = []
        for server in rsp_data:
            role = server.get('role')
            if 'management' in role:
                fsm_ip.append(server.get('management_ip'))
        logger.info("Get FSM ips:%s" % fsm_ip)
        return fsm_ip

    def modify_resolv_file(self, host_ip, user, password):
        # 注释/etc/resolv.conf文件
        cmd_items = [
            "sed -i 's/^#*/#/g' /etc/resolv.conf",
            "echo last_cmd=$?"
        ]
        cmd_clr = "; ".join(cmd_items)
        ssh_client = ssh.ssh_create_client(host_ip, "root", password)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_clr, 1200, 0)
        logger.info("Run cmd[%s], result: %s" % (cmd_clr, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    def unmodify_resolv_file(self, host_ip, user, password):
        # 取消/etc/resolv.conf文件注释
        cmd_items = [
            "sed -i 's/^#*//g' /etc/resolv.conf",
            "echo last_cmd=$?"
        ]
        cmd_clr = "; ".join(cmd_items)
        ssh_client = ssh.ssh_create_client(host_ip, "root", password)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_clr, 1200, 0)
        logger.info("Run cmd[%s], result: %s" % (cmd_clr, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    def check_third_party_server_disk(self, disk_info, node_list):
        """
        针对戴尔服务器，检查是否安装了扫盘工具perccli
        无论主存槽位的磁盘是什么类型，如果没有安装戴尔的扫盘工具，通过storcli都无法识别到主存槽位对应的磁盘类型

        disk_info: class
        node_list: list
        return:
        """
        osd_ip_list = [osd_info.get('om_ip') for osd_info in node_list]
        osd_ip_slot_list = [(osd_info.get('om_ip'), osd_info.get('primary_slot')) for osd_info in node_list]

        res = disk_info.get_query_data()
        if not res.get('disks'):
            err_msg = 'Failed to query cluster information. Detail:[{}]'.format(res)
            logger.error(err_msg)
            raise Exception(err_msg)

        check_boolean, passed_list = self.check_pool_disk_method(res, osd_ip_slot_list)
        if not check_boolean:
            failed_list = list(set(osd_ip_list).difference(set(passed_list)))
            failed_ip_str = ','.join(failed_list)
            err_msg = 'The number of main storage disks is 0: [{}]'.format(failed_ip_str)
            logger.error(err_msg)
            raise HCCIException(626322, err_msg, failed_ip_str, failed_ip_str)

    def check_pool_disk_method(self, res, osd_ip_slot_list):
        """
        只要主存槽位有盘类型信息，说明非戴尔服务器或安装了扫盘工具perccli，检查通过
        """
        pass_osd_node = list()
        for om_ip, slot_info in osd_ip_slot_list:
            all_disk_info = res.get('disks').get(om_ip)
            logger.info("All disk on node[%s]: %s" % (om_ip, all_disk_info))
            start_slot, end_slot = slot_info.split('-')
            slot = int(start_slot)
            while slot <= int(end_slot):
                current_disk_type = self.get_current_disk_type(all_disk_info, slot)
                if current_disk_type:
                    pass_osd_node.append(om_ip)
                    break
                slot += 1
        if len(pass_osd_node) == len(osd_ip_slot_list):
            return True, ''
        return False, pass_osd_node

    def get_az_id(self, service_name):
        """
        返回业务存储所在az id
        :return:
        部署首region，从region时，Type2,Type3:openstack_region，Type1：cascaded_region
        扩AZ时，新建的AZ ID：Type2/3：openstack_expansion_az，Type1：cascaded_expansion_azk
        """
        az_id = self.param_util.get_param_value(self.pod_id, service_name, 'openstack_expansion_az')
        if not az_id:
            logger.info("get az_id from openstack_region")
            az_id = self.param_util.get_param_value(self.pod_id, service_name, 'openstack_region')
        return az_id

    def parallel_run(self, func_name, nodes, timeout=120,
                     parallel_threads=20, **kwargs):
        offset = 0
        total_ip = len(nodes)
        all_ssh_threads = {}
        nodes_ip = [node.get('om_ip') for node in nodes]
        while True:
            this_round_node_list = nodes[offset: offset + parallel_threads]
            this_round_ip_list = [node.get('om_ip') for node in this_round_node_list]
            logger.info(
                'Starting new threads for %s' % ','.join(this_round_ip_list))
            logger.info('Remaining ips: %s' % ','.join(
                nodes_ip[offset + parallel_threads:]))
            for node in this_round_node_list:
                os_ip = node.get('om_ip')
                kwargs.update(node)
                _t = threading.Thread(target=func_name, kwargs=kwargs)
                _t.start()
                all_ssh_threads[os_ip] = _t
            time.sleep(1)
            start_time = time.time()
            while start_time + timeout > time.time():
                if not all_ssh_threads:
                    break
                for os_ip in this_round_ip_list:
                    self.check_threading_is_alive(all_ssh_threads, os_ip)
            offset = offset + parallel_threads
            if offset >= total_ip:
                break

    def upload_cert_and_get_private_key(self, webui_client):
        logger.info('Upload certificates')
        cert_file_dir = DeployConstant.FS_DUMP_TMP_DIR
        storage_tool = DistributedStorageTool(self.project_id, self.pod_id, self.db)
        fsc_cli_cert = storage_tool.get_fsc_cli_cert_file(cert_file_dir, self.fs_args.get('float_ip'))
        resource_pool = ResourcePool(webui_client, self.project_id)
        # 调用cps-web上传证书文件
        try:
            for key, value in fsc_cli_cert.items():
                self.upload_certificates(key, resource_pool, value)
        finally:
            shutil.rmtree(cert_file_dir)
        logger.info('Get private key')
        private_key_psw = fsc_cli_cert.get("password")
        return private_key_psw

    def check_es3000_driver(self, ssh_client, cpu_arch, driver_list, role='business'):
        not_list = None
        if not driver_list:
            return not_list
        kernel_dic = DeployConstant.KERNEL_DIC.get(cpu_arch)
        es3000_driver_list = DeployConstant.NVME_NAME.get(cpu_arch)
        self.check_es3000_driverlist_kernel_dic(cpu_arch, es3000_driver_list, kernel_dic)
        logger.info('es3000 pkg list:{}'.format(es3000_driver_list))
        logger.info('The cpu arch:{}'.format(cpu_arch))
        logger.info('driver list:{}'.format(driver_list))
        kernel_pattern = kernel_dic.get(role)
        if es3000_driver_list[0] not in driver_list:
            return driver_list
        driver_list = self._get_driver_list(driver_list, es3000_driver_list, kernel_pattern, ssh_client)
        return driver_list

    def check_es3000_driverlist_kernel_dic(self, cpu_arch, es3000_driver_list, kernel_dic):
        if not es3000_driver_list or not kernel_dic:
            err_msg = 'Failed to get kernel or es3000_driver_list by {}'.format(cpu_arch)
            logger.error(err_msg)
            raise HCCIException(626109, err_msg)

    def _add_sshd_for_one_node(self, **node):
        om_ip = node.get("management_internal_ip")
        user = node.get("user_name")
        passwd = node.get("password")
        root_pwd = node.get("root_password")
        logger.info("add sshd for node[%s] start" % om_ip)
        ssh_client = self.create_ssh_root_client(om_ip, user, passwd, root_pwd)
        # 查询/etc/ssh/sshd_config中HostKeyAlgorithms配置项是否缺协议
        query_cmd = "sed -n '/^#HostKey \/etc\/ssh\/ssh_host_ecdsa_key$/p' /etc/ssh/sshd_config"
        query_result2 = ssh.ssh_exec_command_return(ssh_client, query_cmd)
        if re.findall('#HostKey /etc/ssh/ssh_host_ecdsa_key',
                      "".join(query_result2)):
            operate_cmd = "sed -i 's/^#HostKey \/etc\/ssh\/ssh_host_ecdsa_key$/" \
                          "HostKey \/etc\/ssh\/ssh_host_ecdsa_key/' /etc/ssh/sshd_config"
            ssh.ssh_exec_command_return(ssh_client, operate_cmd)
        query_cmd = "sed -n '/^HostKeyAlgorithms/p' /etc/ssh/sshd_config"
        query_result1 = ssh.ssh_exec_command_return(ssh_client, query_cmd)
        logger.info("Run query cmd[%s] on node[%s],result:%s" % (query_cmd, om_ip, str(query_result1)))
        if not re.findall('ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521',
                          "".join(query_result1)):
            logger.info("Add the ssh configuration to the /etc/ssh/sshd_config on node %s." % om_ip)
            operate_cmd = "sed -i 's/^HostKeyAlgorithms.*/&,ecdsa-sha2-nistp256," \
                          "ecdsa-sha2-nistp384,ecdsa-sha2-nistp521/' /etc/ssh/sshd_config"
            ssh.ssh_exec_command_return(ssh_client, operate_cmd)
        # HostKeyAlgorithms配置项行末添加ssh协议
        query_result2 = ssh.ssh_exec_command_return(ssh_client, query_cmd)
        # ssh-rsa协议添加成功则重启sshd服务使配置生效
        if re.findall('ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521',
                      "".join(query_result2)):
            restart_cmd = "systemctl restart sshd"
            ssh.ssh_exec_command_return(ssh_client, restart_cmd)
            logger.info("restart sshd for node[%s]." % om_ip)
        ssh.ssh_close(ssh_client)

    def _get_driver_list(self, driver_list, es3000_driver_list, kernel_pattern, ssh_client):
        cmd = 'uname -r'
        res = ssh.ssh_exec_command_return(ssh_client, cmd)
        logger.info("cmd: %s, result: %s" % (cmd, str(res)))
        if re.search(kernel_pattern, str(res)):
            driver_list = list(set(driver_list).difference(set(es3000_driver_list)))
        logger.info('Res:{}'.format(driver_list))
        return driver_list


class InstallDriverOperate(object):
    def __init__(self):
        pass

    @staticmethod
    def sort_drivers(driver_infos, all_drivers):
        """
        以驱动为依据，对驱动列表进行分批次
        return: [(['nvme'], ['192.168.1.201', '192.168.1.202']), (['cx5'], ['192.168.1.201']), ...]
        """
        driver_list = list()
        for driver in set(all_drivers):
            ip_list = []
            for node in driver_infos:
                if driver in driver_infos.get(node):
                    ip_list.append(node)
            if ip_list:
                data = ([driver], ip_list)
                driver_list.append(data)
        return driver_list

    @staticmethod
    def group_by_account_info(host, need_install_driver_hosts, driver_list):
        """
        按相同账户信息的节点分组

        need_install_driver_hosts:
        {'user1pwdpwd': ['user1', 'pwd', 'pwd1', {'192.168.1.201': ['cx4'], ''192.168.1.202': ['cx4', 'cx5']}],...}
        """
        om_ip, user, passwd, root_pwd = host.get('om_ip'), host.get('user'), host.get('passwd'), host.get('root_pwd')
        account_str = user + passwd + root_pwd
        if account_str in need_install_driver_hosts.keys():
            need_install_driver_hosts[account_str][-1][om_ip] = driver_list
        else:
            need_install_driver_hosts[account_str] = [user, passwd, root_pwd, {om_ip: driver_list}]
