# coding=utf-8
import os
import threading
import time
import re
import subprocess
import shlex
import secrets
from IPy import IP

from utils.common.message import Message
from utils.common import log as logger
from utils.common.fic_base import ToolClient
from utils.client.webUI_client import g_webui_client_ctrl
from utils.business.param_util import ParamUtil
from utils.common.exception import FCDException
from utils.DBAdapter.DBConnector import BaseOps
from utils.common.ssh_util import Ssh as ssh
from utils.Driver.CloudDC.OpenStack.baremetal.resource_pool import ResourcePool
import plugins.DistributedStorage.scripts.utils.common.cryptApi as cryptApi
from plugins.DistributedStorage.scripts.utils.client.cpsWebClient import cpsWebRest
from plugins.DistributedStorage.scripts.logic.VMOperate import VMOperate
from plugins.DistributedStorage.scripts.utils.common.DeployConstant import DeployConstant
from plugins.DistributedStorage.scripts.utils.interface.DistributedStorage import DistributedStorageTool


class InstallOperate(object):
    def __init__(self, project_id=None, pod_id=None, fs_args=None):
        self.client = None
        self.body = None
        self.is_pass = None
        self.default_client = ToolClient.DSWARE_CLIENT
        self.project_id = project_id
        self.pod_id = pod_id
        self.fs_args = fs_args
        self.db = BaseOps()
        self.param_util = ParamUtil()

    @staticmethod
    def create_ssh_root_client(host_ip, user, passwd, root_pwd):
        try:
            ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        except Exception as e:
            create_err_msg = "Failed to connect host using user fsp, omip:%s, err:%s" % (host_ip, str(e))
            logger.error(create_err_msg)
            raise Exception(create_err_msg)
        try:
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("exe cmd[su root] result:%s" % (str(cmd_res)))

            cmd_res = ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            logger.info("exe cmd[#] result:%s" % (str(cmd_res)))

            cmd_res = ssh.ssh_send_command(ssh_client, 'TMOUT=0', '#', 20)
            logger.info("exe cmd[TMOUT=0] result:%s" % (str(cmd_res)))
        except Exception as e:
            switch_err_msg = "Failed to switch to root user, omip:%s, err:%s" % (host_ip, str(e))
            logger.error(switch_err_msg)
            if ssh_client:
                ssh.ssh_close(ssh_client)
            raise Exception(switch_err_msg)
        return ssh_client

    def open_sys_root(self, host_ip, user, passwd, root_pwd):
        cmd_items = ["sed -i \"s/PermitRootLogin.*/PermitRootLogin yes/g\" /etc/ssh/sshd_config",
                     "grep -q -w \"AllowUsers.*root\" /etc/ssh/sshd_config || "
                     "sed -i 's/AllowUsers.*/& root/' /etc/ssh/sshd_config",
                     "grep -q -w \"AllowGroups.*root\" /etc/ssh/sshd_config || "
                     "sed -i 's/AllowGroups.*/& root/' /etc/ssh/sshd_config",
                     "service sshd restart"]
        open_cmds = ";".join(cmd_items)
        ssh_client = None
        cmd_res = None
        try:
            ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        except Exception as e:
            logger.error("Failed to connect host using user fsp, omip:%s, err:%s" % (host_ip, str(e)))
        try:
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("exe cmd[su root] result:%s" % (str(cmd_res)))
            cmd_res = ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            cmd_res = ssh.ssh_send_command(ssh_client, 'TMOUT=0', '#', 20)
            logger.info("exe cmd[TMOUT=0] result:%s" % (str(cmd_res)))
        except Exception as e:
            logger.error("Failed to switch to root user, omip:%s, err:%s" % (host_ip, str(e)))
            if ssh_client:
                ssh.ssh_close(ssh_client)
        try:
            cmd_res = ssh.ssh_send_command(ssh_client, open_cmds, "#", 60, 3)
            logger.info("exe cmd[%s] result:%s" % (open_cmds, str(cmd_res)))
        except Exception as e:
            logger.error("Failed to open root privilege, omip:%s, err:%s, cmd_result:%s"
                         % (host_ip, str(e), str(cmd_res)))
            if ssh_client:
                ssh.ssh_close(ssh_client)
        logger.info('Success to to open root privilege, omip:%s.' % (host_ip))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return True

    def close_sys_root(self, host_ip, user, passwd, root_pwd, in_fsm=False):
        if in_fsm:
            cmd_items = ["sed -i \"s/PermitRootLogin.*/PermitRootLogin no/g\" /etc/ssh/sshd_config",
                         "grep -q -w \"AllowUsers.*root\" /etc/ssh/sshd_config && sed -i '"
                         "s/\\(AllowUsers.*\\)root\\(.*\\)/\\1\\2/' /etc/ssh/sshd_config",
                         "service sshd restart"]
        else:
            cmd_items = ["sed -i \"s/PermitRootLogin.*/PermitRootLogin no/g\" /etc/ssh/sshd_config",
                         "grep -q -w \"AllowUsers.*root\" /etc/ssh/sshd_config && sed -i '"
                         "s/\\(AllowUsers.*\\)root\\(.*\\)/\\1\\2/' /etc/ssh/sshd_config",
                         "grep -q -w \"AllowGroups.*root\" /etc/ssh/sshd_config && sed -i "
                         "'s/\\(AllowGroups.*\\)root\\(.*\\)/\\1\\2/' /etc/ssh/sshd_config",
                         "service sshd restart"]
        open_cmds = ";".join(cmd_items)
        ssh_client = None
        cmd_res = None
        try:
            ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        except Exception as e:
            logger.error("Failed to connect host using user fsp, omip:%s, err:%s" % (host_ip, str(e)))
        try:
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("exe cmd[su root] result:%s" % (str(cmd_res)))
            cmd_res = ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            cmd_res = ssh.ssh_send_command(ssh_client, 'TMOUT=0', '#', 20)
            logger.info("exe cmd[TMOUT=0] result:%s" % (str(cmd_res)))
        except Exception as e:
            logger.error("Failed to switch to root user, omip:%s, err:%s" % (host_ip, str(e)))
            if ssh_client:
                ssh.ssh_close(ssh_client)
        try:
            cmd_res = ssh.ssh_send_command(ssh_client, open_cmds, "#", 60, 3)
            logger.info("exe cmd[%s] result:%s" % (open_cmds, str(cmd_res)))
        except Exception as e:
            logger.error("Failed to open root privilege, omip:%s, err:%s, cmd_result:%s"
                         % (host_ip, str(e), str(cmd_res)))
            if ssh_client:
                ssh.ssh_close(ssh_client)
        logger.info('Success to to close root privilege, omip:%s.' % (host_ip))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return True

    def add_sshd_host_key(self, node_list):
        # 多线程使用ssh登陆节点执行命令
        logger.info("Start check and add the ssh configuration.")
        self.parallel_run(self._add_sshd_for_one_node, node_list)
        logger.info("Checking and adding the ssh configuration is complete.")

    def _add_sshd_for_one_node(self, **node):
        om_ip = node.get("management_internal_ip")
        user = node.get("user_name")
        passwd = node.get("password")
        root_pwd = node.get("root_password")
        logger.info("add sshd for node[%s] start" % om_ip)
        ssh_client = self.create_ssh_root_client(om_ip, user, passwd, root_pwd)
        # 查询/etc/ssh/sshd_config中HostKeyAlgorithms配置项是否缺协议
        query_cmd = "sed -n '/^#HostKey \/etc\/ssh\/ssh_host_ecdsa_key$/p' /etc/ssh/sshd_config"
        query_result2 = ssh.ssh_exec_command_return(ssh_client, query_cmd)
        if re.findall('#HostKey /etc/ssh/ssh_host_ecdsa_key',
                      "".join(query_result2)):
            operate_cmd = "sed -i 's/^#HostKey \/etc\/ssh\/ssh_host_ecdsa_key$/" \
                          "HostKey \/etc\/ssh\/ssh_host_ecdsa_key/' /etc/ssh/sshd_config"
            ssh.ssh_exec_command_return(ssh_client, operate_cmd)
        query_cmd = "sed -n '/^HostKeyAlgorithms/p' /etc/ssh/sshd_config"
        query_result1 = ssh.ssh_exec_command_return(ssh_client, query_cmd)
        logger.info("Run query cmd[%s] on node[%s],result:%s" % (query_cmd, om_ip, str(query_result1)))
        if not re.findall('ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521',
                          "".join(query_result1)):
            logger.info("Add the ssh configuration to the /etc/ssh/sshd_config on node %s." % om_ip)
            operate_cmd = "sed -i 's/^HostKeyAlgorithms.*/&,ecdsa-sha2-nistp256," \
                          "ecdsa-sha2-nistp384,ecdsa-sha2-nistp521/' /etc/ssh/sshd_config"
            ssh.ssh_exec_command_return(ssh_client, operate_cmd)
        # HostKeyAlgorithms配置项行末添加ssh协议
        query_result2 = ssh.ssh_exec_command_return(ssh_client, query_cmd)
        # ssh-rsa协议添加成功则重启sshd服务使配置生效
        if re.findall('ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521',
                      "".join(query_result2)):
            restart_cmd = "systemctl restart sshd"
            ssh.ssh_exec_command_return(ssh_client, restart_cmd)
            logger.info("restart sshd for node[%s]." % om_ip)
        ssh.ssh_close(ssh_client)

    @staticmethod
    def get_certificate_path(cert_pattern, dir_path=None, all_match=False):
        if not os.path.exists(dir_path):
            logger.error("Directory path [%s] does not exist. " % dir_path)
            raise FCDException(626367, dir_path)
        items = os.listdir(dir_path)
        cert_name = None
        for names in items:
            if len(re.findall(cert_pattern, names)) > 0:
                cert_name = names
                if all_match:
                    continue
                break
        if not cert_name:
            logger.error("The certificate of %s is not exist." % cert_pattern)
            raise FCDException(626368, cert_pattern, dir_path)
        cert_path = '{}/{}'.format(dir_path, cert_name)
        logger.info("Certificate pattern:%s, Find certificate:%s" % (cert_pattern, cert_path))
        return cert_path

    def check_host_list_ping(self, host_ip_list):
        no_ping_list = list()
        for ip in host_ip_list:
            if not self.host_ping(ip):
                logger.error("Failed to ping host[%s]" % ip)
                no_ping_list.append(ip)
            logger.info("Succeed to ping host[%s]" % ip)
        if len(no_ping_list) > 0:
            logger.error("Failed to ping hosts%s" % no_ping_list)
            return False, no_ping_list
        return True, no_ping_list

    @staticmethod
    def host_ping(host_ip):
        try:
            IP(host_ip)
        except Exception:
            return False
        if IP(host_ip).version() == 6:
            ping_act = "ping6"
        else:
            ping_act = "ping"
        cmd = "%s %s -c 2" % (ping_act, host_ip)
        cmd = shlex.split(cmd)
        p = subprocess.Popen(args=cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        (std_output, err_output) = p.communicate()
        std_output = std_output.decode('utf-8') if isinstance(std_output, bytes) else std_output
        if std_output.find('ttl=') >= 0:
            return True
        return False

    def get_cpu_arch(self, ssh_client):
        cmd_arch = "uname -p"
        ret_cmd = ssh.ssh_exec_command_return(ssh_client, cmd_arch)
        logger.info("Run query cmd[%s], result: %s" % (ret_cmd, str(ret_cmd)))
        ret_cmd = ''.join(ret_cmd)
        ret_arch_list = re.findall('x86_64|aarch64', ret_cmd)
        if len(ret_arch_list) == 1:
            ret_arch = ret_arch_list[0]
        else:
            err_msg = "Failed query cpu arch, cmd result:%s find:%s" % (ret_cmd, ret_arch_list)
            logger.error(err_msg)
            raise Exception(err_msg)
        if ret_arch in ['x86_64', 'aarch64']:
            serv_cpu_arch = ret_arch
        else:
            serv_cpu_arch = 'Unkown'
            err_msg = "Failed get processor type, cmd result:%s find:%s" % (ret_cmd, serv_cpu_arch)
            logger.error(err_msg)
            raise Exception(err_msg)
        logger.info("Processor type:%s" % serv_cpu_arch)
        return serv_cpu_arch

    @staticmethod
    def reboot_host_list(host_data):
        host_ip_list = [host.get('om_ip') for host in host_data]
        logger.info('Rebooting hosts:%s' % host_ip_list)
        cmd_reboot = 'reboot'
        for host in host_data:
            user = host.get('user')
            passwd = host.get('passwd')
            root_pwd = host.get('root_pwd')
            host_ip = host.get('om_ip')
            ssh_client = InstallOperate.create_ssh_root_client(host_ip, user, passwd, root_pwd)
            cmd_ret = ssh.ssh_exec_command_return(ssh_client, cmd_reboot)
            logger.info("Rebooting host[%s], result: %s" % (host_ip, str(cmd_ret)))
            if ssh_client:
                ssh.ssh_close(ssh_client)
        time.sleep(120)
        timeout = 900
        while timeout > 0:
            for host in host_ip_list:
                if InstallOperate.host_ping(host):
                    logger.info('The host[%s] is avaible' % host)
                    host_ip_list.remove(host)
            if len(host_ip_list) == 0:
                break
            logger.info('The remaining hosts%s are still unreachable. Try to ping it after 30 second.' % host_ip_list)
            time.sleep(30)
            timeout -= 30
        if len(host_ip_list) > 0:
            err_msg = "The host%s is still not restarted. Please check it by manually." % host_ip_list
            logger.error(err_msg)
            raise Exception(err_msg)

    def query_hosts_info_from_webui(self, host_and_tool=None):
        """
        :param host_and_tool:
        :return:{manageip管理ip:id}
        """
        if host_and_tool:
            self.client = g_webui_client_ctrl.apply_client(host_and_tool, self)
        uri = "api/v1/hosts"
        method = "get"
        body = None
        cmd = {"method": method, "uri": uri, "body": body}
        rsp_status, rsp_header, rsp_body = self.client.exec_cmd(cmd)
        host_info = {}
        if int(rsp_status) >= 200 and int(rsp_status) <= 299:
            hosts = rsp_body.get("hosts", [])
            for host in hosts:
                host_info[host.get("omip")] = host.get("id")
        return host_info

    def encryp_password(self, password, key):
        def_key_a = "_fMZ88MzCLH_QdRvDRwu3w=="
        skey = cryptApi.sxor(def_key_a, key)
        skey = skey.decode() if isinstance(skey, bytes) else skey
        return cryptApi.pwencrypt(skey, password)

    def upload_packet_to_host(self, host_ip, user, passwd):
        pkg_path = self.fs_args.get("fs_install_path")
        if not os.path.isfile(pkg_path):
            logger.error("The installation package[%s] is not exist" % pkg_path)
            raise Exception("The installation package[%s] is not exist" % pkg_path)

        remote_fs_path = "/opt/fusionstorage"
        remote_repo_path = remote_fs_path + "/repository"
        remote_soft_path = remote_repo_path + "/deploymanager_pkg/DSwareSoft"
        ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        cmd_items = ['test -d %s && rm -rf %s' % (remote_fs_path, remote_soft_path),
                     'mkdir -p %s' % remote_soft_path,
                     'chmod 755 %s' % remote_fs_path,
                     'chmod -R 750 %s' % remote_repo_path]
        cmd_dir = ";".join(cmd_items)
        cmd_res1 = ssh.ssh_send_command(ssh_client, cmd_dir, "#", 60, 3)
        logger.info("Make an directory on host, result: %s" % str(cmd_res1))
        ssh.put_file(host_ip, user, passwd, pkg_path, destination=remote_soft_path)
        logger.info("Start to upload package[%s] to host[%s]" % (pkg_path, host_ip))
        logger.info("Upload package complete")

        fs_pkg = os.path.basename(pkg_path)
        fs_name = fs_pkg.strip('.tar.gz')
        cmd_items = ["tar -zxf %s/'%s' -C %s" % (remote_soft_path, fs_pkg, remote_soft_path),
                     "mv %s/'%s'/deploymanager/* %s" % (remote_soft_path, fs_name, remote_soft_path),
                     "chmod -R 700 %s" % remote_soft_path,
                     "chmod 600 %s/*.xlsx; ls -1 --color=none %s " % (remote_soft_path, remote_soft_path)]
        cmd_untar = ";".join(cmd_items)
        cmd_res2 = ssh.execute_command(ssh_client, cmd_untar)
        logger.info("Extract package[%s.tar.gz] complete, result: %s" % (fs_pkg, str(cmd_res2)))

        if ssh_client:
            ssh.ssh_close(ssh_client)
        return

    def get_netport_of_ip(self, ip, user, passwd):
        ssh_client = ssh.ssh_create_client(ip, user, passwd)
        cmd_eth = "for eth in $(ip addr | grep \"^[1-9]+*\" | awk -F: \'{print $2}\' | " \
                  "awk -F@ \'{print $1}\'); do ip addr show $eth | grep -q \'%s\'; " \
                  "[ $? -eq 0 ] && echo network port $eth && break; done" % ip
        res_eth = ssh.ssh_exec_command_return(ssh_client, cmd_eth)
        logger.info("Get network port by cmd[%s], result: %s" % (cmd_eth, str(res_eth)))
        for res in res_eth:
            if re.match('network port', res):
                return res.strip().split()[-1].strip()
        err_msg = 'Failed to query the network port, result:{}'.format(res_eth)
        logger.error(err_msg)
        raise Exception(err_msg)

    def get_gateway_and_netmask_of_ip(self, ip, user, passwd, eth_port):
        netmask, gateway = None, None
        for retry_time in range(3):
            if not netmask:
                ssh_client = ssh.ssh_create_client(ip, user, passwd)
                cmd_mask = "echo mask=$(ifconfig %s | grep %s | awk '{print $4}')" % (eth_port, ip)
                res_eth = ssh.ssh_exec_command_return(ssh_client, cmd_mask)
                logger.info("Get network port by cmd[%s], result: %s" % (cmd_mask, str(res_eth)))
                netmask = self.re_find_ans(res_eth, 'mask')
            if not gateway:
                cmd_gateway = "echo gw=$(route -n | grep '^0.0.0.0' | grep 'UG' | grep %s | " \
                              "head -n 1|awk '{print $2}')" % eth_port
                res_gateway = ssh.ssh_exec_command_return(ssh_client, cmd_gateway)
                logger.info("Get network port by cmd[%s], result: %s" % (cmd_gateway, str(res_gateway)))
                gateway = self.re_find_ans(res_gateway, 'gw')
            if netmask and gateway:
                return netmask[0], gateway[0]
        else:
            err = []
            if not netmask:
                err.append("netmask")
            if not gateway:
                err.append("gateway")
            err_msg = 'Failed to get ' + 'and'.join(err) + "of the host[%s]" % ip
            logger.error(err_msg)
            raise Exception(err_msg)

    @staticmethod
    def re_find_ans(res_ssh, mark):
        res_ssh = ''.join(res_ssh)
        pattern = '%s=(\d+\.\d+\.\d+\.\d+)' % mark
        res = re.findall(pattern, res_ssh)
        return res

    def get_config_ha_items(self, file_path, master_info, standby_info, auto_key_b, login_pwd):
        items = ["echo 'product_series=atlantic' > %s" % file_path,
                 "echo 'ha_role=primary' >> %s" % file_path,
                 "echo 'init_role=primary' >> %s" % file_path,
                 "echo 'local_sn=' >> %s" % file_path,
                 "echo 'remote_sn=' >> %s" % file_path,
                 "echo 'local_cabinet=' >> %s" % file_path,
                 "echo 'remote_cabinet=' >> %s" % file_path,
                 "echo 'ha_mode=double'>> %s" % file_path,
                 "echo 'net_mode=single' >> %s" % file_path,
                 "echo 'service_float_ip=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
                 "echo 'service_gateway=%s' >> %s" % (master_info['gateway'], file_path),
                 "echo 'service_mask=%s' >> %s" % (master_info['netmask'], file_path),
                 "echo 'service_local_ip=%s' >> %s" % (master_info['om_ip'], file_path),
                 "echo 'service_local_port=%s' >> %s" % (master_info['netport'], file_path),
                 "echo 'service_remote_ip=%s' >> %s" % (standby_info['om_ip'], file_path),
                 "echo 'service_remote_port=%s' >> %s" % (standby_info['netport'], file_path),
                 "echo 'manager_float_ip=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
                 "echo 'float_ip_for_ha=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
                 "echo 'manager_gateway=%s' >> %s" % (master_info['gateway'], file_path),
                 "echo 'manager_mask=%s' >> %s" % (master_info['netmask'], file_path),
                 "echo 'manager_local_ip=%s' >> %s" % (master_info['om_ip'], file_path),
                 "echo 'manager_local_port=%s' >> %s" % (master_info['netport'], file_path),
                 "echo 'manager_remote_ip=%s' >> %s" % (standby_info['om_ip'], file_path),
                 "echo 'manager_remote_port=%s' >> %s" % (standby_info['netport'], file_path),
                 "echo 'local_host_name=' >> %s" % file_path,
                 "echo 'remote_host_name=' >> %s" % file_path,
                 "echo 'pk_b=%s' >> %s" % (auto_key_b, file_path),
                 "echo 'login_user_name=' >> %s" % file_path,
                 "echo 'login_user_pwd=' >> %s" % file_path,
                 "echo 'login_pwd=%s' >> %s" % (login_pwd, file_path),
                 "echo 'remote_login_user_name=' >> %s" % file_path,
                 "echo 'remote_login_user_pwd=' >> %s" % file_path,
                 "echo 'remote_login_pwd=%s' >> %s" % (login_pwd, file_path),
                 "echo 'active_ip=%s' >> %s" % (master_info['om_ip'], file_path),
                 "echo 'standby_ip=%s' >> %s" % (standby_info['om_ip'], file_path),
                 "echo 'install_ha=true' >> %s" % file_path,
                 "echo 'service_ip_list=%s,%s' >> %s" % (master_info['om_ip'], standby_info['om_ip'], file_path),
                 "echo 'internal_ethname=%s' >> %s" % (master_info['netport'], file_path),
                 "echo 'external_ethname=' >> %s" % file_path,
                 "echo 'external_service_gateway=' >> %s" % file_path,
                 "echo 'external_service_mask=' >> %s" % file_path,
                 "echo 'external_service_float_ip=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
                 "echo 'forbid_switch=true' >> %s" % file_path,
                 "echo 'auto_enable_sandbox=1' >> %s" % file_path,
                 "echo 'support_disable_sandbox=0' >> %s" % file_path,
                 "chmod 644 %s;" % file_path]
        return items

    def general_HAConfig_on_master(self, host_ip, user, passwd, master_info, standby_info):
        auto_key_b = cryptApi.gsecrtkey()
        if isinstance(auto_key_b, bytes):
            auto_key_b = auto_key_b.decode()
        login_pwd = self.encryp_password(master_info['root_pwd'], auto_key_b)
        self.add_account_settings(host_ip, user, passwd, auto_key_b)
        file_path = "/home/HAInfoFromInstallTool.properties"
        items = self.get_config_ha_items(file_path, master_info, standby_info, auto_key_b, login_pwd)
        cmd_rw = "; ".join(items)
        ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        cmd_res = ssh.ssh_send_command(ssh_client, cmd_rw, "#", 60, 3)
        logger.info("Make an directory on host, result: %s" % str(cmd_res))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    def general_HAConfig_on_standby(self, host_ip, user, passwd, master_info, standby_info):
        file_path = "/home/HAInfoFromInstallTool.properties"
        items = [
            "echo 'product_series=atlantic' > %s" % file_path,
            "echo 'ha_role=standby' >> %s" % file_path,
            "echo 'init_role=standby' >> %s" % file_path,
            "echo 'ha_mode=double'>> %s" % file_path,
            "echo 'net_mode=single' >> %s" % file_path,
            "echo 'service_float_ip=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
            "echo 'service_gateway=%s' >> %s" % (master_info['gateway'], file_path),
            "echo 'service_mask=%s' >> %s" % (master_info['netmask'], file_path),
            "echo 'service_local_ip=%s' >> %s" % (master_info['om_ip'], file_path),
            "echo 'service_local_port=%s' >> %s" % (master_info['netport'], file_path),
            "echo 'service_remote_ip=%s' >> %s" % (standby_info['om_ip'], file_path),
            "echo 'service_remote_port=%s' >> %s" % (standby_info['netport'], file_path),
            "echo 'manager_float_ip=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
            "echo 'float_ip_for_ha=%s' >> %s" % (self.fs_args.get("float_ip"), file_path),
            "echo 'manager_gateway=%s' >> %s" % (master_info['gateway'], file_path),
            "echo 'manager_mask=%s' >> %s" % (master_info['netmask'], file_path),
            "echo 'manager_local_ip=%s' >> %s" % (master_info['om_ip'], file_path),
            "echo 'manager_local_port=%s' >> %s" % (master_info['netport'], file_path),
            "echo 'manager_remote_ip=%s' >> %s" % (standby_info['om_ip'], file_path),
            "echo 'manager_remote_port=%s' >> %s" % (standby_info['netport'], file_path),
            "echo 'local_host_name=' >> %s" % file_path,
            "echo 'remote_host_name=' >> %s" % file_path
        ]
        cmd_rw = "; ".join(items)

        ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        cmd_res = ssh.ssh_send_command(ssh_client, cmd_rw, "#", 60, 3)
        logger.info("Make an directory on host, result: %s" % str(cmd_res))

        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    def install_fsm(self, host_ip, user, password):
        cmd_int = "sh /opt/fusionstorage/repository/deploymanager_pkg/DSwareSoft/install.sh;echo last_cmd=$?"
        ssh_client = ssh.ssh_create_client(host_ip, "root", password)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_int, 1200, 0)
        cmd_result = str(cmd_res).replace('password', '******')
        logger.info("Run install cmd[%s], result: %s" % (cmd_int, str(cmd_result)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    @staticmethod
    def close_os_first_login(host_ip, user, password, dst_user):
        cmd_int = "chage -d `date +%Y-%m-%d` {}; echo last_cmd=$?".format(dst_user)
        ssh_client = ssh.ssh_create_client(host_ip, "root", password)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_int, 1200, 0)
        cmd_result = str(cmd_res).replace('password', '******')
        logger.info("Run install cmd[%s], result: %s" % (cmd_int, str(cmd_result)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    def clear_fsm(self, host_ip, user, password):
        cmd_items = ["if [ -f /opt/fusionstorage/deploymanager/clouda/clear_all_for_fs_node.sh ]",
                     "then echo file_exist=$?",
                     "elif [ -f /opt/fusionstorage/repository/deploymanager_pkg/DSwareSoft/clear_all_for_fs_node.sh ]",
                     "then echo file_exist=$?",
                     "else echo file_exist=1",
                     "fi",
                     "echo last_cmd=$?"]
        cmd_clr = ";".join(cmd_items)
        ssh_client = ssh.ssh_create_client(host_ip, "root", password)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_clr, 62, 0)
        cmd_res = ''.join(cmd_res)
        if cmd_res.find('file_exist=0') == -1:
            logger.info('Fsm not installed, no need to clear.')
            return cmd_res
        ds_path = "/opt/fusionstorage/repository/deploymanager_pkg/DSwareSoft"
        cmd_items = ["if [ -f /opt/fusionstorage/deploymanager/clouda/clear_all_for_fs_node.sh ]",
                     "then bash /opt/fusionstorage/deploymanager/clouda/clear_all_for_fs_node.sh all",
                     "elif [ -f {}/clear_all_for_fs_node.sh ]".format(ds_path),
                     "then bash {}/clear_all_for_fs_node.sh all".format(ds_path),
                     "fi",
                     "echo last_cmd=$?"]
        cmd_clr = ";".join(cmd_items)
        echo_str = "yes/no"
        ssh.ssh_send_command(ssh_client, cmd_clr, echo_str, 62, 0)
        echo_str = ']#'
        cmd_res = ssh.ssh_send_command(ssh_client, 'yes', echo_str, 1200, 0)
        cmd_res = ''.join(cmd_res[-60:])
        logger.info("Run clear cmd[%s], result: %s" % (cmd_clr, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    def clear_fsa(self, host_ip, user, passwd, root_pwd):
        cmd_items = ["if [ -f /opt/fusionstorage/deploymanager/clouda/clear_all_for_fs_node.sh ]",
                     "then echo file_exist=$?",
                     "else echo file_exist=1",
                     "fi",
                     "echo last_cmd=$?"]
        cmd_clr = ';'.join(cmd_items)
        ssh_client = self.create_ssh_root_client(host_ip, user, passwd, root_pwd)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_clr, 62, 0)
        cmd_res = ''.join(cmd_res)
        if cmd_res.find('file_exist=0') == -1:
            logger.info('Fsa not installed, no need to clear.')
            return 0
        cmd_clr = "bash /opt/fusionstorage/deploymanager/clouda/clear_all_for_fs_node.sh all;echo last_cmd=$?"
        echo_str = "yes/no"
        ssh.ssh_send_command(ssh_client, cmd_clr, echo_str, 62, 0)
        echo_str = '#'
        cmd_res = ssh.ssh_send_command(ssh_client, 'yes', echo_str, 1200, 0)
        cmd_res = ''.join(cmd_res[-60:])
        logger.info("Run clear cmd[%s], result: %s" % (cmd_clr, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        if 0 > str(cmd_res).find('last_cmd=0'):
            return 1
        else:
            return 0

    def add_account_settings(self, host_ip, user, passwd, auto_key_b):
        pwd_setting = {
            'fsm_cmdadmin': self.db.get_value_from_cloudparam(self.pod_id, "DistributedStorage", "FSMCmdadminPassword"),
            'fsm_metadata_backup': self.generate_random_character(),
            'db_omm': self.generate_random_character(),
            'db_ommdba': self.generate_random_character(),
            'db_nodemanager': self.generate_random_character(),
            'db_admin': self.generate_random_character(),
            'db_dmdbadmin': self.generate_random_character(),
            'system_dsware': self.db.get_value_from_cloudparam(self.pod_id, "DistributedStorage", "FSdswarePassword"),
            'oam-u_ibc_os_hs': self.generate_random_character(),
            'oam-u_nodemanager': self.generate_random_character(),
            'dm_cli_admin': self.db.get_value_from_cloudparam(self.pod_id, "DistributedStorage", "FSMPortalPassword")
        }
        file_path = "/home/AccountSetting.ini.tmp"
        items = [
            "echo '[fsm]' > %s" % file_path,
            "echo 'cmdadmin=%s' >> %s" % (self.encryp_password(pwd_setting['fsm_cmdadmin'], auto_key_b), file_path),
            "echo 'metadata_backup=%s' >> %s" % (self.encryp_password(pwd_setting['fsm_metadata_backup'], auto_key_b),
                                                 file_path),
            "echo '[database]' >> %s" % file_path,
            "echo 'omm=%s' >> %s" % (self.encryp_password(pwd_setting['db_omm'], auto_key_b), file_path),
            "echo 'ommdba=%s' >> %s" % (self.encryp_password(pwd_setting['db_ommdba'], auto_key_b), file_path),
            "echo 'nodemanager=%s' >> %s" % (self.encryp_password(pwd_setting['db_nodemanager'], auto_key_b),
                                             file_path),
            "echo 'admin=%s' >> %s" % (self.encryp_password(pwd_setting['db_admin'], auto_key_b), file_path),
            "echo 'dmdbadmin=%s' >> %s" % (self.encryp_password(pwd_setting['db_dmdbadmin'], auto_key_b), file_path),
            "echo '[system]' >> %s" % file_path,
            "echo 'dsware=%s' >> %s" % (self.encryp_password(pwd_setting['system_dsware'], auto_key_b), file_path),
            "echo '[oam-u]' >> %s" % file_path,
            "echo 'ibc_os_hs=%s' >> %s" % (self.encryp_password(pwd_setting['oam-u_ibc_os_hs'], auto_key_b), file_path),
            "echo 'nodemanager=%s' >> %s" % (self.encryp_password(pwd_setting['oam-u_nodemanager'], auto_key_b),
                                             file_path),
            "echo '[dm_cli]' >> %s" % file_path,
            "echo 'admin=%s' >> %s" % (self.encryp_password(pwd_setting['dm_cli_admin'], auto_key_b), file_path),
            "chmod 644 %s" % file_path,
            "echo last_result=$?"
        ]
        cmd_rw = "; ".join(items)

        ssh_client = ssh.ssh_create_client(host_ip, user, passwd)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_rw, 60, 3)
        cmd_res = str(cmd_res)
        if cmd_res.find('last_result=0') != -1:
            logger.info("Add account settings success.")
        else:
            logger.error("Add account settings failed.")
            raise Exception("Add account settings failed.")
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    @staticmethod
    def generate_random_character():
        import random
        # 字母类型
        english_char = ['q', 'w', 'e', 'r', 't', 'y', 'u', 'i', 'o', 'p', 'l',
                        'k', 'j', 'h', 'g', 'f', 'd', 's', 'a', 'z', 'x',
                        'c', 'v', 'b', 'n', 'm', 'Q', 'W', 'E', 'R', 'T', 'Y',
                        'U', 'I', 'O', 'P', 'A', 'S', 'D', 'F', 'G', 'H', 'J',
                        'K', 'L', 'Z', 'X', 'C', 'V', 'B', 'N', 'M']
        # 数字类型
        number_char = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0']
        # 符号类型
        symbol_char = ['@', '_', '-']
        all_char = english_char.copy() + number_char.copy() + symbol_char.copy()
        # 重新打乱数组
        random.shuffle(all_char)
        password_length = 16
        password = ''
        # 循环生成密码
        secrets_generator = secrets.SystemRandom()
        for i in range(password_length):
            index = len(all_char) - 1
            password = password + all_char[secrets_generator.randint(0, index)]
        return password

    def disable_ntp_service(self, host_ip, user, passwd, root_pwd):
        cmd_ntp = "/opt/dfv/oam/oam-u/nma/scripts/nma_ctl.sh stopNtpService;systemctl " \
                  "stop ntpd; systemctl disable ntpd;echo result:$?"
        ssh_client = self.create_ssh_root_client(host_ip, user, passwd, root_pwd)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_ntp, 1200, 0)
        logger.info("Run clear cmd[%s], result: %s" % (cmd_ntp, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        if 0 > str(cmd_res).find('result:0'):
            err_msg = 'Stop ntpd failed.'
            logger.error(err_msg)

    def config_and_deploy_role(self, host_id_list, cpu, memory):
        cps_web_client = cpsWebRest(self.project_id, self.pod_id, self.db)
        logger.info("Step1 clone fusionstorage-block role")
        resp_role = cps_web_client.fusionstorage_block_clone()
        logger.info("Finish to clone role fusionstoarge-block. Detail:%s" % resp_role)
        role = resp_role.get('role_name')

        logger.info("Step2 config %s's resource" % role)
        resource_data = self._get_resource_data(role, cpu, memory)
        resp_res = cps_web_client.fusionstorage_block_resource(resource_data)
        logger.info("Finish to config resource for role[%s]. Detail:%s" % (role, resp_res))

        logger.info("Step3 deploy %s for nodes%s" % (role, host_id_list))
        cps_web_client.deploy_hosts_roles(host_id_list, role)
        logger.info("Step4 checking whether the role[%s] is deployed on nodes%s" % (role, host_id_list))
        cps_web_client.check_hosts_role_deploy(host_id_list, role)

    @staticmethod
    def _get_resource_data(role, cpu, memory):
        data = {"role": role,
                "cpu": {"core": cpu},
                "mem": {"size": (memory * 1024)}}
        return data

    @staticmethod
    def check_fusionstorage_agent_rpm(host_list):
        timeout = 300 + 30 * len(host_list)
        timeout = min(timeout, 600)
        sleep_time = 30
        wait_time = 0
        check_hosts = host_list
        while wait_time <= timeout:
            remain_hosts = check_hosts
            for host in check_hosts:
                host_ip = host.get('om_ip')
                logger.info('Checking if the nodes[%s] install the FusionStorage-Agent RPM package' % host_ip)
                if InstallOperate.check_fusionstorage_agent(host_ip, host.get('user'), host.get('passwd'),
                                                            host.get('root_pwd')):
                    logger.info("Find role rpm package on node[%s]" % host_ip)
                    remain_hosts.remove(host)
            check_hosts = remain_hosts
            if len(check_hosts) == 0:
                break
            check_host_ips = [node.get('om_ip') for node in check_hosts]
            logger.info('The rpm is not installed on hosts[%s], check %sth times after %ss'
                        % (check_host_ips, wait_time / sleep_time, sleep_time))
            time.sleep(sleep_time)
            wait_time += sleep_time
        if len(check_hosts) > 0:
            check_host_ips = [node.get('om_ip') for node in check_hosts]
            err_msg = 'The RPM[FusionStorage-Agent-1.0.0-1.noarch] is not found on ' \
                      'some nodes%s after waitting for %s seconds times out'\
                      % (check_host_ips, timeout)
            raise Exception(err_msg)
        else:
            logger.info('The RPM[FusionStorage-Agent-1.0.0-1.noarch] is found on all nodes')
            return

    @staticmethod
    def check_fusionstorage_agent(host_ip, user, passwd, root_pwd):
        cmd_chk = "rpm -qa | grep FusionStorage[_-]Agent;echo result=$?"
        ssh_client = InstallOperate.create_ssh_root_client(host_ip, user, passwd, root_pwd)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_chk, 1200, 0)
        logger.info("Run check cmd[%s], result:%s" % (cmd_chk, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        ret_str = ','.join(cmd_res).replace('\n', '')
        test_ret = re.findall("result=\d+", ret_str)
        test_ret = test_ret[0].split('=')[1]
        test_ret = int(test_ret)
        if test_ret != 0:
            err_msg = "Failed to install role to node[%s], Detail:%s" % (host_ip, str(ret_str))
            logger.error(err_msg)
            return False
        logger.info("The FusionStorage-Agent rpm package is installed on node[%s]" % host_ip)
        return True

    @staticmethod
    def get_role_on_host(cps_client, host_id, target_role='fusionstorage-block'):
        resp_data = cps_client.do_host_role_list(host_id)
        role_data_list = resp_data.get('roles')
        role_list = list()
        for role_data in role_data_list:
            role = role_data.get('name')
            if role.find(target_role) == 0:
                role_list.append(role)
        len_role = len(role_list)
        if len_role > 1:
            err_msg = "A node cannot have multiple fusionstorage-blockXXX roles."
            logger.info(err_msg)
            raise Exception(err_msg)
        elif len_role == 1:
            role_name = role_list[-1]
        else:
            role_name = None
        return role_name

    @staticmethod
    def remove_fusionstorage_block_role(cps_client, hosts_id_list):
        logger.info("Targeting hosts ids%s" % str(hosts_id_list))
        for host_id in hosts_id_list:
            role = InstallOperate.get_role_on_host(cps_client, host_id)
            if not role:
                continue

            resp = cps_client.do_role_host_list(role)
            host_list = resp.get('hosts')
            host_id_list = list(set(host_list).intersection(set(hosts_id_list)))
            logger.info("Removing role[%s] on hosts%s" % (role, host_id_list))
            cps_client.do_role_host_delete(role, host_id_list)
            cps_client.do_commit()
            time.sleep(30)

            resp = cps_client.do_role_host_list(role)
            last_host_list = resp.get('hosts')
            if len(last_host_list) == 0:
                logger.info("Deleting role[%s]" % role)
                cps_client.do_role_delete(role)
                cps_client.do_commit()
        logger.info("Finish to remove role on nodes%s" % hosts_id_list)

    @staticmethod
    def check_fusionstorage_block_on_host(cps_client, host_id_list):
        exist_role_list = list()
        for host_id in host_id_list:
            role = InstallOperate.get_role_on_host(cps_client, host_id)
            if role:
                logger.info("There is exist fusionstorage-blockXXX%s role on host[%s]" % (role, host_id))
                exist_role_list.append({host_id: role})
        if len(exist_role_list) > 0:
            host_list = [host.keys() for host in exist_role_list]
            role_list = [role.values() for role in exist_role_list]
            role_list = list(set(role_list))
            err_msg = "There is exist role fusionstorage-blockXXX%s on host%s, " \
                      "Please remove cmds[step 1.cps role-host-delete --host " \
                      "[host_id1,host_id2...] fusionstorage-blockXXX; " \
                      "step2.cps commit] on one cps node. Host and role detail:%s" \
                      % (role_list, host_list, str(exist_role_list))
            raise Exception(err_msg)
        else:
            logger.info("There is no fusionstorage-blockXXX role on host%s" % host_id_list)

    @staticmethod
    def check_fusionstorage_block_before_install_role(cps_client, host_id_list):
        exist_role_list = list()
        for host_id in host_id_list:
            role = InstallOperate.get_role_on_host(cps_client, host_id)
            if role:
                logger.info("There is exist fusionstorage-blockXXX%s role on host[%s]" % (role, host_id))
                exist_role_list.append({host_id: role})
        if len(exist_role_list) == len(host_id_list):
            logger.info("All hosts has install fusionstorage-blockXXX role, no need install again.")
            return True
        else:
            logger.info("Not all hosts installed fusionstorage-blockXXX role, should install continue." % host_id_list)
            return False

    @staticmethod
    def restart_fusionstorage_block_template(cps_client, host_id_list):
        operated_host_list = list()
        logger.info("Stop and start fusionstorage-blockXXX tempalate on host%s" % host_id_list)
        for host_id in host_id_list:
            if host_id in operated_host_list:
                continue

            role = InstallOperate.get_role_on_host(cps_client, host_id)
            if not role:
                logger.info("There is no fusionstorage-blockXXX role on host[%s]" % host_id)
                continue

            role_data = cps_client.do_role_show(role)
            template_list = role_data.get('template')
            template = template_list[0]
            service_name = template.split('.')[0]
            template_name = template.split('.')[1]

            resp = cps_client.do_role_host_list(role)
            host_list = resp.get('hosts')
            host_list = list(set(host_list).intersection(set(host_id_list)))

            logger.info("Stop role[%s] on hosts%s" % (role, host_list))
            rsp_data = cps_client.do_host_template_instance_operate(service_name, template_name, 'stop', host_list)
            InstallOperate.check_operate_status(rsp_data, host_list, template, 'stop')

            time.sleep(5)
            logger.info("Start role[%s] on hosts%s" % (role, host_list))
            rsp_data = cps_client.do_host_template_instance_operate(service_name, template_name, 'start', host_list)
            InstallOperate.check_operate_status(rsp_data, host_list, template, 'start')

            operated_host_list.extend(host_list)
            logger.info("Stopped and started hosts%s" % operated_host_list)
            logger.info("Remaining hosts%s" % list(set(host_id_list).difference(set(operated_host_list))))
        logger.info("Finish to stop and start hosts%s" % operated_host_list)

    @staticmethod
    def stop_fusionstorage_block_template(cps_client, host_id_list):
        operated_host_list = list()
        logger.info("Stop and start fusionstorage-blockXXX tempalate on host%s" % host_id_list)
        for host_id in host_id_list:
            if host_id in operated_host_list:
                continue

            role = InstallOperate.get_role_on_host(cps_client, host_id)
            if not role:
                logger.info("There is no fusionstorage-blockXXX role on host[%s]" % host_id)
                continue

            role_data = cps_client.do_role_show(role)
            template_list = role_data.get('template')
            template = template_list[0]
            service_name = template.split('.')[0]
            template_name = template.split('.')[1]

            resp = cps_client.do_role_host_list(role)
            host_list = resp.get('hosts')
            host_list = list(set(host_list).intersection(set(host_id_list)))

            logger.info("Stop role[%s] on hosts%s" % (role, host_list))
            rsp_data = cps_client.do_host_template_instance_operate(service_name, template_name, 'stop', host_list)
            InstallOperate.check_operate_status(rsp_data, host_list, template, 'stop')
            time.sleep(5)

            operated_host_list.extend(host_list)
            logger.info("Stopped hosts%s" % operated_host_list)
            logger.info("Remaining hosts%s" % list(set(host_id_list).difference(set(operated_host_list))))
        logger.info("Finish to stop hosts%s" % operated_host_list)

    @staticmethod
    def check_operate_status(rsp_value, host_id_list, template, operate):
        rsp_hosts = rsp_value.get('hosts')
        fail_host_list = list()
        for host_id in host_id_list:
            if rsp_hosts.get(host_id).get(template).get('code') != 200:
                fail_host_list.append(host_id)
        if len(fail_host_list) > 0:
            service_name = template.split('.')[0]
            template_name = template.split('.')[1]
            err_msg = "Failed to %s FusionStorage-Agent on host%s. Please try it on one of openstack node by cmd" \
                      "[cps host-template-instance-operate --service %s %s --action %s --host %s]," \
                      " Detail:%s" % (operate, fail_host_list, service_name,
                                      template_name, operate, fail_host_list, rsp_value)
            logger.error(err_msg)
            raise Exception(err_msg)
        else:
            logger.info("Succeed start to %s FusionStorage-Agent. Detail:%s" % (operate, rsp_value))

    def config_encryption_license_switch(self, pod_id, float_ip, check_encryption=True):
        user_name = DeployConstant.FSM_USER
        passwd = self.db.get_value_from_cloudparam(pod_id, "DistributedStorage", "FSMfsdminPassword")
        root_pwd = self.db.get_value_from_cloudparam(pod_id, "DistributedStorage", "FSMrootPassword")
        ssh_client = None
        try:
            ssh_client = ssh.ssh_create_client(float_ip, user_name, passwd)
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("Exec cmd[su root] result:%s" % (str(cmd_res)))
            ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            # 检查文件中是否存在配置项
            cmd_items = ["if grep 'CheckEncryptLicense' /opt/dfv/oam/oam-s/manager/apache_tomcat/webapps/ROOT/WEB-INF/"
                         "classes/serviceLayerBackend.properties",
                         "then echo item_exist=$?",
                         "else echo item_exist=1",
                         "fi",
                         "echo last_cmd=$?"]
            cmd_clr = ";".join(cmd_items)
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_clr, 20)
            cmd_res = ''.join(cmd_res)
            if cmd_res.find('item_exist=0') == -1:
                err_msg = 'Check encryption item does not exist in license.'
                logger.error(err_msg)
                raise Exception(err_msg)
            if check_encryption:
                cmd_config = 'sed -i "s/.*CheckEncryptLicense.*$/CheckEncryptLicense=true/g" /opt/dfv/oam/oam-s/' \
                             'manager/apache_tomcat/webapps/ROOT/WEB-INF/classes/serviceLayerBackend.properties; ' \
                             'echo last_code=$?'
                cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_config, 20)
                logger.info("Exec cmd switch license result: %s" % cmd_res)
            else:
                cmd_config = 'sed -i "s/.*CheckEncryptLicense.*$/CheckEncryptLicense=false/g" /opt/dfv/oam/oam-s/' \
                             'manager/apache_tomcat/webapps/ROOT/WEB-INF/classes/serviceLayerBackend.properties; ' \
                             'echo last_code=$?'
                cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_config, 20)
                logger.info("Exec cmd switch license result: %s" % cmd_res)
        except Exception as e:
            err_msg = "Failed to config license, details: %s" % str(e)
            logger.error(err_msg)
            raise Exception(e)
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)

    def restart_oams(self, pod_id, float_ip):
        user_name = DeployConstant.FSM_USER
        passwd = self.db.get_value_from_cloudparam(pod_id, "DistributedStorage", "FSMfsdminPassword")
        root_pwd = self.db.get_value_from_cloudparam(pod_id, "DistributedStorage", "FSMrootPassword")
        ssh_client = None
        try:
            ssh_client = ssh.ssh_create_client(float_ip, user_name, passwd)
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("Exec cmd[su root] result:%s" % (str(cmd_res)))
            ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            cmd_restart = 'sh /opt/dfv/oam/oam-s/manager/action/restart.sh; echo last_cmd=$?'
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_restart, 1200, 0)
            logger.info("Exec restart oam-s cmd[%s] result: %s" % (cmd_restart, cmd_res))
        except Exception as e:
            err_msg = "Failed to restart oam-s, details: %s" % str(e)
            logger.error(err_msg)
            raise Exception(e)
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)

    def config_create_pool_license_switch(self, pod_id, float_ip, delete=False):
        # 配置创建第一个存储池的参数：支持添加和删除，delete=True代表删除功能，默认False代表添加配置
        vm_operate = VMOperate(self.project_id, self.pod_id, None)
        vm_master, vm_standby = vm_operate.get_vm_data(pod_id, float_ip)
        host_ip = vm_master.get('om_ip')
        user_name = vm_master.get('user')
        passwd = vm_master.get('passwd')
        root_pwd = vm_master.get('root_pwd')
        ssh_client = None
        try:
            ssh_client = ssh.ssh_create_client(host_ip, user_name, passwd)
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("Exec cmd[su root] result:%s" % (str(cmd_res)))
            ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            # 无论添加还是删除，先删除一次license.switch配置
            cmd_delete_config = 'su - omm -s /bin/bash -c "sed -i \'/^license.first.pool.switch/d\' /opt/omm/oms/' \
                                'workspace/webapps/dsware/WEB-INF/manager.properties; echo last_code=$?"'
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_delete_config, 20)
            logger.info("Exec cmd delete switch license result: %s" % cmd_res)
            # 如果是删除逻辑，此处就返回
            if delete:
                return
            cmd_config = 'su - omm -s /bin/bash -c \'echo "license.first.pool.switch=close" >> /opt/omm/oms/' \
                         'workspace/webapps/dsware/WEB-INF/manager.properties; echo last_code=$?\''
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_config, 20)
            logger.info("Exec cmd switch license result: %s" % cmd_res)
        except Exception as e:
            msg = 'Failed to config license, details: %s' % str(e)
            logger.error(msg)
            raise Exception(e)
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)

    def config_disk_type_switch(self, support=False):
        logger.info('Turn on the switch to support cache disks from different manufacturers')
        if not self.fs_args.get('open_disk_vendor_switch'):
            return
        float_ip = self.fs_args.get('float_ip')
        fsadmin_pwd = self.fs_args.get('fsadmin_pwd')
        root_pwd = self.fs_args.get('fsm_root_pwd')
        ssh_client = None
        file_name = '/opt/omm/oms/workspace/webapps/dsware/WEB-INF/manager.properties'
        pub_cmd = 'su - omm -s /bin/bash -c \'sed -i "s/^disk.vendor.in.pool.check.switch=.' \
                  '*/disk.vendor.in.pool.check.switch=%s/g" %s;echo last_code=$?\''
        try:
            ssh_client = self.create_ssh_root_client(float_ip, 'fsadmin', fsadmin_pwd, root_pwd)
            if support:
                close_cmd = pub_cmd % ('close', file_name)
                logger.info("Run the command [%s] to turn on the switch to support different manufacturers" % close_cmd)
                cmd_res = ssh.ssh_exec_command_return(ssh_client, close_cmd, 20)
                logger.info("Result: %s" % cmd_res)
            else:
                open_cmd = pub_cmd % ('open', file_name)
                logger.info("Execute the command [%s] to turn off the switch" % open_cmd)
                cmd_res = ssh.ssh_exec_command_return(ssh_client, open_cmd, 20)
                logger.info("Result: %s" % cmd_res)
            ssh.ssh_close(ssh_client)
        except Exception as e:
            msg = 'Failed to config disk vendor switch, details: %s' % str(e)
            logger.error(msg)
            raise FCDException(626003, msg)
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)

    @staticmethod
    def config_license_switch(host_ip, user_name, passwd, root_pwd, delete=False):
        # 配置创建第一个存储池的参数：支持添加和删除，delete=True代表删除功能，默认False代表添加配置
        ssh_client = None
        try:
            ssh_client = ssh.ssh_create_client(host_ip, user_name, passwd)
            cmd_res = ssh.ssh_send_command(ssh_client, 'su -', 'assword:', 20)
            logger.info("Exec cmd[su root] result:%s" % (str(cmd_res)))
            ssh.ssh_send_command(ssh_client, root_pwd, '#', 20)
            # 无论添加还是删除，先删除一次license.switch配置
            cmd_delete_config = 'su - omm -s /bin/bash -c \'sed -i "/^license.first.pool.switch/d" /opt/omm/oms/' \
                                'workspace/webapps/dsware/WEB-INF/manager.properties; echo last_code=$?\''
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_delete_config, 20)
            logger.info("Exec cmd delete switch license result: %s" % cmd_res)
            # 如果是删除逻辑，此处就返回
            if delete:
                return
            cmd_config = 'su - omm -s /bin/bash -c \'echo "license.first.pool.switch=close" >> /opt/omm/oms/' \
                         'workspace/webapps/dsware/WEB-INF/manager.properties; echo last_code=$?\''
            cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_config, 20)
            logger.info("Exec cmd switch license result: %s" % cmd_res)
        except Exception as e:
            msg = 'Failed to config license, details: %s' % str(e)
            logger.error(msg)
            raise Exception(e)
        finally:
            if ssh_client:
                ssh.ssh_close(ssh_client)

    @staticmethod
    def check_user_accessible(host_ip, user, password, timeout=900):
        accessible_flag = False
        exp_err = None
        runtime = 0
        while runtime < timeout:
            runtime += 60
            time.sleep(60)
            try:
                cmd_id = 'id fsadmin; echo code=$?'
                ssh_client = ssh.ssh_create_client(host_ip, user, password)
                cmd_ret = ssh.ssh_send_command(ssh_client, cmd_id, 'code=', 10)
                cmd_ret = str(cmd_ret)
                logger.info("Run cmd[%s], result: %s" % (cmd_id, cmd_ret))
                if 0 <= cmd_ret.find('code=0'):
                    logger.info("Succeed to login host[%s]" % host_ip)
                    accessible_flag = True
                    ssh.ssh_close(ssh_client)
                    break
            except Exception as e:
                exp_err = str(e)
                err_msg = "Failed to connect host[%s] by user[%s], Detail:%s, Runtime:%s, Try again after %ss" \
                          % (host_ip, user, exp_err, runtime, 60)
                logger.error(err_msg)
        if not accessible_flag:
            err_msg = "Failed to connect host[%s] by user[%s] after %ss seconds of time-out wait, Detail:%s" \
                      % (host_ip, user, timeout, exp_err)
            raise Exception(err_msg)
        else:
            logger.info("Succeed to connect host[%s] by user[%s]" % (host_ip, user))
        return True

    @staticmethod
    def get_cabinet_no(node, cablinet_no, prefix_string, cablinet_list):
        """
        用户填写机柜场景： 检查机柜编号格式和机柜分组是否满足要求
        默认生成机柜场景： 每24个节点分为一组，storage_group1， storage_group2, compute_group1, compute_group2 ...
        """
        max_server_one_cabinet = 24
        rack_start_num = 1
        if cablinet_no:
            if re.search('[^\.a-zA-Z0-9_-]', cablinet_no):
                logger.info("The cabinet name[%s] is not standard." % cablinet_no)
                raise FCDException(626113, node)
            if cablinet_list.count(cablinet_no) > max_server_one_cabinet:
                logger.info("More than 24 nodes use the same cabinet name[%s]." % cablinet_no)
                raise FCDException(626114, cablinet_no)
        else:
            node_num = len(cablinet_list)
            suffix_num = rack_start_num + int(node_num / max_server_one_cabinet)
            cablinet_no = "{}{}".format(prefix_string, suffix_num)
        cablinet_list.append(cablinet_no)
        return cablinet_no

    def modify_resolv_file(self, host_ip, user, password):
        # 注释/etc/resolv.conf文件
        cmd_items = [
            "sed -i 's/^#*/#/g' /etc/resolv.conf",
            "echo last_cmd=$?"
        ]
        cmd_clr = "; ".join(cmd_items)
        ssh_client = ssh.ssh_create_client(host_ip, "root", password)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_clr, 1200, 0)
        logger.info("Run cmd[%s], result: %s" % (cmd_clr, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    def unmodify_resolv_file(self, host_ip, user, password):
        # 取消/etc/resolv.conf文件注释
        cmd_items = [
            "sed -i 's/^#*//g' /etc/resolv.conf",
            "echo last_cmd=$?"
        ]
        cmd_clr = "; ".join(cmd_items)
        ssh_client = ssh.ssh_create_client(host_ip, "root", password)
        cmd_res = ssh.ssh_exec_command_return(ssh_client, cmd_clr, 1200, 0)
        logger.info("Run cmd[%s], result: %s" % (cmd_clr, str(cmd_res)))
        if ssh_client:
            ssh.ssh_close(ssh_client)
        return cmd_res

    def check_third_party_server_disk(self, disk_info, node_list):
        """
        针对戴尔服务器，检查是否安装了扫盘工具perccli
        无论主存槽位的磁盘是什么类型，如果没有安装戴尔的扫盘工具，通过storcli都无法识别到主存槽位对应的磁盘类型

        disk_info: class
        node_list: list
        return:
        """
        osd_ip_list = [osd_info.get('om_ip') for osd_info in node_list]
        osd_ip_slot_list = [(osd_info.get('om_ip'), osd_info.get('primary_slot')) for osd_info in node_list]

        res = disk_info.get_query_data()
        if not res.get('disks'):
            err_msg = 'Failed to query cluster information. Detail:[{}]'.format(res)
            logger.error(err_msg)
            raise Exception(err_msg)

        check_boolean, passed_list = self.check_pool_disk_method(res, osd_ip_slot_list)
        if not check_boolean:
            failed_list = list(set(osd_ip_list).difference(set(passed_list)))
            failed_ip_str = ','.join(failed_list)
            err_msg = 'The number of main storage disks is 0: [{}]'.format(failed_ip_str)
            logger.error(err_msg)
            raise FCDException(626322, err_msg, failed_ip_str, failed_ip_str)

    def check_pool_disk_method(self, res, osd_ip_slot_list):
        """
        只要主存槽位有盘类型信息，说明非戴尔服务器或安装了扫盘工具perccli，检查通过
        """
        pass_osd_node = list()
        for om_ip, slot_info in osd_ip_slot_list:
            all_disk_info = res.get('disks').get(om_ip)
            logger.info("All disk on node[%s]: %s" % (om_ip, all_disk_info))
            start_slot, end_slot = slot_info.split('-')
            slot = int(start_slot)
            while slot <= int(end_slot):
                current_disk_type = self.get_current_disk_type(all_disk_info, slot)
                if current_disk_type:
                    pass_osd_node.append(om_ip)
                    break
                slot += 1
        if len(pass_osd_node) == len(osd_ip_slot_list):
            return True, ''
        return False, pass_osd_node

    @staticmethod
    def get_current_disk_type(all_disk_info, slot):
        current_disk_type = None
        for disk_info in all_disk_info:
            if disk_info.get('devSlot') == slot:
                current_disk_type = disk_info.get('devType')
                break
        return current_disk_type

    def get_az_id(self, service_name):
        """
        返回业务存储所在az id
        :return:
        部署首region，从region时，Type2,Type3:openstack_region，Type1：cascaded_region
        扩AZ时，新建的AZ ID：Type2/3：openstack_expansion_az，Type1：cascaded_expansion_azk
        """
        az_id = self.param_util.get_param_value(self.pod_id, service_name, 'openstack_expansion_az')
        if not az_id:
            logger.info("get az_id from openstack_region")
            az_id = self.param_util.get_param_value(self.pod_id, service_name, 'openstack_region')
        return az_id

    def parallel_run(self, func_name, nodes, timeout=120,
                     parallel_threads=20, **kwargs):
        offset = 0
        total_ip = len(nodes)
        all_ssh_threads = {}
        nodes_ip = [node.get('om_ip') for node in nodes]
        while True:
            this_round_node_list = nodes[offset: offset + parallel_threads]
            this_round_ip_list = [node.get('om_ip') for node in this_round_node_list]
            logger.info(
                'Starting new threads for %s' % ','.join(this_round_ip_list))
            logger.info('Remaining ips: %s' % ','.join(
                nodes_ip[offset + parallel_threads:]))
            for node in this_round_node_list:
                os_ip = node.get('om_ip')
                kwargs.update(node)
                _t = threading.Thread(target=func_name, kwargs=kwargs)
                _t.start()
                all_ssh_threads[os_ip] = _t
            time.sleep(1)
            start_time = time.time()
            while start_time + timeout > time.time():
                if not all_ssh_threads:
                    break
                for os_ip in this_round_ip_list:
                    self.check_threading_is_alive(all_ssh_threads, os_ip)
            offset = offset + parallel_threads
            if offset >= total_ip:
                break

    @staticmethod
    def check_threading_is_alive(all_ssh_threads, os_ip):
        if all_ssh_threads.get(os_ip) is None:
            return
        if 'is_alive' not in dir(all_ssh_threads.get(os_ip)):
            logger.info('Threading of %s is not starting.' % os_ip)
            return
        if all_ssh_threads.get(os_ip).is_alive():
            time.sleep(1)
            if int(time.time()) // 5 == 0:
                logger.info('Threading is not finished.')
        else:
            logger.info('Threading of %s is finished.' % os_ip)
            all_ssh_threads.pop(os_ip)

    def upload_cert_and_get_private_key(self, webui_client):
        logger.info('Upload certificates')
        cert_file_dir = "/home/pkg/distribute_storage_cert_file"
        if os.path.exists(cert_file_dir):
            try:
                subprocess.getstatusoutput("rm -rf %s" % cert_file_dir)
            except Exception as e:
                logger.error("Failed to delete the folder(%s). Detail:%s" % (cert_file_dir, e))
                return Message(500, FCDException(626336))
        subprocess.getstatusoutput("mkdir -m 750 -p %s" % cert_file_dir)
        storage_tool = DistributedStorageTool(self.project_id, self.pod_id, self.db)
        fsc_cli_cert = storage_tool.get_fsc_cli_cert_file(cert_file_dir, self.fs_args.get('float_ip'))
        rp = ResourcePool(webui_client, self.project_id)
        # 调用cps-web上传证书文件
        for key, value in fsc_cli_cert.items():
            if key != "password":
                file_name = value.split("/")[-1]
                tiny_data = '{"file":"%s","module":"HWDBS_KVM"}' % file_name
                body = {'tinyFormDatas': tiny_data}
                files = {"tiFile": (file_name, open(value, "rb"))}
                rp.upload_certificates(body, files)
        logger.info('Get private key')
        private_key_psw = fsc_cli_cert.get("password")
        return private_key_psw

    @staticmethod
    def check_es3000_driver(ssh_client, cpu_arch, driver_list, role='business'):
        if not driver_list:
            return
        kernel_dic = DeployConstant.KERNEL_DIC.get(cpu_arch)
        es3000_driver_list = DeployConstant.NVME_NAME.get(cpu_arch)
        if not es3000_driver_list or not kernel_dic:
            err_msg = 'Failed to get kernel or es3000_driver_list by {}'.format(cpu_arch)
            logger.error(err_msg)
            raise FCDException(626109, err_msg)
        logger.info('es3000 pkg list:{}'.format(es3000_driver_list))
        logger.info('The cpu arch:{}'.format(cpu_arch))
        logger.info('driver list:{}'.format(driver_list))
        kernel_pattern = kernel_dic.get(role)
        if es3000_driver_list[0] not in driver_list:
            return driver_list
        cmd = 'uname -r'
        res = ssh.ssh_exec_command_return(ssh_client, cmd)
        logger.info("cmd: %s, result: %s" % (cmd, str(res)))
        if re.search(kernel_pattern, str(res)):
            driver_list = list(set(driver_list).difference(set(es3000_driver_list)))
        logger.info('Res:{}'.format(driver_list))
        return driver_list


class InstallDriverOperate(object):
    def __init__(self):
        pass

    @staticmethod
    def sort_drivers(driver_infos, all_drivers):
        """
        以驱动为依据，对驱动列表进行分批次
        return: [(['nvme'], ['192.168.1.201', '192.168.1.202']), (['cx5'], ['192.168.1.201']), ...]
        """
        driver_list = list()
        for driver in set(all_drivers):
            ip_list = []
            for node in driver_infos:
                if driver in driver_infos.get(node):
                    ip_list.append(node)
            if ip_list:
                data = ([driver], ip_list)
                driver_list.append(data)
        return driver_list

    @staticmethod
    def group_by_account_info(host, need_install_driver_hosts, driver_list):
        """
        按相同账户信息的节点分组

        need_install_driver_hosts:
        {'user1pwdpwd': ['user1', 'pwd', 'pwd1', {'192.168.1.201': ['cx4'], ''192.168.1.202': ['cx4', 'cx5']}],...}
        """
        om_ip, user, passwd, root_pwd = host.get('om_ip'), host.get('user'), host.get('passwd'), host.get('root_pwd')
        account_str = user + passwd + root_pwd
        if account_str in need_install_driver_hosts.keys():
            need_install_driver_hosts[account_str][-1][om_ip] = driver_list
        else:
            need_install_driver_hosts[account_str] = [user, passwd, root_pwd, {om_ip: driver_list}]
