# -*- coding: utf-8 -*-
# Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved.
import time
import os
import re
import subprocess
from tenacity import retry, stop_after_attempt, wait_fixed

import utils.common.log as logger
from utils.DBAdapter.DBConnector import BaseOps
from utils.client.FSPAuthClient import FSPAuthClient
from utils.client.webUI_client import g_webui_client_ctrl
from utils.common.exception import PubException
from utils.common.error.check_resp import check_stop_fsm_rep
from utils.common.error.check_resp import check_delete_fsm_rep
from utils.common.error.check_resp import check_create_fsm_rsp
from utils.business.fusionstorage_util import update_soft_install_info_batch
from utils.business.fusionstorage_util import check_soft_install_info
from utils.business.fusionstorage_util import add_soft_install_info
from utils.common.exception import HCCIException
from utils.common.public_dir_util import PublicDirectoryManager
from plugins.DistributedStorage.utils.common.deploy_constant import DeployConstant
from plugins.DistributedStorage.utils.interface.DistributedStorage import DistributedStorageTool


class VMOperate(object):
    def __init__(self, project_id, pod_id, fs_args):
        self.project_id = project_id
        self.pod_id = pod_id
        self.fs_args = fs_args
        self.db = BaseOps()
        self.dsware_server = None
        self.client = None

    @staticmethod
    def _change_ha_mode_string(evn_str):
        map_dict = {"double": "Active-Standby", "single": "Single"}
        return map_dict.get(evn_str, "Active-Standby")

    def get_version_from_uvp(self, package_path):
        uvp_package = os.path.basename(package_path)
        pkg_name = uvp_package.strip('.tar.gz')
        tmp_dir = PublicDirectoryManager().get_temp_file_path(self.project_id, 'DistributedStorage')
        cmd_items = ["tar -zxf '%s' -C %s %s/FusionStorage_Manager*"
                     % (package_path, tmp_dir, pkg_name),
                     "cd %s/%s" % (tmp_dir, pkg_name),
                     "tar -zxf FusionStorage_Manager*.tar.gz FusionStorage_Manager*/package.sdf",
                     "cat FusionStorage_Manager*/package.sdf | grep 'package_version' | awk -F: '{print $2}'",
                     "rm -rf %s/%s" % (tmp_dir, pkg_name)]
        cmd_query = ";".join(cmd_items)
        (status, output) = subprocess.getstatusoutput(cmd_query)
        output = output.decode('utf-8') if isinstance(output, bytes) else output
        status = status.decode('utf-8') if isinstance(status, bytes) else status
        logger.info('Run cmd[%s], status:%s, output:%s' % (cmd_query, status, output))
        uvp_version_list = re.findall('\"(FusionStorage[_-]Manager[_-]\d[.a-zA-Z0-9]*)\"', output)
        logger.info("Find result:%s" % uvp_version_list)
        if 1 == len(uvp_version_list):
            uvp_version = uvp_version_list[0]
        else:
            logger.error("find FusionStorage_Manager version from %s failed"
                         % package_path)
            uvp_version = "FusionStorage_Manager_8.x"
        return uvp_version

    def process_before_create_vm(self, task_id, subtask_id, float_ip):
        logger.info("Query the db for failure information in %s, %s."
                    % (task_id, subtask_id))
        # 根据数据库失败信息回滚失败的虚拟机
        self.delete_created_vm(task_id, subtask_id, float_ip)
        logger.info("The installation is ready and the installation is about to begin...")

    def check_float_ip_used(self, float_ip):
        # 查询浮动IP使用已有虚拟机在使用
        webui_host_and_tool = FSPAuthClient.get_cps_web_host_and_tool(self.db, self.project_id, self.pod_id)
        g_webui_client_ctrl.client_list = list()
        vm_data = self.query_dsware_vm_with_ip(webui_host_and_tool, float_ip)
        if len(vm_data.keys()) > 0:
            err_msg = "VMs using the float IP[%s] address exist on the CPS." \
                      " vm detail:%s" % (float_ip, vm_data)
            logger.error(err_msg)
            raise HCCIException(626318, float_ip, vm_data)

    def delete_created_vm(self, task_id, subtask_id, float_ip):
        logger.info("Query the virtual machine from fail db.")
        dsware_vm = list()
        fail_vm_data = dict()
        # 从数据库中查询当前工步状态,vm_dsware_list保存dsware_name,floatIP
        # 成功状态为2，失败或其它状态为1
        get_fsm_install_info = check_soft_install_info(self.project_id, task_id, subtask_id)
        if len(get_fsm_install_info) == 1 and str(get_fsm_install_info[0].status) == "1":
            dsware_vm = get_fsm_install_info[0].failed_msg.split(",")
            webui_host_and_tool = FSPAuthClient.get_cps_web_host_and_tool(self.db, self.project_id, self.pod_id)
            g_webui_client_ctrl.client_list = list()
            # 工步创建虚拟机失败后回滚失败且修改了LLD浮动IP时，先查失败虚拟机是否存在，如不存在直接返回
            fail_vm_data = self.query_dsware_vm_with_ip(webui_host_and_tool, dsware_vm[1])
            if not fail_vm_data:
                err_msg = "The VM that fails to be created based on " \
                          "the float IP[%s] address does not exist." \
                          % (dsware_vm)
                logger.info(err_msg)
                # 更新数据库
                datas = list()
                data_info = dict()
                data_info["project_id"] = self.project_id
                data_info["task_id_str"] = task_id
                data_info["subtask_id_str"] = subtask_id
                data_info["status"] = "0"
                data_info["failed_msg"] = get_fsm_install_info[0].failed_msg
                datas.append(data_info)
                info_log = 'The vm[%s] has been deleted' % data_info
                logger.info(info_log)
                update_soft_install_info_batch(datas=datas)
                return

        logger.info('The failed vm at db: %s.' % dsware_vm)
        logger.info('The actual vm on cps: %s.' % fail_vm_data)
        if dsware_vm and fail_vm_data:
            self.before_create_vm(task_id, subtask_id, dsware_vm, fail_vm_data)

            webui_host_and_tool = FSPAuthClient.get_cps_web_host_and_tool(self.db, self.project_id, self.pod_id)
            g_webui_client_ctrl.client_list = list()
            self.client = g_webui_client_ctrl.apply_client(webui_host_and_tool, self)
            vm_dsware_infos = self.query_dsware_vm_with_ip(webui_host_and_tool, dsware_vm[1])
            if vm_dsware_infos:
                err_msg = "Fail to delete vm, the float ip %s is same with vm: %s " \
                          % (float_ip, vm_dsware_infos)
                logger.error(err_msg)
                raise Exception(err_msg)
        else:
            logger.info('No virtual machine found. Skip..')

    def before_create_vm(self, task_id_str, subtask_id_str, dsware_vm, fail_vm_data):
        """
        dsware_vm_list记录安装失败的fsm信息，从数据库读取
        dsware_vm_list[0]记录虚拟机名称，用于删除虚拟机
        dsware_vm_list[1]记录浮动IP，用于重试的时候匹配ID去暂停虚拟机
        fail_vm_data 根据db记录的浮动IP查出来的虚拟机
        """
        instanceid_list = list()
        need_stop_id_list = list()
        dsware_server_num = None
        if len(fail_vm_data.keys()) > 1:
            err_msg = "Multiple groups of VMs exist in the CPS, " \
                      "and the float IP is %s." % dsware_vm[1]
            logger.info(err_msg)
            raise HCCIException(626319, fail_vm_data)
        for num, vms in fail_vm_data.items():
            # 根据浮动IP查出来的fail_vm_data，检查dsware_serverX序列号是否一致
            if num != dsware_vm[0]:
                err_msg = "The data[%s] of the VM to be deleted are" \
                           " inconsistent with the actual data[%s] " \
                           "of the VM on the CPS." \
                           % (dsware_vm, vms)
                logger.info(err_msg)
                raise HCCIException(626320, dsware_vm[1], dsware_vm[0], dsware_vm[1], num)
            for item in vms:
                if item.get('external_om_float_ip') != dsware_vm[1]:
                    continue
                instanceid_list.append(item.get('id'))
                if item.get("status") == "Active":
                    need_stop_id_list.append(item.get('id'))
            dsware_server_num = dsware_vm[0]
        if instanceid_list and dsware_server_num:
            delete_vm_params = {'task_id_str': task_id_str,
                                'subtask_id_str': subtask_id_str,
                                'dsware_server_num': dsware_server_num}
            self.delete_vm(delete_vm_params, instanceid_list, need_stop_id_list)
        return True

    def query_dsware_vm_with_ip(self, webui_host_and_tool, float_ip):
        self.client = g_webui_client_ctrl.apply_client(webui_host_and_tool, self)
        server_state = dict()
        # 数据库中保存的浮动IP，先查询虚拟机信息，根据浮动IP匹配虚拟机ID
        rsp_status, rsp_header, rsp_body = self.query_dsware_vm()
        for key, value in rsp_body.items():
            if 'dsware-server' not in key:
                continue
            for item in value:
                if item.get('external_om_float_ip') == float_ip:
                    server_state[key] = value
        return server_state

    def delete_vm(self, delete_vm_params, instanceid_list, need_stop_id_list):
        task_id_str = delete_vm_params.get('task_id_str')
        subtask_id_str = delete_vm_params.get('subtask_id_str')
        dsware_server_num = delete_vm_params.get('dsware_server_num')
        logger.info("Start to delete the Virtual machine [%s:%s]."
                    % (dsware_server_num, instanceid_list))
        logger.info('stop dsware vm %s' % instanceid_list)
        # 删除虚拟机前根据虚拟机ID下电虚拟机
        self.stop_dsware_vm(need_stop_id_list)

        # 查询虚拟机是否下电成功，下电成功后才能删除
        del_vm_shutoff = self.check_dsware_vm_shutoff(dsware_server_num)
        if not del_vm_shutoff:
            error_msg = "There are VMs[%s] that have not been powered off. " \
                        "Login to CPS and manually delete them." \
                        % dsware_server_num
            logger.error(error_msg)
            raise Exception(error_msg)

        # 通过dsware_serverX序号删除虚拟机
        logger.info('delete dsware vm %s' % dsware_server_num)
        self.delete_dsware_vm(dsware_server_num)
        # 删除流程完成之后查询虚拟机是否删除成功
        del_status, del_header, del_body = self.query_dsware_vm()
        logger.info("del_status:%s, del_header:%s" % (del_status, del_header))
        for key in del_body.keys():
            if str(dsware_server_num) == str(key):
                error_msg = 'Delete dsware vm[%s:%s] failed.' % (dsware_server_num, instanceid_list)
                logger.error(error_msg)
                raise Exception(error_msg)
        # 构造数据库信息
        datas = list()
        data_info = dict()
        data_info["project_id"] = self.project_id
        data_info["task_id_str"] = task_id_str
        data_info["subtask_id_str"] = subtask_id_str
        data_info["status"] = "0"
        data_info["failed_msg"] = "delete vms success"
        datas.append(data_info)
        info_log = 'Delete fsm %s succeeded, ready to reinstall.' % data_info
        logger.info(info_log)
        # 更新数据
        update_soft_install_info_batch(datas=datas)
        logger.info("Delete Virtual machines[%s:%s] succeeded."
                    % (dsware_server_num, instanceid_list))
        return True

    def check_dsware_vm_shutoff(self, dsware_vm_num):
        try_times = 6
        del_vm_shutoff = True
        while try_times > 0:
            vm_data = self.query_dsware_vm_by_num(dsware_vm_num)
            for vm in vm_data.get(dsware_vm_num):
                if vm.get('status') == 'Active':
                    del_vm_shutoff = False
            if del_vm_shutoff:
                break
            try_times -= 1
            if try_times > 0:
                del_vm_shutoff = True
                msg_info = "There are VM[%s] that are not stopped. " \
                           "try to check again[%s] after 60s" % (vm_data, try_times)
                logger.info(msg_info)
                time.sleep(60)
        return del_vm_shutoff

    def query_dsware_vm_by_num(self, dsware_vm_num):
        vm_data = dict()
        vm_status, vm_header, vm_body = self.query_dsware_vm()
        logger.info("vm_status:%s, vm_header:%s" % (vm_status, vm_header))
        for key, value in vm_body.items():
            if str(dsware_vm_num) == str(key):
                vm_data[key] = value
        return vm_data

    def query_dsware_vm(self):
        """查询虚拟机"""
        uri = "api/v1/cloud_service/fusion_storage"
        method = "GET"
        body = None
        cmd = {"method": method, "uri": uri, "body": body}
        rsp_status, rsp_header, rsp_body = self.client.exec_cmd(cmd)
        if int(rsp_status) < 200 or int(rsp_status) > 299:
            raise Exception('Failed to query virtual machine information.')
        return rsp_status, rsp_header, rsp_body

    def get_vm_data_with_ip(self, host_and_tool, float_ip):
        vm_data = self.query_dsware_vm_with_ip(host_and_tool, float_ip)
        if len(vm_data.keys()) > 1:
            err_msg = "Multiple groups of VMs exist in the CPS, " \
                      "and the float IP is %s." % float_ip
            logger.error(err_msg)
            raise HCCIException(err_msg)
        for _key, value in vm_data.items():
            return value, True
        return "", False

    def get_vm_data(self, pod_id, float_ip):
        # 获取FSM虚拟机所在的节点,回滚等操作需要通过float_ip确认是否操作的是目标vm
        fsm_fsadmin_password, fsm_root_password = self.get_fsm_pwd(float_ip)
        webui_host_and_tool = FSPAuthClient.get_cps_web_host_and_tool(self.db, self.project_id, pod_id)
        vm_data, status = self.get_vm_data_with_ip(webui_host_and_tool, float_ip)
        vm_nodes = list()
        if status:
            for vm in vm_data:
                vm_node = {'om_ip': vm.get('external_om_ip'),
                           'hostname': vm.get('name'),
                           'root_pwd': fsm_root_password,
                           'user': DeployConstant.FSM_USER,
                           'passwd': fsm_fsadmin_password,
                           'om_float_ip': vm.get('external_om_float_ip'),
                           'id': vm.get("id")}
                vm_nodes.append(vm_node)
        else:
            err_msg = "Failed to query VM info[%s]" % vm_data
            logger.error(err_msg)
            raise Exception(err_msg)
        if len(vm_nodes) < 2:
            err_msg = "Failed to query VM info[%s]" % vm_data
            logger.error(err_msg)
            raise Exception(err_msg)
        return vm_nodes[0], vm_nodes[1]

    def get_fsm_pwd(self, float_ip):
        fsm_fsadmin_password = ""
        fsm_root_password = ""
        storage_float_ip_list = DistributedStorageTool(self.project_id, self.pod_id, self.db).get_storage_float_ip()
        for cur_float_ip_info in storage_float_ip_list:
            if cur_float_ip_info.get("float_ip") == float_ip:
                logger.info("The floating IP:{}, and the corresponding information is found".format(float_ip))
                fsm_fsadmin_password = cur_float_ip_info.get("fsadmin_pwd")
                fsm_root_password = cur_float_ip_info.get("root_pwd")
                break
        if not fsm_fsadmin_password or not fsm_root_password:
            err_msg = "The floating IP:{}. Failed to query authentication information".format(float_ip)
            logger.error(err_msg)
            raise Exception(err_msg)
        return fsm_fsadmin_password, fsm_root_password

    def stop_dsware_vm(self, instanceid_list):
        """暂停虚拟机，三次重试"""
        logger.info('Start to stop the Virtual machine.')
        for instanceid in instanceid_list:
            for _ in range(1, 4):
                uri = "api/v1/cloud_service/managevm/operate"
                method = "POST"
                body = {
                    "action": "stop",
                    "instanceid": instanceid
                }
                logger.info(body)
                cmd = {"method": method, "uri": uri, "body": body}
                rsp_status, rsp_header, rsp_body = self.client.exec_cmd(cmd)
                try:
                    # 虚拟机暂停结果分析处理，暂停成功进入删除流程，失败直接抛出异常
                    check_stop_fsm_rep(rsp_status, rsp_body)
                except Exception as e:
                    error_log = 'Stop an existing FSM failure, try again after 10s. Detail: %s' % str(e)
                    logger.error(error_log)
                    time.sleep(10)
                info_log = 'Stop fsm successfully, ready to delete.'
                logger.info(info_log)
                break
            else:
                error_log = 'Stop an existing FSM failed 3 times'
                logger.error(error_log)
        logger.info('End to stop the Virtual machine.')
        return True

    @retry(stop=stop_after_attempt(2), wait=wait_fixed(10), reraise=True)
    def dsware_vm_action(self, instanceid, action):
        """
        停止|启动虚拟机

        :param instanceid: 虚拟机ID
        :param action: stop | start
        """
        uri = "api/v1/cloud_service/managevm/operate"
        method = "POST"
        body = {
            "action": action,
            "instanceid": instanceid
        }
        logger.info(body)
        cmd = {"method": method, "uri": uri, "body": body}
        rsp_status, _, rsp_body = self.client.exec_cmd(cmd)
        logger.info("rsp_status:{}, response:{}".format(rsp_status, rsp_body))
        if int(rsp_status) != 200:
            error_log = 'Failed to start VM. instanceid:{}, ' \
                        'status code:{}, response:{}'.format(instanceid, rsp_status, rsp_body)
            logger.error(error_log)
            raise Exception(error_log)
        logger.info('{} fsm VM successfully, instanceid:{}'.format(action, instanceid))

    def restart_dsware_vm(self, instanceid_list):
        logger.info("stop vm list:{}".format(instanceid_list))
        for instanceid in instanceid_list:
            self.dsware_vm_action(instanceid, action="stop")
        time.sleep(60)

        logger.info("start vm list:{}".format(instanceid_list))
        for instanceid in instanceid_list:
            self.dsware_vm_action(instanceid, action="start")
        time.sleep(60)
        logger.info("The VM is restarted successfully: {}".format(instanceid_list))

    def delete_dsware_vm(self, dsware_server_no):
        """删除虚拟机，三次重试"""
        logger.info('Start to delete the virtual machine.')
        uri = "api/v1/cloud_service/fusion_storage"
        method = "DELETE"
        body = {
            "poolserialnumber": dsware_server_no
        }
        logger.info(body)
        cmd = {"method": method, "uri": uri, "body": body}
        for i in range(1, 4):
            logger.info('Delete fsm by cmd: %s' % cmd)
            rsp_status, rsp_header, rsp_body = self.client.exec_cmd(cmd)
            try:
                # 虚拟机删除结果处理，失败进行三次重试
                check_delete_fsm_rep(rsp_status, rsp_body)
                break
            except Exception as e:
                if i == 3:
                    error_log = 'Deleting an existing FSM failed, Detail: %s' % str(e)
                    logger.error(error_log)
                else:
                    error_log = 'Delete an existing FSM failure,try again after 10s.Detail: %s' % str(e)
                    logger.error(error_log)
                    time.sleep(10)
        logger.info('End to delete the Virtual machine.')
        return True

    def install_create_fsm_from_webui(self, host_and_tool=None,
                                      vcpu_affinity=None, flavor=None, virtualization_type=None):
        g_webui_client_ctrl.client_list = list()
        self.client = g_webui_client_ctrl.apply_client(host_and_tool, self)
        uri = "api/v1/cloud_service/fusion_storage"
        method = "POST"
        kwargs = {
            "host_and_tool": host_and_tool,
            "vcpuAffinity": vcpu_affinity,
            "flavor": flavor,
            "VirtualizationType": virtualization_type}
        body = self.buildbody(kwargs)
        logger.info(body)
        cmd = {"method": method, "uri": uri, "body": body}
        rsp_status, rsp_header, rsp_body = self.client.exec_cmd(cmd)
        try:
            check_create_fsm_rsp(rsp_status, rsp_body)
            return {"result": True}
        except PubException as e:
            error_log = 'create fsm failed,the exception is:' + str(e)
            logger.error(error_log)
            raise Exception(error_log) from e

    def buildbody(self, buid_hand):
        master = self.fs_args.get("master").get("hostname")
        standby = self.fs_args.get("standby").get("hostname")

        rsp_dict = {}
        # 冗余判断，无单主场景
        if self.fs_args.get("ha_mode") == "single":
            rsp_dict["host"] = [master]
        else:
            rsp_dict["host"] = [master, standby]
            rsp_dict["om-floatip"] = self.fs_args.get("float_ip")

        rsp_dict["hamode"] = self._change_ha_mode_string(self.fs_args.get("ha_mode"))
        rsp_dict["vcpuAffinity"] = buid_hand.get("vcpuAffinity", "Yes")
        if rsp_dict.get("vcpuAffinity") is None:
            rsp_dict["vcpuAffinity"] = "No"
        rsp_dict["flavor"] = buid_hand.get("flavor", "3~64")
        if not rsp_dict.get("flavor"):
            rsp_dict["flavor"] = "3~64"
        rsp_dict["VirtualizationType"] = buid_hand.get("VirtualizationType", "KVM")
        if rsp_dict.get("VirtualizationType") is None:
            rsp_dict["VirtualizationType"] = "KVM"

        uvp_pkg_path = self.fs_args.get('uvp_install_path')
        rsp_dict["version"] = self.get_version_from_uvp(uvp_pkg_path)
        return rsp_dict

    def query_create_fsm_from_webui(self, host_and_tool, float_ip):
        g_webui_client_ctrl.client_list = list()
        self.client = g_webui_client_ctrl.apply_client(host_and_tool, self)
        try:
            query_status = self._wait_install_fsm_complete(float_ip)
        except Exception as e:
            error_log = 'Query FSM Installation progress failed, the exception is:%s' % str(e)
            logger.error(error_log)
            raise Exception(error_log) from e
        return query_status

    def update_fusionstorage_table(self, task_id, subtask_id, query_status):
        datas = list()
        data_info = dict()
        # 从数据库中查询当前工步状态
        get_fsm_install_info = check_soft_install_info(self.project_id, task_id, subtask_id)
        logger.info('query fsm install info from db is %s' % get_fsm_install_info)
        # 查询环境中的虚拟机信息，根据浮动IP获取虚拟机ID
        float_ip = self.fs_args.get('float_ip')

        if not self.dsware_server:
            logger.error("update table return for dsware_server is None")
            return

        # 检查浮动IP和虚拟机dsware_serverX序列对应的虚拟机是否一致
        webui_host_and_tool = FSPAuthClient.get_cps_web_host_and_tool(self.db, self.project_id, self.pod_id)
        g_webui_client_ctrl.client_list = list()
        vm_data = self.query_dsware_vm_with_ip(webui_host_and_tool, float_ip)
        if len(vm_data.keys()) > 1:
            err_msg = "The float IP[%s] corresponds to multiple VMs[%s], " \
                      "Failed VM cannot be recorded." \
                      % (float_ip, vm_data)
            logger.error(err_msg)
            return
        if self.dsware_server not in vm_data.keys():
            err_msg = "The float IP[%s] to be recorded is inconsistent with" \
                      " the vm serial number[%s]." \
                      % (float_ip, self.dsware_server)
            logger.error(err_msg)
            return

        # 根据查询结果，刷新数据库信息
        # 构造数据库信息
        vm_dsware = list()
        vm_dsware.append(self.dsware_server)
        vm_dsware.append(float_ip)
        data_info["project_id"] = self.project_id
        data_info["task_id_str"] = task_id
        data_info["subtask_id_str"] = subtask_id
        data_info["failed_msg"] = ",".join(vm_dsware)
        # 成功状态为2，失败或其它状态为1
        if query_status:
            data_info["status"] = "2"
        else:
            data_info["status"] = "1"
        datas.append(data_info)
        logger.info("Update table data:%s" % datas)
        if len(get_fsm_install_info) == 0:
            # 插入数据
            add_soft_install_info(datas=datas)
        else:
            # 更新数据
            update_soft_install_info_batch(datas=datas)

    def _query_host_info(self):
        # 冗余接口
        uri = "api/v1/hosts"
        method = "get"
        body = None
        cmd = {"method": method, "uri": uri, "body": body}
        rsp_status, rsp_header, rsp_body = self.client.exec_cmd(cmd)
        rsp_dict = {}
        if 200 <= int(rsp_status) <= 299:
            hosts = rsp_body.get("hosts", [])
            for host in hosts:
                omip = host.get("omip")
                host_id = host.get("id")
                rsp_dict[omip] = host_id
        logger.info(rsp_dict)
        return rsp_dict

    def _get_cur_state(self, vm_float_ip):
        uri = "api/v1/cloud_service/fusion_storage"
        method = "GET"
        body = None
        cmd = {"method": method, "uri": uri, "body": body}
        rsp_status, rsp_header, rsp_body = self.client.exec_cmd(cmd)
        if int(rsp_status) < 200 or int(rsp_status) > 299:
            return int(rsp_status), False
        server_state = {}
        key_list = []
        for key, value in rsp_body.items():
            if 'dsware-server' not in key:
                continue
            for vm_node in value:
                if vm_float_ip != vm_node.get("external_om_float_ip"):
                    continue
                key_list.append(key)
                server_state[key] = value
                break
        if len(key_list) > 1:
            err_msg = "Multiple groups of VMs exist in the CPS, " \
                      "and the floating IP address is %s." % vm_float_ip
            logger.error(err_msg)
            raise HCCIException(626321, vm_float_ip, server_state)
        if len(key_list) == 0:
            err_msg = "The created VM is not found based on the float IP[%s]." % vm_float_ip
            logger.error(err_msg)
            return 0, False

        dsware_name = key_list[-1]
        server_state_list = server_state.get(dsware_name)
        self.dsware_server = dsware_name
        msg_info = "The VM created based on the floatIP[%s] is %s." % (vm_float_ip, server_state)
        logger.info(msg_info)
        status_list = [x.get("status", "success") for x in server_state_list]
        if "ERROR" in status_list:
            return 0, False
        install_process_list = [int(x.get("installprocess", "0")) for x in server_state_list]
        try:
            average = sum(install_process_list) / len(install_process_list)
        except ZeroDivisionError as e:
            err_msg = 'ZeroDivisionError, Detail:%s' % str(e)
            logger.error(err_msg)
            raise e
        return average, True

    def _wait_install_fsm_complete(self, float_ip, time_out=3600):
        """
        查询虚拟机创建进度
        :return:
        """
        start_time = time.time()
        retry_time = 3
        while True:
            process, status = self._get_cur_state(float_ip)
            if status is False:
                logger.error("failed to create fsm, please check resource,error:%s" % process)
                if retry_time <= 0:
                    logger.error("No VM are found after retry 3 times.")
                    return False
                time.sleep(90)
                retry_time -= 1
                continue
            time_cost = time.time() - start_time
            if process >= 100:
                logger.info("it cost %s install FSM, and success!" % time_cost)
                return True
            msg_info = "Wait until the FSM VM creation is complete, " \
                       "current progress is %s, time cost %s" % (process, time_cost)
            logger.info(msg_info)
            if time_cost > time_out:
                logger.error("wait FSM install time out, it cost %s time" % time_cost)
                return False
            time.sleep(30)
