# -*- coding: utf-8 -*-
import random
import string
import os
import traceback
import shutil
import platform
import utils.common.log as logger
from utils.common.exception import HCCIException
from plugins.DistributedStorage.common.RestClient import StorageSSHClient
from plugins.DistributedStorage.common.UpgradeOperate import UpgradeOperate
from plugins.DistributedStorage.common.base import TestCase
from plugins.DistributedStorage.common.constants import ErrorInfo
from plugins.DistributedStorage.common.PublicHandleNew import SystemHandle
from plugins.DistributedStorage.common.PublicHandleNew import SshPublicMethod


class DeployManagerSelfUpgrade(TestCase):
    def __init__(self, project_id, pod_id, fs_args, condition=None,
                 metadata=None, **kwargs):
        super(DeployManagerSelfUpgrade, self).__init__(project_id, pod_id)
        self.condition = condition
        self.metadata = metadata
        self.more_args = kwargs
        self.fs_args = fs_args
        self.opr = UpgradeOperate(fs_args)
        self.sys_opr = SystemHandle()
        self.float_ip = fs_args.get("float_ip")
        self.user_name = fs_args.get("user_name")
        self.password = fs_args.get("password")
        self.master_ip = fs_args.get("master_ip")
        self.slaver_ip = fs_args.get("slaver_ip")
        self.master_username = fs_args.get("master_username")
        self.master_password = fs_args.get("master_password")
        self.master_root_pwd = fs_args.get("master_root_pwd")
        self.slaver_username = fs_args.get("slaver_username")
        self.slaver_password = fs_args.get("slaver_password")
        self.slaver_root_pwd = fs_args.get("slaver_root_pwd")
        self.master_account = [self.master_username, self.master_password, self.master_root_pwd]
        self.slaver_account = [self.slaver_username, self.slaver_password, self.slaver_root_pwd]
        self.package_file = fs_args.get("package_path")
        self.package_name = fs_args.get("package_name")
        self.tmp_path_suffix = ''.join(random.choices(string.digits + string.ascii_letters, k=10))
        self.remote_pkg_tmp_dir_name_remote = "self_upgrade_deployManager_tmp"
        if len(self.package_file) > 60:
            current_platform = platform.platform()
            if current_platform.startswith("Windows"):
                local_dst_base_dir = 'c:\\{}'.format(
                    self.remote_pkg_tmp_dir_name_remote)
            else:
                local_dst_base_dir = "/tmp/{}".format(
                    self.remote_pkg_tmp_dir_name_remote)
        else:
            local_dst_base_dir = os.path.splitext(self.package_file)[0]
        self.local_dst_dir = local_dst_base_dir + self.tmp_path_suffix
        self.local_pkg_dir = os.path.join(
            self.local_dst_dir,
            os.path.splitext(os.path.splitext(self.package_name)[0])[0],
            "deploymanager")
        self.single_mode = False
        self.upgrade_mode = ""
        self.root_dir = "/root/"
        self.remote_pkg_dir_remote = self.root_dir + self.remote_pkg_tmp_dir_name_remote

    def procedure(self):
        logger.info('Start self upgrade task.')
        try:
            SystemHandle().unzip_file(self.package_file, self.local_dst_dir)
            status_code, error_code, error_des = self.opr.try_login(self.user_name, self.password)
            if status_code != 200 or error_code != 0:
                err_msg = "Failed to login deploy manager, Detail:[status:{},code:{}]{}".format(
                    status_code, error_code, error_des)
                logger.error(err_msg)
                raise HCCIException(621001, err_msg)
            ssh_client = StorageSSHClient(self.master_ip, self.master_username,
                                          self.master_password)
            logger.info('Create ssh client for %s success' % self.master_ip)
            ssh_client.switch_root(self.master_root_pwd)
            logger.info('Change user to root success')
            # 获取当前升级模式：单fsm和主备模式
            self._get_ha_standby(ssh_client)
            self._clear_self_upgrade_status_file(
                self.master_ip, *self.master_account)
            # 非单fsm模式清理备节点状态文件
            if not self.single_mode:
                self._clear_self_upgrade_status_file(
                    self.slaver_ip, *self.slaver_account)
            ret_value = self._try_self_upgrade(
                self.master_ip, *self.master_account, *self.slaver_account)
            if ret_value == ErrorInfo.ERROR_IS_MASTER:
                ret_value = self._try_self_upgrade(
                    self.slaver_ip, *self.slaver_account, *self.master_account)
                if ret_value == ErrorInfo.ERROR_IS_MASTER:
                    err_msg = "Node {}  and Node {} are the master " \
                              "node.".format(self.master_ip, self.slaver_ip)
                    raise HCCIException(621006, err_msg)

        except HCCIException as e:
            logger.error('deploymanager self upgrade failed:{}'.format(e))
            logger.error(traceback.format_exc())
            raise e
        except Exception as e:
            logger.error('deploymanager self upgrade failed:{}'.format(e))
            logger.error(traceback.format_exc())
            raise HCCIException(621003, str(e))
        finally:
            logger.info('rm -rf {}.'.format(self.local_dst_dir))
            if os.path.exists(self.local_dst_dir):
                shutil.rmtree(self.local_dst_dir, onerror=self.sys_opr.error_remove_read_only)
            logger.info('procedure end!')

    def retry(self):
        logger.info('retry self upgrade task.')
        try:
            self.sys_opr.unzip_file(self.package_file, self.local_dst_dir)
            status_code, error_code, error_des = self.opr.try_login(self.user_name, self.password)
            if status_code != 200 or error_code != 0:
                err_msg = "Failed to login deploy manager, Detail:[status:%s,code:%s]%s" \
                          % (status_code, error_code, error_des)
                logger.error(err_msg)
                raise HCCIException(621001, err_msg)
            ret_value = self._try_self_upgrade_retry(
                self.master_ip, *self.master_account, *self.slaver_account)
            if ret_value == ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST and self.single_mode:
                self._try_self_upgrade(
                    self.master_ip, *self.master_account, *self.slaver_account)
            if ret_value == ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST and not self.single_mode:
                self._ha_try_self_upgrade_retry()
        except HCCIException as e:
            logger.error('self upgrade failed')
            logger.error(traceback.format_exc())
            raise e
        except Exception as e:
            logger.error('self upgrade failed')
            logger.error(traceback.format_exc())
            raise HCCIException(621003, str(e))
        finally:
            logger.info('delete dst dir:{}.'.format(self.local_dst_dir))
            if os.path.exists(self.local_dst_dir):
                shutil.rmtree(self.local_dst_dir, onerror=self.sys_opr.error_remove_read_only)

    def _ha_try_self_upgrade_retry(self):
        ret_value = self._try_self_upgrade_retry(
            self.slaver_ip, *self.slaver_account, *self.master_account)
        if ret_value == ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST:
            err_msg = "Node {}  and Node {} are not package " \
                      "node.".format(self.master_ip, self.slaver_ip)
            logger.info(err_msg)
            ret_value = self._try_self_upgrade(
                self.master_ip, *self.master_account, *self.slaver_account)
            if ret_value == ErrorInfo.ERROR_IS_MASTER:
                ret_value = self._try_self_upgrade(
                    self.slaver_ip, *self.slaver_account, *self.master_account)
                if ret_value == ErrorInfo.ERROR_IS_MASTER:
                    err_msg = "Node {}  and Node {} are the master." \
                        .format(self.master_ip, self.slaver_ip)
                    raise HCCIException(621006, err_msg)

    def _upload_pkg(self, ssh_client, deploymanager_pkg_name, node_ip):
        local_pkg_path = os.path.join(self.local_pkg_dir,
                                      deploymanager_pkg_name)
        tmp_upload_path = "self_upgrade" + self.tmp_path_suffix
        tmp_upload_dir = "/home/" + tmp_upload_path
        mkdir_cmd = "mkdir -p {pkg}; chmod 777 {pkg}; ls -l {tgt_dir} " \
                    "| grep {tmp_pkg_dir}" \
            .format(pkg=tmp_upload_dir, tgt_dir="/home",
                    tmp_pkg_dir=tmp_upload_path)
        cmd_ret = ssh_client.send_cmd(mkdir_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if tmp_upload_path not in cmd_ret:
            err_msg = "make dir failed in node %s, ret: %s" \
                      % (node_ip, cmd_ret)
            logger.error(err_msg)
            raise HCCIException(621003, err_msg)

        ssh_client.upload(local_pkg_path.replace('\\', '/'),
                          tmp_upload_dir.replace('\\', '/'))

    def _un_tar_pkg2(self, ssh_client, deploymanager_pkg_name,
                     remote_tmp_pkg_dir, node_ip):
        remote_pkg_path = os.path.join(
            remote_tmp_pkg_dir, deploymanager_pkg_name).replace('\\', '/')
        tmp_upload_path = "self_upgrade" + self.tmp_path_suffix
        tmp_upload_dir = "/home/" + tmp_upload_path
        mkdir_cmd = "mkdir -p {pkg}; chmod 777 {pkg}; ls -l {root_dir} " \
                    "| grep {tmp_pkg_dir}" \
            .format(pkg=remote_tmp_pkg_dir,
                    root_dir=self.root_dir,
                    tmp_pkg_dir=self.remote_pkg_tmp_dir_name_remote)
        cmd_ret = ssh_client.send_cmd(mkdir_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if self.remote_pkg_tmp_dir_name_remote not in cmd_ret:
            err_msg = "make dir failed in node %s, ret: %s" \
                      % (node_ip, cmd_ret)
            logger.error(err_msg)
            raise HCCIException(621003, err_msg)

        mv_pkg_cmd = "mv {upload_dir}/* {remote_dir} ; echo last_result=$?;" \
                     " rm -rf {upload_dir}"\
            .format(upload_dir=tmp_upload_dir,
                    remote_dir=remote_tmp_pkg_dir.replace('\\', '/'))
        cmd_ret = ssh_client.send_cmd(mv_pkg_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if "last_result=0" not in cmd_ret:
            logger.error("un tar %s failed, ret: %s" % (remote_pkg_path,
                                                        cmd_ret))
            err_msg = "un tar %s failed" % remote_pkg_path
            raise HCCIException(621007, err_msg)
        self.sys_opr.un_tar_pkg(ssh_client, remote_pkg_path, self.remote_pkg_dir_remote)

    def _try_self_upgrade(self, node_ip, node_username, node_password,
                          root_password, remote_username, remote_password,
                          remote_root_password):
        remote_tmp_pkg_dir = self.remote_pkg_dir_remote
        try:
            ssh_client = StorageSSHClient(node_ip, node_username,
                                          node_password)
            logger.info('Create ssh client for %s success' % node_ip)
            ssh_client.switch_root(root_password)

            if ErrorInfo.ERROR_IS_MASTER == self._check_ha_status(ssh_client):
                return ErrorInfo.ERROR_IS_MASTER

            if ErrorInfo.ERROR_PACKAGE_DIR_IS_EXIST == self._check_dir_exist(
                    ssh_client, self.root_dir,
                    self.remote_pkg_tmp_dir_name_remote):
                err_msg = "pkg dir %s in %s has exist, rm it first." \
                          % (remote_tmp_pkg_dir, node_ip)
                logger.error(err_msg)
                raise HCCIException(621003, err_msg)
            deploymanager_pkg_name = SshPublicMethod().get_deploymanager_pkg_name(ssh_client, self.local_pkg_dir)
            self_upgrade_action_dir = "{}/{}/action/".format(
                self.remote_pkg_dir_remote, os.path.splitext(os.path.splitext(
                    deploymanager_pkg_name)[0])[0])
            if ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST == \
                self._check_dir_exist(
                    ssh_client, self_upgrade_action_dir, "upgrade.sh"):
                err_msg = "pkg dir %s in %s exist."\
                          % (self_upgrade_action_dir, node_ip)
                logger.info(err_msg)
                self._upload_pkg(ssh_client, deploymanager_pkg_name,
                                 node_ip)
                ssh_client = StorageSSHClient(node_ip, node_username,
                                              node_password)
                logger.info('Create ssh client for %s success' % node_ip)
                ssh_client.switch_root(root_password)
                self._un_tar_pkg2(ssh_client, deploymanager_pkg_name,
                                  remote_tmp_pkg_dir, node_ip)
            self._cd_work_dir(ssh_client, self_upgrade_action_dir)

            self._deploymanager_self_upgrade(
                ssh_client, node_ip, node_username, node_password,
                root_password, remote_username, remote_password,
                remote_root_password, self_upgrade_action_dir)

            self._rm_pkg_dir(ssh_client, remote_tmp_pkg_dir)

        except Exception as e:
            logger.error("try self upgrade node:{} failed".format(node_ip))
            logger.error(traceback.format_exc())
            if "system TMOUT is too short" in str(e):
                raise HCCIException(621003, "set TMOUT=0 in /etc/profile")
            else:
                raise HCCIException(621003, e)
        finally:
            logger.info("deploymanager self upgrade finished")

    @staticmethod
    def _cd_work_dir(ssh_client, self_upgrade_action_dir):
        change_dir_cmd = "cd {}; echo last_result=$?".format(
            self_upgrade_action_dir)
        logger.info("exec cd cmd: %s" % change_dir_cmd)
        cmd_ret = ssh_client.send_cmd(change_dir_cmd, '#', 10)
        cmd_ret = ''.join(cmd_ret)
        if "last_result=0" not in cmd_ret:
            logger.error(cmd_ret)
            err_msg = "check %s is not exist" % (
                self_upgrade_action_dir)
            raise HCCIException(621008, err_msg)

    @staticmethod
    def _rm_pkg_dir(ssh_client, remote_tmp_pkg_dir):
        clear_pkg_cmd = "rm -rf %s ; " \
                        "echo last_result=$?" % remote_tmp_pkg_dir
        cmd_ret = ssh_client.send_cmd(clear_pkg_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if "last_result=0" not in cmd_ret:
            logger.error("rm tmp dir: %s failed, echo: %s"
                         % (remote_tmp_pkg_dir, str(cmd_ret)))
        else:
            logger.info("rm dir %s" % remote_tmp_pkg_dir)

    def _check_dir_exist(self, ssh_client, dirname, filename):
        cmd_ret = self._ssh_cmd_find_pkg(
            ssh_client, dirname, filename)
        cmd_ret = ''.join(cmd_ret)
        if "last_result=0" in cmd_ret:
            err_msg = "pkg dir %s/%s has exist" % (dirname, filename)
            logger.error(err_msg)
            return ErrorInfo.ERROR_PACKAGE_DIR_IS_EXIST
        else:
            logger.info("find dir %s/%s is not exist" % (dirname, filename))
            return ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST

    @staticmethod
    def _ssh_cmd_find_pkg(ssh_client, dir_name, file_name):
        has_package_dir = \
            "ls {root_dir} | grep {pkg};echo last_result=$?" \
            "".format(root_dir=dir_name,
                      pkg=file_name)
        cmd_ret = ssh_client.send_cmd(has_package_dir, '#')
        return cmd_ret

    def _try_self_upgrade_retry(self, node_ip, node_username, node_password,
                                root_password, remote_username,
                                remote_password, remote_root_password):
        remote_tmp_pkg_dir = self.remote_pkg_dir_remote
        try:
            ssh_client = StorageSSHClient(node_ip, node_username,
                                          node_password)
            logger.info('Create ssh client for %s success' % node_ip)
            ssh_client.switch_root(root_password)

            logger.info('Change user to root success')

            self._get_ha_standby(ssh_client)
            if ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST == self._check_dir_exist(
                    ssh_client, self.root_dir,
                    self.remote_pkg_tmp_dir_name_remote):
                logger.error("find dir %s not exist" % remote_tmp_pkg_dir)
                return ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST

            deploymanager_pkg_name = SshPublicMethod().get_deploymanager_pkg_name(ssh_client, self.local_pkg_dir)

            self_upgrade_action_dir = "{}/{}/action/" \
                .format(self.remote_pkg_dir_remote,
                        os.path.splitext(os.path.splitext(
                            deploymanager_pkg_name)[0])[0])
            self._cd_work_dir(ssh_client, self_upgrade_action_dir)

            self._deploymanager_self_upgrade(
                ssh_client, node_ip, node_username, node_password,
                root_password, remote_username, remote_password,
                remote_root_password, self_upgrade_action_dir)
            self._rm_pkg_dir(ssh_client, remote_tmp_pkg_dir)

        except Exception as e:
            logger.error("try self upgrade node:%s failed" % node_ip)
            logger.error(traceback.format_exc())
            if "system TMOUT is too short" in str(e):
                err_msg = "set TMOUT=0 in /etc/profile"
                raise HCCIException(621003, err_msg)
            else:
                logger.error(traceback.format_exc())
                raise HCCIException(621003, e)
        finally:
            logger.info("deploymanager self upgrade finished")

    def _deploymanager_self_upgrade_for_80(
            self, ssh_client, node_username, node_password, root_password,
            remote_username, remote_password, remote_root_password):

        self._check_deploymanager_self_upgrade(ssh_client)

        if self.single_mode:
            echo_str = "]#"
            logger.error("single mode, wait: %s" % echo_str)
            self_upgrade_cmd = "sh ./upgrade_execute.sh single"
            cmd_ret = ssh_client.send_cmd(self_upgrade_cmd, echo_str, 3600)
        else:
            echo_str = "please input remote username"
            logger.info("double mode, wait: %s" % echo_str)
            self_upgrade_cmd = \
                "sh ./upgrade.sh %s" % self.upgrade_mode
            logger.info(self_upgrade_cmd)
            cmd_ret = ssh_client.send_cmd(self_upgrade_cmd, echo_str, 61)
            echo_str = "please input remote {} password" \
                .format(remote_username)

            cmd_ret = ssh_client.send_cmd(remote_username, echo_str, 62)
            echo_str = "please input remote root password"

            cmd_ret = ssh_client.send_cmd(remote_password, echo_str, 63,
                                          sensitive=True)
            echo_str = "please input local username"
            cmd_ret = ssh_client.send_cmd(
                remote_root_password, echo_str, 64)
            echo_str = "please input local {} password" \
                .format(node_username)
            cmd_ret = ssh_client.send_cmd(node_username, echo_str, 65)
            echo_str = "please input local root password"
            cmd_ret = ssh_client.send_cmd(node_password, echo_str, 66,
                                          sensitive=True)
            echo_str = "]#"
            cmd_ret = ssh_client.send_cmd(root_password, echo_str, 3600,
                                          sensitive=True)
        cmd_ret = ''.join(cmd_ret)
        if "Upgraded successfully" not in cmd_ret:
            logger.error("self upgrade failed, ret: %s" % str(cmd_ret))
            err_msg = "deploy manage self upgrade failed. "
            raise HCCIException(621003, err_msg)
        elif "same version upgrade forbidden" in cmd_ret:
            logger.warn("the deploy manage version is th same")
        else:
            logger.info("exec deploy manager self upgrade success")

    def _deploymanager_self_upgrade_for_81(
            self, ssh_client, node_username, node_password, root_password,
            remote_username, remote_password, remote_root_password):
        if ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST == \
                self._check_dir_exist(ssh_client, "/opt/fusionstorage/", "selfupgrade") and \
                ErrorInfo.ERROR_SAME_VERSION == \
                self._check_deploymanager_self_upgrade_81(ssh_client):
            logger.info("The version is same, return")
            self._check_self_upgrade_result(ssh_client)
            return
        status = self._check_result_for_retry(ssh_client)
        if self.single_mode:
            cmd_ret = self.single_self_upgrade(ssh_client, status)
        else:
            cmd_ret = self.ha_self_upgrade(
                ssh_client, status, node_username, node_password,
                root_password, remote_username, remote_password,
                remote_root_password)
        cmd_ret = ''.join(cmd_ret)
        if "Succeeded in performing the upgrade" in cmd_ret:
            logger.info("exec deploy manager self upgrade success")
        else:
            logger.error("self upgrade failed, ret: %s" % str(cmd_ret))
            echo_str = "]#"
            log_cmd = "tail -n 100 /var/log/DeployManagerScreen.log"
            cmd_ret = ssh_client.send_cmd(log_cmd, echo_str, 62)
            logger.error("self upgrade err info: %s" % str(cmd_ret))
            err_msg = "deploy manage self upgrade failed. "
            raise HCCIException(621003, err_msg)
        self._check_self_upgrade_result(ssh_client)

    def single_self_upgrade(self, ssh_client, status):
        echo_str = "]#"
        logger.error("single mode, wait: %s" % echo_str)
        echo_str = "yes/no"
        self_upgrade_cmd = "sh ./upgrade.sh single"
        ssh_client.send_cmd(self_upgrade_cmd, echo_str, 61)
        if status == "failed":
            echo_str = "please input DeviceManager username:"
            ssh_client.send_cmd("yes", echo_str, 62)
            echo_str = "please input local {} password:".format(self.user_name)
            ssh_client.send_cmd(self.user_name, echo_str, 62)
            echo_str = "]#"
            cmd_ret = ssh_client.send_cmd(self.password, echo_str, 3600)
        elif status == "running":
            echo_str = "]#"
            cmd_ret = ssh_client.send_cmd("yes", echo_str, 3600, sensitive=True)
        return cmd_ret

    def ha_self_upgrade(self, ssh_client, status, node_username,
                        node_password, root_password, remote_username,
                        remote_password, remote_root_password):
        echo_str = "yes/no"
        self_upgrade_cmd = \
            "sh ./upgrade.sh"
        ssh_client.send_cmd(self_upgrade_cmd, echo_str, 61)
        if status == "failed":
            echo_str = "please input DeviceManager username"
            ssh_client.send_cmd("yes", echo_str, 62)
            echo_str = "please input local {} password:".format(self.user_name)
            ssh_client.send_cmd(self.user_name, echo_str, 62)
            echo_str = "please input remote username:"
            ssh_client.send_cmd(self.password, echo_str, 62, sensitive=True)
            echo_str = "please input remote {} password:".format(remote_username)
            ssh_client.send_cmd(remote_username, echo_str, 62)
            echo_str = "please input remote root password:"
            ssh_client.send_cmd(remote_password, echo_str, 62, sensitive=True)
            echo_str = "please input local username:"
            ssh_client.send_cmd(remote_root_password, echo_str, 62, sensitive=True)
            echo_str = "please input local {} password:".format(node_username)
            ssh_client.send_cmd(node_username, echo_str, 62)
            echo_str = "please input local root password:"
            ssh_client.send_cmd(node_password, echo_str, 62, sensitive=True)
            echo_str = "]#"
            cmd_ret = ssh_client.send_cmd(root_password, echo_str, 3600,
                                          sensitive=True)
        elif status == "running":
            echo_str = "]#"
            cmd_ret = ssh_client.send_cmd("yes", echo_str, 3600, sensitive=True)
        return cmd_ret

    def _deploymanager_self_upgrade(
            self, ssh_client, node_ip, node_username, node_password,
            root_password, remote_username, remote_password,
            remote_root_password, self_upgrade_action_dir):
        """
            deploy manager self-upgrade
        """
        if self.sys_opr.check_pkg_version(self.fs_args) == 80:
            logger.info("start self upgrade in %s for 8.0" % node_ip)
            self._deploymanager_self_upgrade_for_80(
                ssh_client, node_username, node_password, root_password,
                remote_username, remote_password, remote_root_password)
        else:
            logger.info("start self upgrade in %s for 8.1" % node_ip)
            self._deploymanager_self_upgrade_for_81(
                ssh_client, node_username, node_password, root_password,
                remote_username, remote_password, remote_root_password)

        if ErrorInfo.ERROR_IS_STANDBY == self._check_ha_status(ssh_client) and not self.single_mode:
            if node_ip == self.master_ip:
                remote_ip = self.slaver_ip
            else:
                remote_ip = self.master_ip
            ssh_client = StorageSSHClient(remote_ip, remote_username, remote_password)
            ssh_client.switch_root(remote_root_password)
        SshPublicMethod().check_status_opr(ssh_client)

        clear_component_not_upgrade_path = os.path.join(
            self_upgrade_action_dir, "clear_component_not_upgrade.sh")
        if os.path.isfile(clear_component_not_upgrade_path):
            self_clear_cmd = "sh clear_component_not_upgrade.sh" \
                             " clear_component"
            cmd_ret = ssh_client.send_cmd(self_clear_cmd, '#')
            cmd_ret = ''.join(cmd_ret)
            if "failed" in cmd_ret:
                logger.error("deploy manage self upgrade clear failed, "
                             "ret: %s " % str(cmd_ret))

    def _check_ha_status(self, ssh_client):
        cmd_ret = self._ssh_cmd_get_ha_status(ssh_client)
        if "Abnormal\n" in cmd_ret:
            logger.error("status abnormal, %s" % ''.join(cmd_ret))
            err_msg = "ha status abnormal"
            raise HCCIException(621003, err_msg)
        return self._get_ha_standby(ssh_client)

    @staticmethod
    def _ssh_cmd_get_ha_status(ssh_client):
        ha_status_cmd = "/opt/dfv/oam/oam-u/ha/ha/module/hacom/tools" \
                        "/ha_client_tool --gethastatus " \
                        "| grep '^ha' | awk '{print $4}'"
        cmd_ret = ssh_client.send_cmd(ha_status_cmd, '#')
        return cmd_ret

    def _get_ha_standby(self, ssh_client):
        cmd_ret = self._ssh_cmd_get_ha_mode(ssh_client, "HA_MODE")
        if "single\n" in cmd_ret:
            logger.error("single mode, %s " % ''.join(cmd_ret))
            self.single_mode = True
            return ErrorInfo.ERROR_IS_STANDBY
        else:
            logger.info("double mode, %s " % ''.join(cmd_ret))
            cmd_ret = self._ssh_cmd_get_ha_mode(ssh_client, "LOCAL_ROLE")
            if "active\n" in cmd_ret:
                logger.info("get current node is active")
                return ErrorInfo.ERROR_IS_MASTER
            if "standby\n" in cmd_ret:
                logger.info("get current node is standby")
                return ErrorInfo.ERROR_IS_STANDBY
        logger.error("dont get current node mode")
        return ErrorInfo.ERROR_IS_MASTER

    @staticmethod
    def _ssh_cmd_get_ha_mode(ssh_client, matched):
        ha_info_cmd = "/opt/dfv/oam/oam-u/ha/tools/getHAStatus.sh | grep %s " \
                      "| awk -F : '{print $2}'|awk '$1=$1'" % matched
        cmd_ret = ssh_client.send_cmd(ha_info_cmd, '#')
        return cmd_ret

    def _check_deploymanager_self_upgrade(self, ssh_client):
        cmd_ret = self._ssh_cmd_get_TMOUT(ssh_client)
        cmd_ret = ''.join(cmd_ret)
        if "last_result=0" in cmd_ret:
            logger.info(cmd_ret)
            self.upgrade_mode = "without_TMOUT_check_for_hcsu"
        else:
            logger.error(cmd_ret)
            cmd_ret = self._ssh_cmd_read_TMOUT(ssh_client)
            tm_out = int(cmd_ret[1])
            if 0 < tm_out < 3600:
                raise HCCIException(
                    621008, "TMOUT={} is too short".format(tm_out))

        cmd_ret = self._ssh_cmd_check_dup_upg(ssh_client)
        cmd_ret = ''.join(cmd_ret)
        if "deploymanager_no_self_upgrade" in cmd_ret:
            logger.info("no self upgrade, continue.")
        elif "deploymanager_self_upgrade_running" in cmd_ret:
            logger.error("self upgrade is running, ret: %s" % cmd_ret)
            err_msg = "self upgrade is running"
            raise HCCIException(621011, err_msg)
        else:
            err_msg = "Not support check_duplicate_upgrade param"
            logger.error(err_msg)

    @staticmethod
    def _check_deploymanager_self_upgrade_81(ssh_client):
        check_version_cmd = "sh ./upgrade.sh check_version_for_upgrade"
        logger.info("check version: %s" % check_version_cmd)
        cmd_ret = ssh_client.send_cmd(check_version_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if "The same version no need to upgrade" in cmd_ret:
            logger.info("Version is same: %s." % cmd_ret)
            return ErrorInfo.ERROR_SAME_VERSION
        elif "The versions are not same can be upgraded" in cmd_ret:
            logger.info("Version is not same. ret: %s." % cmd_ret)
        elif "Failed to get version information cannot be upgraded" in cmd_ret:
            err_msg = "get Version failed, %s" % cmd_ret
            logger.error(err_msg)
            raise HCCIException(621011, err_msg)
        else:
            err_msg = "Failed to verify version for self upgrade"
            logger.error(err_msg)

    @staticmethod
    def _check_self_upgrade_result(ssh_client):
        check_result_cmd = "cat /tmp/.self_upgrade_status"
        logger.info("check version: %s" % check_result_cmd)
        cmd_ret = ssh_client.send_cmd(check_result_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if "success" in cmd_ret:
            logger.info("Self upgrade is success.")
        elif "running" in cmd_ret:
            log_cmd = "tail -n 50 /var/log/DeployManagerScreen.log"
            logger.info("get log: %s" % log_cmd)
            cmd_ret = ssh_client.send_cmd(log_cmd, '#')
            err_msg = "Self upgrade is running: %s" % cmd_ret
            logger.error("Failed: %s" % err_msg)
            raise HCCIException(621011, err_msg)
        elif "failed" in cmd_ret:
            log_cmd = "tail -n 50 /var/log/DeployManagerScreen.log"
            cmd_ret = ssh_client.send_cmd(log_cmd, '#')
            err_msg = "Self upgrade is failed: %s" % cmd_ret
            logger.error("Failed: %s" % err_msg)
            raise HCCIException(621011, err_msg)
        else:
            err_msg = "Failed to check result for self upgrade"
            logger.error(err_msg)

    @staticmethod
    def _check_result_for_retry(ssh_client):
        check_result_cmd = "cat /tmp/.self_upgrade_status"
        logger.info("check version: %s" % check_result_cmd)
        cmd_ret = ssh_client.send_cmd(check_result_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if "running" in cmd_ret:
            return "running"
        return "failed"

    @staticmethod
    def _ssh_cmd_get_TMOUT(ssh_client):
        check_params_cmd = "sh ./upgrade.sh get_params | " \
                           "grep without_TMOUT_check_for_hcsu ; " \
                           "echo last_result=$?"
        cmd_ret = ssh_client.send_cmd(check_params_cmd, '#')
        return cmd_ret

    @staticmethod
    def _ssh_cmd_read_TMOUT(ssh_client):
        cmd_ret = ssh_client.send_cmd("echo $TMOUT", '#', 60)
        return cmd_ret

    @staticmethod
    def _ssh_cmd_check_dup_upg(ssh_client):
        check_has_self_upgrade_cmd = \
            "sh ./upgrade.sh check_duplicate_upgrade"
        cmd_ret = ssh_client.send_cmd(check_has_self_upgrade_cmd, '#')
        return cmd_ret

    def _clear_self_upgrade_status_file(self, node_ip, node_username,
                                        node_password, root_password):
        self_upgrade_status_file = "/tmp/.self_upgrade_status"
        try:
            ssh_client = StorageSSHClient(node_ip, node_username,
                                          node_password)
            logger.info('Create ssh client for %s success' % node_ip)
            ssh_client.switch_root(root_password)
            cmd_ret = self._ssh_cmd_rm_file(ssh_client,
                                            self_upgrade_status_file)
            cmd_ret = ''.join(cmd_ret)
            if "last_result=0" in cmd_ret:
                err_msg = "remove file {file_name}".format(
                    file_name=self_upgrade_status_file)
                logger.info(err_msg)
            else:
                err_msg = "remove file {file_name}, failed:{ret}".format(
                    file_name=self_upgrade_status_file, ret=cmd_ret)
                logger.error(err_msg)
        except Exception as e:
            logger.error("clear file:%s in node %s failed, exc %s"
                         % (self_upgrade_status_file, node_ip, e))
            logger.error(traceback.format_exc())
        finally:
            logger.info("clear self upgrade status file:%s in node %s"
                        % (self_upgrade_status_file, node_ip))

    @staticmethod
    def _ssh_cmd_rm_file(ssh_client, file_name):
        delete_file_cmd = "rm -f {file_name}; echo last_result=$?".format(
                file_name=file_name)
        logger.info('execute cmd %s' % delete_file_cmd)
        cmd_ret = ssh_client.send_cmd(delete_file_cmd, '#')
        return cmd_ret
