# -*- coding: utf-8 -*-
# Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved.
import random
import pathlib
import string
import os
import traceback
import shutil
import platform
from collections import namedtuple
from utils.business.project_condition_utils import get_project_condition_boolean
import utils.common.log as logger
from utils.common.exception import HCCIException
from plugins.DistributedStorage.common.rest_client import StorageSSHClient
from plugins.DistributedStorage.common.upgrade_operate import UpgradeOperate
from plugins.DistributedStorage.common.base import TestCase
from plugins.DistributedStorage.common.constants import ErrorInfo
from plugins.DistributedStorage.common.public_handle_new import SystemHandle
from plugins.DistributedStorage.common.public_handle_new import SshPublicMethod
from plugins.DistributedStorage.common.public_handle_new import RestPublicMethod


class DeployManagerSelfUpgrade(TestCase):
    def __init__(self, project_id, pod_id, fs_args, **kwargs):
        super(DeployManagerSelfUpgrade, self).__init__(project_id, pod_id)
        self.more_args = kwargs
        self.fs_args = fs_args
        self.opr = UpgradeOperate(fs_args)
        self.sys_opr = SystemHandle()
        self.rest_opr = RestPublicMethod(project_id, pod_id, fs_args)
        self.float_ip = fs_args.get("float_ip")
        self.user_name = fs_args.get("user_name")
        self.password = fs_args.get("password")
        self.master_ip = fs_args.get("master_ip")
        self.slaver_ip = fs_args.get("slaver_ip")
        self.master_username = fs_args.get("master_username")
        self.master_password = fs_args.get("master_password")
        self.master_root_pwd = fs_args.get("master_root_pwd")
        self.slaver_username = fs_args.get("slaver_username")
        self.slaver_password = fs_args.get("slaver_password")
        self.slaver_root_pwd = fs_args.get("slaver_root_pwd")
        self.master_account = [self.master_username, self.master_password, self.master_root_pwd]
        self.slaver_account = [self.slaver_username, self.slaver_password, self.slaver_root_pwd]
        self.package_file = fs_args.get("package_path")
        self.package_name = fs_args.get("package_name")
        self.tmp_path_suffix = ''.join(random.choices(string.digits + string.ascii_letters, k=10))
        self.remote_pkg_tmp_dir_name_remote = "self_upgrade_deployManager_tmp"
        if len(self.package_file) > 60:
            current_platform = platform.platform()
            if current_platform.startswith("Windows"):
                local_dst_base_dir = 'c:\\{}'.format(
                    self.remote_pkg_tmp_dir_name_remote)
            else:
                local_dst_base_dir = "/tmp/{}".format(
                    self.remote_pkg_tmp_dir_name_remote)
        else:
            local_dst_base_dir = os.path.splitext(self.package_file)[0]
        self.local_dst_dir = local_dst_base_dir + self.tmp_path_suffix
        self.local_pkg_dir = os.path.join(
            self.local_dst_dir,
            os.path.splitext(os.path.splitext(self.package_name)[0])[0],
            "deploymanager")
        self.single_mode = False
        self.upgrade_mode = ""
        self.root_dir = "/root/"
        self.remote_pkg_dir_remote = self.root_dir + self.remote_pkg_tmp_dir_name_remote
        self.tmp_upload_dir = pathlib.Path.joinpath(pathlib.Path('/home/'), "self_upgrade" + self.tmp_path_suffix)
        self.is_one_step_upgrade = get_project_condition_boolean(self.project_id, "OneStepUpgrade")

    @staticmethod
    def _ssh_cmd_find_pkg(ssh_client, dir_name, file_name):
        has_package_dir = \
            "ls {root_dir} | grep {pkg};echo last_result=$?" \
            "".format(root_dir=dir_name,
                      pkg=file_name)
        cmd_ret = ssh_client.send_cmd(has_package_dir, '#')
        return cmd_ret

    @staticmethod
    def _cd_work_dir(ssh_client, self_upgrade_action_dir):
        change_dir_cmd = "cd {}; echo last_result=$?".format(
            self_upgrade_action_dir)
        logger.info("exec cd cmd: %s" % change_dir_cmd)
        cmd_ret = ssh_client.send_cmd(change_dir_cmd, '#', 10)
        cmd_ret = ''.join(cmd_ret)
        if "last_result=0" not in cmd_ret:
            logger.error(cmd_ret)
            err_msg = "check %s is not exist" % (
                self_upgrade_action_dir)
            raise HCCIException(621008, err_msg)

    @staticmethod
    def _rm_pkg_dir(ssh_client, remote_tmp_pkg_dir):
        clear_pkg_cmd = "rm -rf %s ; " \
                        "echo last_result=$?" % remote_tmp_pkg_dir
        cmd_ret = ssh_client.send_cmd(clear_pkg_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if "last_result=0" not in cmd_ret:
            logger.error("rm tmp dir: %s failed, echo: %s"
                         % (remote_tmp_pkg_dir, str(cmd_ret)))
        else:
            logger.info("rm dir %s" % remote_tmp_pkg_dir)

    @staticmethod
    def _ssh_cmd_get_ha_status(ssh_client):
        ha_status_cmd = "/opt/dfv/oam/oam-u/ha/ha/module/hacom/tools" \
                        "/ha_client_tool --gethastatus " \
                        "| grep '^ha' | awk '{print $4}'"
        cmd_ret = ssh_client.send_cmd(ha_status_cmd, '#')
        return cmd_ret

    @staticmethod
    def _ssh_cmd_get_ha_mode(ssh_client, matched):
        ha_info_cmd = "/opt/dfv/oam/oam-u/ha/tools/getHAStatus.sh | grep %s " \
                      "| awk -F : '{print $2}'|awk '$1=$1'" % matched
        cmd_ret = ssh_client.send_cmd(ha_info_cmd, '#')
        return cmd_ret

    @staticmethod
    def _check_deploymanager_self_upgrade_81(ssh_client):
        check_version_cmd = "sh ./upgrade.sh check_version_for_upgrade"
        logger.info("check version: %s" % check_version_cmd)
        cmd_ret = ssh_client.send_cmd(check_version_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if "The same version no need to upgrade" in cmd_ret:
            logger.info("Version is same: %s." % cmd_ret)
            return ErrorInfo.ERROR_SAME_VERSION
        elif "The versions are not same can be upgraded" in cmd_ret:
            not_same_version = None
            logger.info("Version is not same. ret: %s." % cmd_ret)
            return not_same_version
        elif "Failed to get version information cannot be upgraded" in cmd_ret:
            err_msg = "get Version failed, %s" % cmd_ret
            logger.error(err_msg)
            raise HCCIException(621011, err_msg)
        else:
            err_msg = "Failed to verify version for self upgrade"
            logger.error(err_msg)
            no_version = None
            return no_version

    @staticmethod
    def _check_self_upgrade_result(ssh_client):
        check_result_cmd = "cat /tmp/.self_upgrade_status"
        logger.info("check version: %s" % check_result_cmd)
        cmd_ret = ssh_client.send_cmd(check_result_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if "success" in cmd_ret:
            logger.info("Self upgrade is success.")
        elif "running" in cmd_ret:
            log_cmd = "tail -n 50 /var/log/DeployManagerScreen.log"
            logger.info("get log: %s" % log_cmd)
            cmd_ret = ssh_client.send_cmd(log_cmd, '#')
            err_msg = "Self upgrade is running: %s" % cmd_ret
            logger.error("Failed: %s" % err_msg)
            raise HCCIException(621011, err_msg)
        elif "failed" in cmd_ret:
            log_cmd = "tail -n 50 /var/log/DeployManagerScreen.log"
            cmd_ret = ssh_client.send_cmd(log_cmd, '#')
            err_msg = "Self upgrade is failed: %s" % cmd_ret
            logger.error("Failed: %s" % err_msg)
            raise HCCIException(621011, err_msg)
        else:
            err_msg = "Failed to check result for self upgrade"
            logger.error(err_msg)

    @staticmethod
    def _check_result_for_retry(ssh_client):
        check_result_cmd = "cat /tmp/.self_upgrade_status"
        logger.info("check version: %s" % check_result_cmd)
        cmd_ret = ssh_client.send_cmd(check_result_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if "running" in cmd_ret:
            return "running"
        return "failed"

    @staticmethod
    def _ssh_cmd_get_tmout(ssh_client):
        check_params_cmd = "sh ./upgrade.sh get_params | " \
                           "grep without_TMOUT_check_for_hcsu ; " \
                           "echo last_result=$?"
        cmd_ret = ssh_client.send_cmd(check_params_cmd, '#')
        return cmd_ret

    @staticmethod
    def _ssh_cmd_read_tmout(ssh_client):
        cmd_ret = ssh_client.send_cmd("echo $TMOUT", '#', 60)
        return cmd_ret

    @staticmethod
    def _ssh_cmd_check_dup_upg(ssh_client):
        check_has_self_upgrade_cmd = \
            "sh ./upgrade.sh check_duplicate_upgrade"
        cmd_ret = ssh_client.send_cmd(check_has_self_upgrade_cmd, '#')
        return cmd_ret

    @staticmethod
    def _ssh_cmd_rm_file(ssh_client, file_name):
        delete_file_cmd = "rm -f {file_name}; echo last_result=$?".format(
            file_name=file_name)
        logger.info('execute cmd %s' % delete_file_cmd)
        cmd_ret = ssh_client.send_cmd(delete_file_cmd, '#')
        return cmd_ret

    def procedure(self):
        logger.info('Start self upgrade task.')
        if not self.is_this_step_upgrade_deploymanager():
            return
        try:
            SystemHandle().unzip_file(self.package_file, self.local_dst_dir)

            self.get_fsm_ha_mode()

            self.clear_self_upgrade_status_file()

            ret_value = self._try_self_upgrade(
                self.master_ip, *self.master_account, *self.slaver_account)
            if ret_value == ErrorInfo.ERROR_IS_MASTER:
                ret_value = self._try_self_upgrade(
                    self.slaver_ip, *self.slaver_account, *self.master_account)
                if ret_value == ErrorInfo.ERROR_IS_MASTER:
                    err_msg = "Node {}  and Node {} are the master " \
                              "node.".format(self.master_ip, self.slaver_ip)
                    raise HCCIException(621006, err_msg)
        finally:
            logger.info('rm -rf {}.'.format(self.local_dst_dir))
            if os.path.exists(self.local_dst_dir):
                shutil.rmtree(self.local_dst_dir, onerror=self.sys_opr.error_remove_read_only)
            logger.info('procedure end!')

    def get_fsm_ha_mode(self):
        """获取fsm升级模式：单fsm和主备模式"""
        ssh_client = StorageSSHClient(self.master_ip, self.master_username, self.master_password)
        logger.info('Create ssh client for %s success' % self.master_ip)
        ssh_client.switch_root(self.master_root_pwd)
        logger.info('Change user to root success')
        self._get_ha_standby(ssh_client)

    def clear_self_upgrade_status_file(self):
        try:
            self._clear_self_upgrade_status_file(self.master_ip, *self.master_account)
        except Exception as e:
            logger.error("clear file in node %s failed, exc %s" % (self.master_ip, e))
            logger.error(traceback.format_exc())
        # 非单fsm模式清理备节点状态文件
        if not self.single_mode:
            try:
                self._clear_self_upgrade_status_file(self.slaver_ip, *self.slaver_account)
            except Exception as e:
                logger.error("clear file in node %s failed, exc %s" % (self.master_ip, e))
                logger.error(traceback.format_exc())

    def retry(self):
        logger.info('retry self upgrade task.')
        if not self.is_this_step_upgrade_deploymanager():
            return
        try:
            self.sys_opr.unzip_file(self.package_file, self.local_dst_dir)

            ret_value = self._try_self_upgrade_retry(
                self.master_ip, *self.master_account, *self.slaver_account)
            if ret_value == ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST and self.single_mode:
                self._try_self_upgrade(
                    self.master_ip, *self.master_account, *self.slaver_account)
            if ret_value == ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST and not self.single_mode:
                self._ha_try_self_upgrade_retry()
        finally:
            logger.info('delete dst dir:{}.'.format(self.local_dst_dir))
            if os.path.exists(self.local_dst_dir):
                shutil.rmtree(self.local_dst_dir, onerror=self.sys_opr.error_remove_read_only)
            logger.info('retry end!')

    def is_this_step_upgrade_deploymanager(self):
        status_code, error_code, error_des = self.opr.try_login(self.user_name, self.password)
        if status_code != 200 or error_code != 0:
            err_msg = "Failed to login deploy manager, Detail:[status:{},code:{}]{}".format(
                status_code, error_code, error_des)
            logger.error(err_msg)
            raise HCCIException(621001, err_msg)
        is_this_step_upgrade = True
        if self.rest_opr.storage_version_after_813():
            if not self.is_one_step_upgrade:
                # 813及之后版本，常规升级场景，在升级前准备工步，调rest接口自升级
                is_this_step_upgrade = False
        return is_this_step_upgrade

    def single_self_upgrade(self, ssh_client, status):
        echo_str = "]#"
        logger.error("single mode, wait: %s" % echo_str)
        echo_str = "yes/no"
        self_upgrade_cmd = "sh ./upgrade.sh single"
        ssh_client.send_cmd(self_upgrade_cmd, echo_str, 61)
        cmd_ret = ""
        if status == "failed":
            echo_str = "please input DeviceManager username:"
            ssh_client.send_cmd("yes", echo_str, 62)
            echo_str = "please input local {} password:".format(self.user_name)
            ssh_client.send_cmd(self.user_name, echo_str, 62)
            echo_str = "]#"
            cmd_ret = ssh_client.send_cmd(self.password, echo_str, 3600)
        elif status == "running":
            echo_str = "]#"
            cmd_ret = ssh_client.send_cmd("yes", echo_str, 3600, sensitive=True)
        return cmd_ret

    def ha_self_upgrade(self, info):
        ssh_client, status, node_username, node_password, root_password, \
        remote_username, remote_password, remote_root_password = \
            info[0], info[1], info[2], info[3], info[4], info[5], info[6], info[7]
        echo_str = "yes/no"
        self_upgrade_cmd = \
            "sh ./upgrade.sh"
        ssh_client.send_cmd(self_upgrade_cmd, echo_str, 61)
        cmd_ret = ""
        if status == "failed":
            echo_str = "please input DeviceManager username"
            ssh_client.send_cmd("yes", echo_str, 62)
            echo_str = "please input local {} password:".format(self.user_name)
            ssh_client.send_cmd(self.user_name, echo_str, 62)
            echo_str = "please input remote username:"
            ssh_client.send_cmd(self.password, echo_str, 62, sensitive=True)
            echo_str = "please input remote {} password:".format(remote_username)
            ssh_client.send_cmd(remote_username, echo_str, 62)
            echo_str = "please input remote root password:"
            ssh_client.send_cmd(remote_password, echo_str, 62, sensitive=True)
            echo_str = "please input local username:"
            ssh_client.send_cmd(remote_root_password, echo_str, 62, sensitive=True)
            echo_str = "please input local {} password:".format(node_username)
            ssh_client.send_cmd(node_username, echo_str, 62)
            echo_str = "please input local root password:"
            ssh_client.send_cmd(node_password, echo_str, 62, sensitive=True)
            echo_str = "]#"
            cmd_ret = ssh_client.send_cmd(root_password, echo_str, 3600,
                                          sensitive=True)
        elif status == "running":
            echo_str = "]#"
            cmd_ret = ssh_client.send_cmd("yes", echo_str, 3600, sensitive=True)
        return cmd_ret

    def _ha_try_self_upgrade_retry(self):
        ret_value = self._try_self_upgrade_retry(
            self.slaver_ip, *self.slaver_account, *self.master_account)
        if ret_value == ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST:
            err_msg = "Node {}  and Node {} are not package " \
                      "node.".format(self.master_ip, self.slaver_ip)
            logger.info(err_msg)
            ret_value = self._try_self_upgrade(
                self.master_ip, *self.master_account, *self.slaver_account)
            if ret_value == ErrorInfo.ERROR_IS_MASTER:
                ret_value = self._try_self_upgrade(
                    self.slaver_ip, *self.slaver_account, *self.master_account)
                if ret_value == ErrorInfo.ERROR_IS_MASTER:
                    err_msg = "Node {}  and Node {} are the master." \
                        .format(self.master_ip, self.slaver_ip)
                    raise HCCIException(621006, err_msg)

    def _upload_pkg(self, ssh_client, deploymanager_pkg_name, node_ip):
        local_pkg_path = os.path.join(self.local_pkg_dir,
                                      deploymanager_pkg_name)
        tmp_upload_path = self.tmp_upload_dir.name
        mkdir_cmd = "mkdir -p {pkg}; chmod 777 {pkg}; ls -l {tgt_dir} " \
                    "| grep {tmp_pkg_dir}" \
            .format(pkg=self.tmp_upload_dir, tgt_dir="/home",
                    tmp_pkg_dir=tmp_upload_path)
        cmd_ret = ssh_client.send_cmd(mkdir_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if tmp_upload_path not in cmd_ret:
            err_msg = "make dir failed in node %s, ret: %s" \
                      % (node_ip, cmd_ret)
            logger.error(err_msg)
            raise HCCIException(621003, err_msg)

        ssh_client.upload(local_pkg_path.replace('\\', '/'),
                          str(self.tmp_upload_dir).replace('\\', '/'))

    def _un_tar_pkg2(self, ssh_client, deploymanager_pkg_name, remote_tmp_pkg_dir, node_ip):
        remote_pkg_path = os.path.join(
            remote_tmp_pkg_dir, deploymanager_pkg_name).replace('\\', '/')
        mkdir_cmd = "mkdir -p {pkg}; chmod 777 {pkg}; ls -l {root_dir} " \
                    "| grep {tmp_pkg_dir}" \
            .format(pkg=remote_tmp_pkg_dir,
                    root_dir=self.root_dir,
                    tmp_pkg_dir=self.remote_pkg_tmp_dir_name_remote)
        cmd_ret = ssh_client.send_cmd(mkdir_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if self.remote_pkg_tmp_dir_name_remote not in cmd_ret:
            err_msg = "make dir failed in node %s, ret: %s" \
                      % (node_ip, cmd_ret)
            logger.error(err_msg)
            raise HCCIException(621003, err_msg)

        mv_pkg_cmd = "mv {upload_dir}/* {remote_dir} ; echo last_result=$?;" \
                     " rm -rf {upload_dir}" \
            .format(upload_dir=str(self.tmp_upload_dir),
                    remote_dir=remote_tmp_pkg_dir.replace('\\', '/'))
        cmd_ret = ssh_client.send_cmd(mv_pkg_cmd, '#')
        cmd_ret = ''.join(cmd_ret)
        if "last_result=0" not in cmd_ret:
            logger.error("un tar %s failed, ret: %s" % (remote_pkg_path,
                                                        cmd_ret))
            err_msg = "un tar %s failed" % remote_pkg_path
            raise HCCIException(621007, err_msg)
        self.sys_opr.un_tar_pkg(ssh_client, remote_pkg_path, self.remote_pkg_dir_remote)

    def _try_self_upgrade(self, *args):
        node_ip, node_username, node_password, root_password, remote_username, remote_password, \
        remote_root_password = args
        remote_tmp_pkg_dir = self.remote_pkg_dir_remote
        try:
            ret = self._do_self_upgrade(*[node_ip, node_password, node_username, remote_password, remote_root_password,
                                          remote_tmp_pkg_dir, remote_username, root_password])
        except Exception as e:
            logger.error("try self upgrade node:{} failed".format(node_ip))
            logger.error(traceback.format_exc())
            if "system TMOUT is too short" in str(e):
                raise HCCIException(621003, "set TMOUT=0 in /etc/profile") from e
            else:
                raise HCCIException(621003, e) from e
        finally:
            logger.info("deploymanager self upgrade finished")
        return ret

    def _do_self_upgrade(self, *args):
        node_ip, node_password, node_username, remote_password, remote_root_password, \
        remote_tmp_pkg_dir, remote_username, root_password = args
        ssh_client = StorageSSHClient(node_ip, node_username, node_password)
        logger.info('Create ssh client for %s success' % node_ip)
        ssh_client.switch_root(root_password)
        if ErrorInfo.ERROR_IS_MASTER == self._check_ha_status(ssh_client):
            return ErrorInfo.ERROR_IS_MASTER
        if ErrorInfo.ERROR_PACKAGE_DIR_IS_EXIST == self._check_dir_exist(
                ssh_client, self.root_dir, self.remote_pkg_tmp_dir_name_remote):
            err_msg = "pkg dir %s in %s has exist, rm it first." % (remote_tmp_pkg_dir, node_ip)
            logger.error(err_msg)
            raise HCCIException(621003, err_msg)
        deploymanager_pkg_name = SshPublicMethod().get_deploymanager_pkg_name(ssh_client, self.local_pkg_dir)
        self_upgrade_action_dir = "{}/{}/action/".format(
            self.remote_pkg_dir_remote, os.path.splitext(os.path.splitext(deploymanager_pkg_name)[0])[0])
        if ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST == self._check_dir_exist(
                ssh_client, self_upgrade_action_dir, "upgrade.sh"):
            err_msg = "pkg dir %s in %s exist." % (self_upgrade_action_dir, node_ip)
            logger.info(err_msg)
            self._upload_pkg(ssh_client, deploymanager_pkg_name, node_ip)
            ssh_client = StorageSSHClient(node_ip, node_username, node_password)
            logger.info('Create ssh client for %s success' % node_ip)
            ssh_client.switch_root(root_password)
            self._un_tar_pkg2(ssh_client, deploymanager_pkg_name, remote_tmp_pkg_dir, node_ip)
        self._cd_work_dir(ssh_client, self_upgrade_action_dir)
        args = [ssh_client, node_ip, node_username, node_password, root_password, remote_username, remote_password,
                remote_root_password, self_upgrade_action_dir]
        self._deploymanager_self_upgrade(*args)
        self._rm_pkg_dir(ssh_client, remote_tmp_pkg_dir)
        no_error = None
        return no_error

    def _check_dir_exist(self, ssh_client, dirname, filename):
        cmd_ret = self._ssh_cmd_find_pkg(
            ssh_client, dirname, filename)
        cmd_ret = ''.join(cmd_ret)
        if "last_result=0" in cmd_ret:
            err_msg = "pkg dir %s/%s has exist" % (dirname, filename)
            logger.error(err_msg)
            return ErrorInfo.ERROR_PACKAGE_DIR_IS_EXIST
        else:
            logger.info("find dir %s/%s is not exist" % (dirname, filename))
            return ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST

    def _try_self_upgrade_retry(self, *args):
        node_ip, node_username, node_password, root_password, remote_username, \
        remote_password, remote_root_password = args
        remote_tmp_pkg_dir = self.remote_pkg_dir_remote
        try:
            ret = self._do_retry_self_upgrade(*[node_ip, node_password, node_username, remote_password,
                                                remote_root_password, remote_tmp_pkg_dir, remote_username,
                                                root_password])
        except Exception as e:
            logger.error("try self upgrade node:%s failed" % node_ip)
            logger.error(traceback.format_exc())
            if "system TMOUT is too short" in str(e):
                err_msg = "set TMOUT=0 in /etc/profile"
                raise HCCIException(621003, err_msg) from e
            else:
                logger.error(traceback.format_exc())
                raise HCCIException(621003, e) from e
        finally:
            logger.info("deploymanager self upgrade finished")
        return ret

    def _do_retry_self_upgrade(self, *args):
        node_ip, node_password, node_username, remote_password, remote_root_password, \
        remote_tmp_pkg_dir, remote_username, root_password = args
        ret = None
        ssh_client = StorageSSHClient(node_ip, node_username, node_password)
        logger.info('Create ssh client for %s success' % node_ip)
        ssh_client.switch_root(root_password)
        logger.info('Change user to root success')
        self._get_ha_standby(ssh_client)
        if ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST == self._check_dir_exist(
                ssh_client, self.root_dir, self.remote_pkg_tmp_dir_name_remote):
            logger.error("find dir %s not exist" % remote_tmp_pkg_dir)
            return ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST
        deploymanager_pkg_name = SshPublicMethod().get_deploymanager_pkg_name(ssh_client, self.local_pkg_dir)
        self_upgrade_action_dir = "{}/{}/action/".format(
            self.remote_pkg_dir_remote, os.path.splitext(os.path.splitext(deploymanager_pkg_name)[0])[0])
        self._cd_work_dir(ssh_client, self_upgrade_action_dir)
        self._deploymanager_self_upgrade(*[ssh_client, node_ip, node_username, node_password,
                                           root_password, remote_username, remote_password,
                                           remote_root_password, self_upgrade_action_dir])
        self._rm_pkg_dir(ssh_client, remote_tmp_pkg_dir)
        return ret

    def _deploymanager_self_upgrade_for_80(self, *args):
        ssh_client, node_username, node_password, root_password, remote_username, \
        remote_password, remote_root_password = args
        self._check_deploymanager_self_upgrade(ssh_client)

        if self.single_mode:
            echo_str = "]#"
            logger.error("single mode, wait: %s" % echo_str)
            self_upgrade_cmd = "sh ./upgrade_execute.sh single"
            cmd_ret = ssh_client.send_cmd(self_upgrade_cmd, echo_str, 3600)
        else:
            echo_str = "please input remote username"
            logger.info("double mode, wait: %s" % echo_str)
            self_upgrade_cmd = \
                "sh ./upgrade.sh %s" % self.upgrade_mode
            logger.info(self_upgrade_cmd)
            cmd_ret = ssh_client.send_cmd(self_upgrade_cmd, echo_str, 61)
            echo_str = "please input remote {} password" \
                .format(remote_username)

            cmd_ret = ssh_client.send_cmd(remote_username, echo_str, 62)
            echo_str = "please input remote root password"

            cmd_ret = ssh_client.send_cmd(remote_password, echo_str, 63,
                                          sensitive=True)
            echo_str = "please input local username"
            cmd_ret = ssh_client.send_cmd(
                remote_root_password, echo_str, 64)
            echo_str = "please input local {} password" \
                .format(node_username)
            cmd_ret = ssh_client.send_cmd(node_username, echo_str, 65)
            echo_str = "please input local root password"
            cmd_ret = ssh_client.send_cmd(node_password, echo_str, 66,
                                          sensitive=True)
            echo_str = "]#"
            cmd_ret = ssh_client.send_cmd(root_password, echo_str, 3600,
                                          sensitive=True)
        cmd_ret = ''.join(cmd_ret)
        if "Upgraded successfully" not in cmd_ret:
            logger.error("self upgrade failed, ret: %s" % str(cmd_ret))
            err_msg = "deploy manage self upgrade failed. "
            raise HCCIException(621003, err_msg)
        elif "same version upgrade forbidden" in cmd_ret:
            logger.warn("the deploy manage version is th same")
        else:
            logger.info("exec deploy manager self upgrade success")

    def _deploymanager_self_upgrade_for_81(self, *args):
        ssh_client, node_username, node_password, root_password, \
        remote_username, remote_password, remote_root_password = args
        if ErrorInfo.ERROR_PACKAGE_DIR_NOT_EXIST == \
                self._check_dir_exist(ssh_client, "/opt/fusionstorage/", "selfupgrade") and \
                ErrorInfo.ERROR_SAME_VERSION == \
                self._check_deploymanager_self_upgrade_81(ssh_client):
            logger.info("The version is same, return")
            self._check_self_upgrade_result(ssh_client)
            return
        status = self._check_result_for_retry(ssh_client)
        if self.single_mode:
            cmd_ret = self.single_self_upgrade(ssh_client, status)
        else:
            data_tuple = namedtuple('data_tuple', ['ssh_client', 'status', 'node_username',
                                                   'node_password', 'root_password', 'remote_username',
                                                   'remote_password', 'remote_root_password'])
            data_info = data_tuple(ssh_client, status, node_username, node_password,
                                   root_password, remote_username, remote_password,
                                   remote_root_password)
            cmd_ret = self.ha_self_upgrade(data_info)
        cmd_ret = ''.join(cmd_ret)
        if "Succeeded in performing the upgrade" in cmd_ret:
            logger.info("exec deploy manager self upgrade success")
        else:
            logger.error("self upgrade failed, ret: %s" % str(cmd_ret))
            echo_str = "]#"
            log_cmd = "tail -n 100 /var/log/DeployManagerScreen.log"
            cmd_ret = ssh_client.send_cmd(log_cmd, echo_str, 62)
            logger.error("self upgrade err info: %s" % str(cmd_ret))
            err_msg = "deploy manage self upgrade failed. "
            raise HCCIException(621003, err_msg)
        self._check_self_upgrade_result(ssh_client)

    def _deploymanager_self_upgrade(self, *args):
        """
            deploy manager self-upgrade
        """
        ssh_client, node_ip, node_username, node_password, root_password, \
        remote_username, remote_password, remote_root_password, self_upgrade_action_dir = args
        if self.sys_opr.check_pkg_version(self.fs_args) == 80:
            logger.info("start self upgrade in %s for 8.0" % node_ip)
            self._deploymanager_self_upgrade_for_80(
                *[ssh_client, node_username, node_password, root_password,
                  remote_username, remote_password, remote_root_password])
        else:
            logger.info("start self upgrade in %s for 8.2" % node_ip)
            self._deploymanager_self_upgrade_for_81(
                *[ssh_client, node_username, node_password, root_password,
                  remote_username, remote_password, remote_root_password])

        if ErrorInfo.ERROR_IS_STANDBY == self._check_ha_status(ssh_client) and not self.single_mode:
            if node_ip == self.master_ip:
                remote_ip = self.slaver_ip
            else:
                remote_ip = self.master_ip
            ssh_client = StorageSSHClient(remote_ip, remote_username, remote_password)
            ssh_client.switch_root(remote_root_password)
        SshPublicMethod().check_status_opr(ssh_client)

        clear_component_not_upgrade_path = os.path.join(
            self_upgrade_action_dir, "clear_component_not_upgrade.sh")
        if os.path.isfile(clear_component_not_upgrade_path):
            self_clear_cmd = "sh clear_component_not_upgrade.sh" \
                             " clear_component"
            cmd_ret = ssh_client.send_cmd(self_clear_cmd, '#')
            cmd_ret = ''.join(cmd_ret)
            if "failed" in cmd_ret:
                logger.error("deploy manage self upgrade clear failed, "
                             "ret: %s " % str(cmd_ret))

    def _check_ha_status(self, ssh_client):
        cmd_ret = self._ssh_cmd_get_ha_status(ssh_client)
        if "Abnormal" in str(cmd_ret):
            logger.error("status abnormal, %s" % ''.join(cmd_ret))
            err_msg = "ha status abnormal"
            raise HCCIException(621003, err_msg)
        return self._get_ha_standby(ssh_client)

    def _get_ha_standby(self, ssh_client):
        cmd_ret = self._ssh_cmd_get_ha_mode(ssh_client, "HA_MODE")
        if "single" in str(cmd_ret):
            logger.error("single mode, %s " % ''.join(cmd_ret))
            self.single_mode = True
            return ErrorInfo.ERROR_IS_STANDBY
        else:
            logger.info("double mode, %s " % ''.join(cmd_ret))
            cmd_ret = self._ssh_cmd_get_ha_mode(ssh_client, "LOCAL_ROLE")
            logger.info("cmd_ret={}".format(cmd_ret))
            if "active" in str(cmd_ret):
                logger.info("get current node is active")
                return ErrorInfo.ERROR_IS_MASTER
            if "standby" in str(cmd_ret):
                logger.info("get current node is standby")
                return ErrorInfo.ERROR_IS_STANDBY
        logger.error("dont get current node mode")
        return ErrorInfo.ERROR_IS_MASTER

    def _check_deploymanager_self_upgrade(self, ssh_client):
        cmd_ret = self._ssh_cmd_get_tmout(ssh_client)
        cmd_ret = ''.join(cmd_ret)
        if "last_result=0" in cmd_ret:
            logger.info(cmd_ret)
            self.upgrade_mode = "without_TMOUT_check_for_hcsu"
        else:
            logger.error(cmd_ret)
            cmd_ret = self._ssh_cmd_read_tmout(ssh_client)
            tm_out = int(cmd_ret[1])
            if 0 < tm_out < 3600:
                raise HCCIException(
                    621008, "TMOUT={} is too short".format(tm_out))

        cmd_ret = self._ssh_cmd_check_dup_upg(ssh_client)
        cmd_ret = ''.join(cmd_ret)
        if "deploymanager_no_self_upgrade" in cmd_ret:
            logger.info("no self upgrade, continue.")
        elif "deploymanager_self_upgrade_running" in cmd_ret:
            logger.error("self upgrade is running, ret: %s" % cmd_ret)
            err_msg = "self upgrade is running"
            raise HCCIException(621011, err_msg)
        else:
            err_msg = "Not support check_duplicate_upgrade param"
            logger.error(err_msg)

    def _clear_self_upgrade_status_file(self, node_ip, node_username, node_password, root_password):
        self_upgrade_status_file = "/tmp/.self_upgrade_status"
        try:
            ssh_client = StorageSSHClient(node_ip, node_username, node_password)
            logger.info('Create ssh client for %s success' % node_ip)
            ssh_client.switch_root(root_password)
            cmd_ret = self._ssh_cmd_rm_file(ssh_client, self_upgrade_status_file)
            cmd_ret = ''.join(cmd_ret)
            if "last_result=0" in cmd_ret:
                err_msg = "remove file {file_name}".format(file_name=self_upgrade_status_file)
                logger.info(err_msg)
            else:
                err_msg = "remove file {}, failed:{}".format(self_upgrade_status_file, cmd_ret)
                logger.error(err_msg)
        finally:
            logger.info("clear self upgrade status file:%s in node %s" % (self_upgrade_status_file, node_ip))
