#!/usr/bin/env python3
# -*- coding:UTF-8 -*-
"""
Copyright (c): 2021, Huawei Tech. Co., Ltd.
Description  : A utility to check configurations of servers deployed with gds or roach_client.
"""
try:
    import os
    import re
    import sys
    import getopt
    import time
    from enum import Enum

    sys.path.append(os.path.split(os.path.realpath(__file__))[0] + "/../../script/")
    from gspylib.common.GaussLog import GaussLog
    from gspylib.common.ErrorCode import ErrorCode
    from common_tools import GDSUtils

except ImportError as e:
    sys.exit("[GAUSS-52200] : Unable to import module: %s." % str(e))


# APP_NAME: gds or roach_client
APP_NAME = __file__[(0 if __file__.rfind("/") == -1 else __file__.rfind("/") + 1):__file__.rfind("_")]
WORK_PATH = os.path.dirname(os.path.realpath(__file__))
CONF_FILE = os.path.join(WORK_PATH, "check_os_params.conf")
MAX_HANDLE_STANDARD = 1000000
NET_THRESHOLD = 10000


class CheckStatus(Enum):
    Normal = 0
    Warning = 1
    Abnormal = 2


action = None
ping_list = []
host_list = []
detailed = False
statistics = {'total': 0, CheckStatus.Abnormal.name: 0, CheckStatus.Warning.name: 0, 'install_disallow': False}

ping_list_raw = ""
host_list_raw = ""


def usage():
    """
Usage:
    %s_check -t check --host [/path/to/hostfile | ipaddr1,ipaddr2...] --ping-host\
     [/path/to/pinghostfile | ipaddr1,ipaddr2...] [--detail]
    %s_check -t fix --host [/path/to/hostfile | ipaddr1,ipaddr2...] [--detail]
Common options:
    -t              The operation for this script: check or fix.
    --ping-host     IP addresses of nodes to ping, such as DN, CN, Gateway.
                    Two styles of setting are supported:
                    (1) A String consisting of IP addresses separated with commas, like 192.168.2.100,192.168.2.101.
                    (2) An absolute path to the ip list file, like /opt/sample_user/ping-host.list,\
                     and the content format is as follows:
                        192.168.2.100
                        192.168.2.101
    --host          IP addresses of nodes where %s to be deployed.
                    Two styles of setting are supported:
                    (1) A String consisting of IP addresses separated with commas, like 192.168.1.100,192.168.1.101.
                    (2) An absolute path to the ip list file, like /opt/sample_user/host.list,\
                     and the content format is as follows:
                        192.168.1.100
                        192.168.1.101
    --detail        Show detailed information, only used in check mode.
    -V              Show version information.
    -h|--help       Show help information.
    """
    print(usage.__doc__ % ((APP_NAME,) * 3))


def read_params(argv):
    try:
        opts, args = getopt.getopt(argv, "hVt:", ['help', 'detail', "ping-host=", "host="])
        if len(args) > 0:
            GaussLog.exitWithError(ErrorCode.GAUSS_500['GAUSS_50000'] % str(args[0]))

        global ping_list
        global host_list
        global detailed
        global action
        global ping_list_raw
        global host_list_raw

        for opt, arg in opts:
            if opt in ('-h', "--help"):
                usage()
                sys.exit()
            elif opt == '-t':
                action = arg
            elif opt == '-V':
                GDSUtils.get_version_info(APP_NAME)
                sys.exit(0)
            elif opt == "--host":
                host_list_raw = arg
            elif opt == "--ping-host":
                ping_list_raw = arg
            elif opt == "--detail":
                detailed = True
            else:
                GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50002"] % opt)
    except Exception as e:
        GaussLog.printMessage(ErrorCode.GAUSS_500["GAUSS_50015"] % str(e))
        usage()
        sys.exit()


def check_parameters():
    """
    function: check parameters
    """
    try:
        global ping_list
        global host_list
        global detailed
        global action

        if action not in ["check", "fix"]:
            GaussLog.printMessage(ErrorCode.GAUSS_500['GAUSS_50011'] % ("-t", action))
            usage()
            sys.exit()
        if action == "check":
            ping_list = GDSUtils.get_hosts_from_param(ping_list_raw)
        else:
            ping_list = []
        host_list = GDSUtils.get_hosts_from_param(host_list_raw)
        if len(host_list) == 0:
            GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50012"] % '--host')

    except Exception as e:
        GaussLog.printMessage(ErrorCode.GAUSS_500["GAUSS_50015"] % str(e))
        usage()
        sys.exit()


def init_logger():
    """
    Init logger
    """
    try:
        log_file = os.path.join(WORK_PATH, GDSUtils.CHECK_LOG_FILE)
        my_log = GaussLog(log_file)
        return my_log
    except Exception:
        raise(ErrorCode.GAUSS_502["GAUSS_50206"] % "logfile")


class _OsParameterItems:
    """
    Used for Check OS Parameters. Store a parameter's status.
    """
    def __init__(self, line_raw):
        self.is_valid = True
        self.subject = ""
        self.param_num = []
        self.need_fix = False
        self.check_msg = ""
        self.fix_msg = ""
        self.read_line_raw(line_raw)

    def read_line_raw(self, line_raw):
        """
        read input line from file check_os_params.conf.
        """
        params = line_raw.split('=')
        if len(params) != 2:
            self.is_valid = False
            return
        self.subject = params[0].strip()
        self.param_num = [x.strip() for x in re.split(r'[,;\s]', params[1].strip())]
        return

    def check_item(self, check_link):
        """
        check parameter and store its status. Notice, this function is also used in repair_item, the statistics differ.
        input: check_link is an object of GdsCheckLink, including an paramiko client and other network information.
        """
        # skip invalid input, such as blank lines.
        if not self.is_valid:
            return ""

        if action == "check":
            statistics['total'] += 1

        if self.subject == 'MTU':
            if check_link.netcard:
                cmd = "cat /sys/class/net/%s/mtu" % check_link.netcard
            else:
                # if check_link don't include one node's network card information, won't check or fix MTU.
                if action == "check":
                    statistics[CheckStatus.Warning.name] += 1
                self.check_msg = "Warning: Failed to obtain network interface for %s, won't check MTU.\n"\
                                 % check_link.ip
                return self.check_msg
        else:
            filename = '/proc/sys/' + '/'.join(self.subject.split('.'))
            cmd = "cat %s" % filename

        # get system parameters
        status, output = GDSUtils.paramiko_cmd_raw(check_link.client, cmd)
        if status:
            # result without error
            values_in_os = [x.strip() for x in re.split(r'[,;\s]', output.replace('\n', ''))]
            # result not match
            if self.param_num != values_in_os:
                if action == "check":
                    statistics[CheckStatus.Abnormal.name] += 1
                    statistics['install_disallow'] = True
                self.need_fix = True
                self.check_msg = "Error: %s RealValue: %s ExpectedValue: %s.\n" % \
                                 (self.subject, ','.join(values_in_os), ','.join(self.param_num))
        else:
            # result with error
            if action == "check":
                statistics[CheckStatus.Abnormal.name] += 1
                statistics['install_disallow'] = True
            self.check_msg = (ErrorCode.GAUSS_512["GAUSS_51203"] % self.subject + "\n")
        return self.check_msg

    def repair_item(self, check_link):
        """
        Used in '-t fix' mode.
        """
        conf_position = "/etc/sysctl.conf"
        num = ' '.join(self.param_num)
        newline = (self.subject + ' = ' + num).strip()
        statistics['total'] += 1
        if self.subject == 'MTU':
            if not check_link.netcard:
                statistics[CheckStatus.Abnormal.name] += 1
                self.fix_msg = "Error: Failed to obtain network interface for %s, won't fix MTU.\n" % check_link.ip
                return self.fix_msg
            netcard_conf = check_link.get_network_card_config_path()
            cmd01 = "ip link set dev %s mtu %s;" % (check_link.netcard, num)
            cmd02 = "sed -i '/.*MTU.*/d' %s;" % netcard_conf
            cmd03 = "echo 'MTU=%s' >> %s;" % (num, netcard_conf)
            status_1, output_1 = GDSUtils.paramiko_cmd_raw(check_link.client, cmd01 + cmd02 + cmd03)
            if not status_1:
                statistics[CheckStatus.Abnormal.name] += 1
                self.fix_msg = "Error: %s while fixing MTU for network card %s\n" % (output_1, check_link.netcard)
            else:
                self.fix_msg = "Success: Set MTU to value %s." % num
        else:
            cmd1 = "sed -i '/.*%s.*/d' %s;" % (self.subject, conf_position)
            cmd2 = "echo '%s' >> %s;" % (newline, conf_position)
            status_2, output_2 = GDSUtils.paramiko_cmd_raw(check_link.client, cmd1 + cmd2)
            if not status_2:
                statistics[CheckStatus.Abnormal.name] += 1
                self.fix_msg = "Error: %s while fixing %s\n" % (output_2, self.subject)
            else:
                self.fix_msg = "Success: Set %s to value %s." % (self.subject, num)
        return self.fix_msg


class GdsCheckItem:
    """
    Used for store statistics, and output in both detailed mode and not-detailed mode, of checked items.
    """

    def __init__(self, title):
        self.title_raw = title
        self.title = ""
        self.status = CheckStatus.Normal.name
        self.detailed_msg = ""
        self.count_done = False  # if don't want to run calculate() automatically, set it True.

    def set_status(self, status):
        """
        set status string in output title of checked item.
        """
        self.status = status

    def set_title(self):
        """
        set output title string.
        """
        mylen = len(self.title_raw)
        MAX_TITLE_LENTH = 56
        if mylen < MAX_TITLE_LENTH:
            self.title = (self.title_raw + (MAX_TITLE_LENTH - mylen) * " " + ":" + self.status)
        else:
            self.title = (self.title_raw + ':' + self.status)
        return self.title

    def calculate(self):
        """
        according to the status of the checked item, refresh the statistics.
        """
        if not self.count_done:
            if self.status != CheckStatus.Normal.name:
                statistics[self.status] += 1
        self.count_done = True

    def get_msg(self):
        """
        get output message of the checked item, varied from whether in detailed mode or not.
        """
        if not self.count_done:
            self.calculate()
        self.set_title()
        if detailed:
            return (self.title + '\n' + self.detailed_msg + '\n')
        else:
            return (self.title + '\n')


class GdsCheckLink:
    """
    A set of check or fix method for one node.
    """
    def __init__(self, host):
        self.client = GDSUtils.connect_paramiko_ssh_host(host, "root", _root_passwd)
        self.ip = host
        self.netcard = self.get_netcard()

    def __del__(self):
        try:
            self.client.close()
        except Exception:
            pass

    def get_netcard(self):
        """
        get network card name according to the input ip, return none if network card information cannot find.
        """
        cmd = "ip route | grep %s | awk -F '[ \t*]' '{print $3}'" % self.ip
        status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd)
        if not status or not output:
            raise Exception(ErrorCode.GAUSS_506["GAUSS_50604"] % self.ip)
        return output.strip()

    def check_os_parameter(self):
        """
        check OS parameter
        """
        c_item = GdsCheckItem("[ Check OS Parameters ]")
        # open CONF_FILE
        try:
            with open(CONF_FILE) as f:
                check_objs = f.readlines()
        except Exception:
            statistics['total'] += 1
            statistics['install_disallow'] = True
            c_item.detailed_msg += (ErrorCode.GAUSS_502["GAUSS_50204"] % "config file" + '\n')
            c_item.set_status(CheckStatus.Abnormal.name)
            ret_msg = c_item.get_msg()
            return ret_msg
        # check each line of the CONF_FILE
        c_item.count_done = True
        for obj in check_objs:
            check_obj = _OsParameterItems(obj)
            c_item.detailed_msg += check_obj.check_item(self)

        if "Warning" in c_item.detailed_msg:
            c_item.set_status(CheckStatus.Warning.name)
        elif c_item.detailed_msg != "":
            c_item.set_status(CheckStatus.Abnormal.name)

        ret_msg = c_item.get_msg()
        return ret_msg

    def repair_os_parameter(self):
        """
        repair os parameters in fix mode
        """
        c_item = GdsCheckItem("[ Fix OS Parameters ]")
        try:
            with open(CONF_FILE) as f:
                check_objs = f.readlines()
        except Exception:
            c_item.detailed_msg += (ErrorCode.GAUSS_502["GAUSS_50204"] % "config file" + '\n')
            statistics['total'] += 1
            c_item.set_status(CheckStatus.Abnormal.name)
            ret_msg = c_item.get_msg()
            return ret_msg

        # check each line of the CONF_FILE
        c_item.count_done = True
        need_reload = False
        for obj in check_objs:
            check_obj = _OsParameterItems(obj)
            check_obj.check_item(self)
            if check_obj.need_fix:
                res = check_obj.repair_item(self)
                if "Warning" in res or "Error" in res or "[GAUSS" in res:
                    c_item.set_status(CheckStatus.Abnormal.name)
                c_item.detailed_msg += (res + '\n')
                need_reload = True

        # if repaired item is not none, reload system parameters.
        if need_reload:
            cmd = "/sbin/sysctl -p;"
            GDSUtils.paramiko_cmd_raw(self.client, cmd)

        ret_msg = c_item.get_msg() if need_reload else ""
        return ret_msg

    def check_max_handle(self):
        """
        check max handle of open files.
        """
        c_item = GdsCheckItem("[ Check Max Handle ]")
        install_disallow_before = statistics['install_disallow']

        # if this function is used by repair_max_handle, it will not do statistics.
        if action == "fix":
            c_item.count_done = True
        else:
            statistics['total'] += 1

        status, output = GDSUtils.paramiko_cmd_raw(self.client, "ulimit -n")
        if status:
            res = output.strip()
            if res == "unlimited":
                c_item.set_status(CheckStatus.Normal.name)
            elif not res.isdigit():
                c_item.set_status(CheckStatus.Abnormal.name)
                statistics['install_disallow'] = (action == 'check') or install_disallow_before
                c_item.detailed_msg += "Error: Unexpected value %s, which is not a number." \
                                       " ExpectedValue: Not less than %d open files.\n" % (res, MAX_HANDLE_STANDARD)
            elif int(res) < MAX_HANDLE_STANDARD:
                c_item.set_status(CheckStatus.Abnormal.name)
                statistics['install_disallow'] = (action == 'check') or install_disallow_before
                c_item.detailed_msg += "Error: Max handle %s ExpectedValue: Not less than %d open files.\n"\
                                       % (res, MAX_HANDLE_STANDARD)
            else:
                c_item.set_status(CheckStatus.Normal.name)
        else:
            c_item.set_status(CheckStatus.Abnormal.name)
            statistics['install_disallow'] = (action == 'check') or install_disallow_before
            c_item.detailed_msg += "Error: Failed to read max value of handle information.\n"

        ret_msg = c_item.get_msg()
        return ret_msg

    def repair_max_handle(self, amount=MAX_HANDLE_STANDARD):
        """
        repair max handle amount of open files in fix mode.
        """

        check_result = self.check_max_handle()
        if (":" + CheckStatus.Normal.name) in check_result:
            return ""

        c_item = GdsCheckItem("[ Fix Max Handle ]")
        statistics['total'] += 1
        limit_path = '/etc/security/limits.d/'
        if os.path.isfile(os.path.join(limit_path, '91-nofile.conf')):
            limit_file = os.path.join(limit_path, '91-nofile.conf')
        else:
            limit_file = os.path.join(limit_path, '90-nofile.conf')
        cmd = "sed -i '/.*nofile.*/d' %s;" % limit_file
        cmd += "echo '*    soft    nofile    %d' >> %s;" % (amount, limit_file)
        cmd += "echo '*    hard    nofile    %d' >> %s;" % (amount, limit_file)
        cmd += "echo 'root    soft    nofile    %d' >> %s;" % (amount, limit_file)
        cmd += "echo 'root    hard    nofile    %d' >> %s;" % (amount, limit_file)

        status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd)
        if not status:
            c_item.set_status(CheckStatus.Abnormal.name)
            c_item.detailed_msg += "Error: Failed to set max value of open file handle to %d\n" % amount

        ret_msg = c_item.get_msg()
        return ret_msg

    def check_storage_module(self, subject, command):
        """
        analyzing module, used for "df -h" and "df -i"
        """
        c_item = GdsCheckItem("[ Check %s ]" % subject)
        statistics['total'] += 1
        check_status = CheckStatus.Normal.name
        status, output = GDSUtils.paramiko_cmd_raw(self.client, command)
        if status:
            df = output.split('\n')
            for line in df[1:]:
                if line.strip() == "":
                    continue
                msg = line.split()
                if not msg[0].startswith("/dev/"):
                    continue
                if msg[-2].endswith('%'):
                    p = int(msg[-2][:-1])
                else:
                    p = 0
                if p == 0:
                    c_item.detailed_msg += "%s %s not used\n" % (subject, msg[0])
                elif p >= 70:
                    c_item.detailed_msg += "Warning: %s %s used %s\n" % (subject, msg[0], msg[-2])
                    check_status = CheckStatus.Warning.name
            c_item.set_status(check_status)
        else:
            c_item.set_status(CheckStatus.Warning.name)
            c_item.detailed_msg = "Warning: Failed to read %s information.\n" % subject
        ret_msg = c_item.get_msg()
        return ret_msg

    def check_storage(self):
        ret = ""
        ret += self.check_storage_module("storage", "df -h")
        ret += self.check_storage_module("inode", "df -i")
        return ret

    def check_ping(self):
        """
        check ping
        """
        # if ping_list is empty, do nothing.
        if len(ping_list) == 0:
            return ""
        else:
            c_item = GdsCheckItem("[ Check Ping ]")
            statistics['total'] += 1
            ping_status = {CheckStatus.Warning.name: False, CheckStatus.Abnormal.name: False}
            for p in ping_list:
                status, output = GDSUtils.paramiko_cmd_raw(self.client, "ping -c 5 -q %s" % p)
                if status:
                    res = output.replace('\r', '').split('\n')
                    statistic_str = re.findall(r'transmitted,(.*)received', res[3])
                    if len(statistic_str) == 0:
                        ping_status[CheckStatus.Abnormal.name] = True
                        statistics['install_disallow'] = True
                        c_item.detailed_msg += "Error: Failed to read statistics while ping %s.\n" % p
                    else:
                        success_num = statistic_str[0].strip()
                        if not success_num or success_num[0] == "0":
                            ping_status[CheckStatus.Abnormal.name] = True
                            statistics['install_disallow'] = True
                            c_item.detailed_msg += "Error: Ping %s failed, %s. \n" % (p, res[3])
                        elif success_num[0] in [1, 2, 3, 4]:
                            ping_status[CheckStatus.Warning.name] = True
                            c_item.detailed_msg += "Warning: Ping %s: %s, %s\n" % (p, res[3], res[4])
                        else:
                            c_item.detailed_msg += "Success: Ping %s: %s, %s\n" % (p, res[3], res[4])
                else:
                    ping_status[CheckStatus.Abnormal.name] = True
                    statistics['install_disallow'] = True
                    c_item.detailed_msg += "Error: Ping %s failed.\n" % p

            if ping_status[CheckStatus.Abnormal.name]:
                c_item.set_status(CheckStatus.Abnormal.name)
            elif ping_status[CheckStatus.Warning.name]:
                c_item.set_status(CheckStatus.Warning.name)
            ret_msg = c_item.get_msg()
            return ret_msg

    def get_network_card(self):
        """
        get network card information and bonding status
        including team/slave network cards
        used for CPU_IRQ Multi Queue Binding check
        """
        network_cards = []
        try:
            network = self.netcard
            if network:
                network_cards.append(network)
            # network config file has two type, suse11 and redhat different.
            conf_file = self.get_network_card_config_path()

            cmd = "if [ -f %s ]; then echo 0; else echo 1; fi" % conf_file
            status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd)
            if output.strip() == "1":
                bond_mode = "BondMode Null"
            else:
                bonding_conf_file = "/proc/net/bonding/" + network
                cmd = "if [ -f %s ]; then echo 0; else echo 1; fi" % bonding_conf_file
                status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd)
                bonding_conf_file_exist = False if output.strip() == "1" else True

                cmd = "grep -i 'BONDING_OPTS\|BONDING_MODULE_OPTS' %s" % conf_file
                status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd)
                # Analysis results
                res = output
                bondmode_info = "BondMode Null"
                if res:
                    if (res.find("mode") > 0) and bonding_conf_file_exist:
                        cmd = "grep -w '\<Bonding Mode\>' %s |awk  -F ':' '{print $NF}'" % bonding_conf_file
                        status_1, output_1 = GDSUtils.paramiko_cmd_raw(self.client, cmd)
                        bondmode_info = "BondMode %s" % output_1
                    else:
                        raise Exception(ErrorCode.GAUSS_506["GAUSS_50611"] % res)
                elif bonding_conf_file_exist:
                    cmd = "grep -w '\<Bonding Mode\>' %s |awk  -F ':' '{print $NF}'" % bonding_conf_file
                    status_1, output_1 = GDSUtils.paramiko_cmd_raw(self.client, cmd)
                    bondmode_info = "BondMode %s" % output_1
                    bondmode_info += "\nNo 'BONDING_OPTS' or 'BONDING_MODULE_OPTS' in bond config file[%s]." % conf_file
                else:
                    bondmode_info = "BondMode Null"
                bond_mode = bondmode_info

            if not network or not bond_mode or not conf_file:
                raise Exception(ErrorCode.GAUSS_506["GAUSS_50611"])
            elif bond_mode != "BondMode Null":
                # get network cards in bonding mode
                bond_file = '/proc/net/bonding/%s' % network
                cmd = "cat %s | grep 'Slave Interface' | awk -F : '{print $2}'" % bond_file
                status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd)
                if not status or not output:
                    raise Exception(ErrorCode.GAUSS_506["GAUSS_50611"] + "Failed to execute get_network_card.")
                else:
                    bond_slaves = [x.strip() for x in output.split("\n")]
                    network_cards.extend(bond_slaves)
                    return network_cards
            else:
                # get network cards in team mode.
                cmd = "cat %s | grep -E 'DEVICETYPE=Team|TEAM_CONFIG' | wc -l" % conf_file
                status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd)
                flag = status and (output.strip() != "0")
                if flag:
                    cmd = "export PATH=/bin:/usr/local/bin:/usr/bin:/sbin:/usr/sbin:$PATH && teamdctl %s stat" % network
                    status_2, output_2 = GDSUtils.paramiko_cmd_raw(self.client, cmd)
                    team_info = [x.strip() for x in output_2.split('\n')]
                    network_cards.extend(team_info)
                return network_cards
        except Exception as e:
            GaussLog.printMessage("[%s] %s" % (self.ip, str(e)))
            raise e

    def is_network_card_level_higher_than_threshold(self, netcard, threshold):
        """
        judge whether a network card speed is higher than input threshold.
        used for check CPU-IRQ Multi Queue binding.
        """
        cmd_get_speed = "/sbin/ethtool %s | grep 'Speed:'" % netcard
        status_0, output_0 = GDSUtils.paramiko_cmd_raw(self.client, cmd_get_speed)
        if not status_0:
            return False
        lines = output_0.split('\n')
        speedstr = ""
        if len(lines) > 0:
            for line in lines:
                if line.find("Speed:") >= 0:
                    speedstr = line
                    break
        if speedstr.find("Speed:") >= 0 and speedstr.find("Mb/s") >= 0:
            net_level = int(speedstr.split(':')[1].strip()[:-4])
            if net_level >= threshold:
                return True
        return False

    def check_multi_queue(self):
        """
        check CPU-IRQ multi queue
        """
        c_item = GdsCheckItem("[ Check CPU Multi Queue Binding ]")

        # if this function is used by repair_multi_queue, it will not do statistics.
        if action == "fix":
            c_item.count_done = True
        else:
            statistics['total'] += 1

        if not self.netcard:
            c_item.set_status(CheckStatus.Warning.name)
            c_item.detailed_msg += "Warning: Failed to obtain network interface for %s, won't check Multi-Queue.\n"\
                                   % self.ip
            ret_msg = c_item.get_msg()
            return ret_msg
        netcards = self.get_network_card()

        total_status = CheckStatus.Normal.name
        for netcard in netcards:
            line_list = []
            if self.is_network_card_level_higher_than_threshold(netcard, NET_THRESHOLD):
                # get irq information
                cmd = "for i in `cat /proc/interrupts | grep '%s' | awk -F ' ' '{print $1}' | " \
                      "awk -F ':' '{print $1}'`; do cat /proc/irq/$i/smp_affinity ; done" % netcard
                status_1, output_1 = GDSUtils.paramiko_cmd_raw(self.client, cmd)
                irq_file = output_1.strip()
                if not status_1 or not irq_file or len(irq_file.splitlines()) == 0:
                    total_status = CheckStatus.Warning.name
                    c_item.detailed_msg += "Warning: Failed to obtain network card [%s] interrupt value. " % netcard
                    continue

                # calculate percentage of CPU which is bind with irq, return normal
                try:
                    line_list = _parse_interrupt(irq_file)
                    statistic_line = "Binded CPU: "\
                                  + str(round(len(set(line_list)) * 100 / len(irq_file.splitlines()), 2)) \
                                  + "% standard:over 80%."
                    if len(irq_file.splitlines()) * 0.8 > len(set(line_list)):
                        total_status = CheckStatus.Warning.name
                        c_item.detailed_msg += "Warning: %s Network card [%s] multi-queue support is disabled.\n" %\
                                      (statistic_line, netcard)
                    else:
                        c_item.detailed_msg += "Success: %s Network card [%s] multi-queue support is enabled.\n" % \
                                      (statistic_line, netcard)
                    continue
                except Exception as ex:
                    total_status = CheckStatus.Warning.name
                    c_item.detailed_msg += "Warning: Failed to parse the [%s]'s interrupt value. Reason: %s.\n" \
                                  "Output:%s\n" % (netcard, str(ex), output_1)
                    continue
            else:
                total_status = CheckStatus.Warning.name
                c_item.detailed_msg += "Warning: The speed of current card \"%s\" is less than %s Mb/s." \
                              "Or Failed to obtain the network card [%s] speed value. \n"  \
                              % (netcard, NET_THRESHOLD, netcard)
                continue
        c_item.set_status(total_status)
        ret_msg = c_item.get_msg()
        return ret_msg

    def get_network_card_config_path(self):
        """
        get network card config path, differs from os system.
        """
        net_card_conf = "/etc/sysconfig/network-scripts/ifcfg-%s" % self.netcard
        cmd = "if [ -f %s ]; then echo 0; else echo 1; fi" % net_card_conf
        status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd)
        if output.strip() == "1":
            net_card_conf = "/etc/sysconfig/network/ifcfg-%s" % self.netcard
        return net_card_conf

    def repair_multi_queue(self):
        """
        repair CPU-IRQ multi queue binding in fix mode.
        """
        check_result = self.check_multi_queue()
        if (":" + CheckStatus.Normal.name) in check_result:
            return ""

        c_item = GdsCheckItem("[ Fix CPU Multi Queue Binding ]")
        try:
            statistics['total'] += 1
            if not self.netcard:
                raise Exception("Warning: Failed to obtain network interface for %s, won't fix Multi-Queue.\n"
                                % self.ip)

            network_cards = self.get_network_card()
            cmd = "ps ax | grep -v grep | grep -q irqbalance; echo $?"
            status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd)
            if status and output.strip() == "0":
                GDSUtils.paramiko_cmd_raw(self.client, "killall irqbalance")
            else:
                raise Exception("Warning: Failed to obtain irqbalance for %s, won't fix Multi-Queue.\n" % self.ip)

            total_status = CheckStatus.Normal.name
            for network_cardnum in network_cards:
                if not self.is_network_card_level_higher_than_threshold(network_cardnum, NET_THRESHOLD):
                    total_status = CheckStatus.Warning.name
                    c_item.detailed_msg += "Warning: Network card %s speed less than 10000Mbps. " \
                                           "Binding process canceled.\n" % network_cardnum
                    continue
                cmd = "cat /proc/interrupts | grep '%s' | wc -l" % network_cardnum
                status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd)
                period = int(output.strip()) if str(output.strip()).isdigit() else 0

                counter = 0
                for i in range(period):
                    # the dev name type like this: eth1-1, eth1-rx-1, eth1-tx-1, eth1-TxRx-1
                    # eth1-rx1, eth-tx1 in arm, get all network name interrupt
                    cmd_irq = "cat /proc/interrupts | grep '%s' | awk -F ' ' '{print $1}' |" \
                              " awk -F ':' '{print $1}'|awk 'NR==%s'" % (network_cardnum, str(i + 1))
                    status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd_irq)
                    if not status or output.strip() == "":
                        total_status = CheckStatus.Warning.name
                        # Failed to obtain current IRQ value, continue to next.
                        continue
                    else:
                        irq = output.strip()
                        num = 2 ** i
                        # Under SuSE platform, when the length is greater than 8, the ',' must be used.
                        value = str(hex(num))[2:]
                        # Decimal 63 or more long number sending in L
                        if len(value) > 16 and value[-1] == 'L':
                            value = value[:-1]
                        result_value = ''
                        while len(value) > 8:
                            result_value = ",%s%s" % (value[-8:], result_value)
                            value = value[:-8]
                        result_value = "%s%s" % (value, result_value)

                        cmd_set = "echo '%s'> /proc/irq/%s/smp_affinity" % (result_value, irq)
                        status, output = GDSUtils.paramiko_cmd_raw(self.client, cmd_set)

                        if not status:
                            total_status = CheckStatus.Warning.name
                            # Failed to set the current IRQ, continue to next
                        else:
                            counter += 1
                if counter == 0:
                    total_status = CheckStatus.Warning.name
                    c_item.detailed_msg += "Warning: Failed to set any CPU IRQ binding to network %s\n"\
                                           % network_cardnum
                elif counter * 100 / period >= 80:
                    c_item.detailed_msg += "Success: Set CPU bind with network %s: %d total: %d\n"\
                                           % (network_cardnum, counter, period)
                else:
                    total_status = CheckStatus.Warning.name
                    c_item.detailed_msg += "Warning: Set CPU bind with network %s: %d" \
                                           " total: %d Still below Standard\n" % (network_cardnum, counter, period)
            c_item.set_status(total_status)
            ret_msg = c_item.get_msg()
            return ret_msg
        except Exception as e:
            c_item.set_status(CheckStatus.Warning.name)
            c_item.detailed_msg += (str(e) + "\n")
            ret_msg = c_item.get_msg()
            return ret_msg


def _parse_interrupt(interrupt_lines):
    """
    used for CPU-IRQ multi queue checking and fixing.
    """
    line_list = []
    for index in range(len(interrupt_lines.splitlines()[0]) - 1):
        for line in interrupt_lines.splitlines():
            line = line.replace(",", "")
            if not line.lstrip("0") or line.find("f") < 0:
                line_list.append(line.lstrip("0"))
    return line_list


def clean_resource():
    """
    function: clean_resource, close files and so on.
    """
    try:
        my_log.closeLog()
        os.remove(my_log.tmpFile)
    except Exception:
        pass


def do_check(host):
    """
    function: Check all items, and print check results, done by super user.
    """
    try:
        ret = "\n================Check %s Result================\n" % host
        my_client = GdsCheckLink(host)
        ret += my_client.check_os_parameter()
        ret += my_client.check_max_handle()
        ret += my_client.check_storage()
        ret += my_client.check_ping()
        ret += my_client.check_multi_queue()
        return {host: ret}
    except Exception as e:
        raise Exception("[%s] %s" % (host, str(e)))


def do_fix(host):
    """
    function: Fix items which can be fixed, such as OS parameters, max handler, netcard multi-queue, done by super user.
    """
    try:
        ret = "\n================Fix %s Result================\n" % host
        ret0 = ""
        my_client = GdsCheckLink(host)
        ret0 += my_client.repair_os_parameter()
        ret0 += my_client.repair_max_handle()
        ret0 += my_client.repair_multi_queue()
        if not ret0:
            ret0 = "Everything OK. Nothing to fix."
        ret += ret0
        return {host: ret}
    except Exception as e:
        raise Exception("[%s] %s" % (host, str(e)))


def __login_auth():
    try:
        root_passwd = GDSUtils.get_user_password("root")
        root_passwd = GDSUtils.two_more_chances_for_passwd(host_list, "root", root_passwd)
        return root_passwd
    except Exception:
        GaussLog.exitWithError(ErrorCode.GAUSS_503["GAUSS_50306"] % "root" + " Or network is abnormal")


def print_title():
    if action == "check":
        print("============Start Checking============")
    elif action == "fix":
        print("============Start Fixing============")


def do_in_parallel():
    parallel_result = {}
    GaussLog.printMessage("Performing system %s. Output the result to the log file %s. Please wait."
                          % (action, my_log.logFile))

    ping_wait = len(ping_list) * 5
    if action == "check":
        parallel_result = GDSUtils.exec_command_remote_parallelly(do_check, host_list, GDSUtils.TIMEOUT * 2 + ping_wait)
    elif action == 'fix':
        parallel_result = GDSUtils.exec_command_remote_parallelly(do_fix, host_list, GDSUtils.TIMEOUT * 2)

    GaussLog.printMessage("%s program is completed." % action)
    statistic_line = "Total numbers: %d. Abnormal numbers: %d. Warning number: %d." % \
                     (statistics['total'], statistics[CheckStatus.Abnormal.name], statistics[CheckStatus.Warning.name])
    GaussLog.printMessage(statistic_line)
    log_result(parallel_result, statistic_line)


def log_result(parallel_result, statistic_line):
    result_all = ""
    try:
        for host, result in parallel_result.items():
            result_all += result

        if action == "check":
            mypid = os.getpid()
            mytime = int(time.time())

            statistic_log_line = "[Check_Process %d][%d] %s [Install_Disallow_%s]\n" % \
                                 (mypid, mytime, statistic_line, statistics['install_disallow'])
            my_log.debug(statistic_log_line)

        if detailed:
            my_log.log(result_all)
        else:
            my_log.debug(result_all)
    except Exception as e:
        GaussLog.printMessage(result_all)
        raise e


if __name__ == '__main__':
    """
    main function
    """
    try:
        GDSUtils.load_gds_env()
        read_params(sys.argv[1:])
        check_parameters()
        print_title()
        _root_passwd = __login_auth()
        my_log = init_logger()
        do_in_parallel()
    except Exception as e:
        if str(e).find("[GAUSS-") != -1:
            GaussLog.exitWithError(str(e))
        GaussLog.exitWithError(ErrorCode.GAUSS_535["GAUSS_53514"] % str(e))
    finally:
        clean_resource()
