#!/usr/bin/env python
# ***************************************************************************
# Copyright 2022-2025 VMware, Inc.  All rights reserved. VMware Confidential.
# ***************************************************************************

"""
This script is a drop-in replacement for curl.  The purpose of this script is
to be a common entry point for the validation of server certs from client
code making API calls to the NSX Manager.  For security reasons, client code
should validate the remote (server) cert when making API connections. This
script validates the certificates as defined by
https://vmw-confluence.broadcom.net/display/NSBU/X509+Certificate+Validations
This script acts as a drop-in replacement for curl in the sense that it
supports the same options as curl, and returns the same error messages and
exit codes.  Not all curl options are supported. The script doesn't reject
any options that it doesn't support but silently ignores them.  This script
supports IPv4 and IPv6.  An example IPv6 curl command is: curl_wrapper -u
"admin:password" -k -i --thumbprint
6D:4B:5C:FE:F6:EA:C3:0D:EC:28:7A:E1:31:3C:F7:59:E4:35:1B:6A:6C:6E:6C:67:64:AE:A5:A8:BF:C0:9F:D8
https://[fd01:0:106:209:0:a:0:1cc0]/api/v1/node/aaa/providers/vidm
The thumbprint can be provided in upper or lower case, and with or without
the colons.
Errata: curl_wrapper currently supports at most one -d (--data) option.
When multiple -d options are provided curl_wrapper doesn't complain and
sends the last -d option.
"""


from __future__ import print_function
import sys
import ssl
import importlib
import time
import argparse
import socket
import struct
import json
import subprocess
import traceback
import tempfile
import uuid
import os
import random
import string
import mimetypes
from os.path import exists
from datetime import datetime
from datetime import timedelta
from base64 import b64encode
from xml.dom import minidom
from binascii import hexlify

import warnings
warnings.filterwarnings("ignore")  # for CryptographyDeprecationWarning

have_crypto = False
have_cryptography = False
try:
    from cryptography import x509                                  # noqa: E402
    from cryptography.x509.extensions import BasicConstraints      # noqa: E402
    from cryptography.x509.oid import ExtendedKeyUsageOID          # noqa: E402
    from cryptography.hazmat.primitives.asymmetric import rsa      # noqa: E402
    from cryptography.hazmat.primitives.asymmetric import ec       # noqa: E402
    from cryptography.hazmat.primitives.asymmetric import padding  # noqa: E402
    from cryptography.exceptions import InvalidSignature           # noqa: E402
    from cryptography.hazmat.backends import default_backend       # noqa: E402
    from cryptography.hazmat.primitives import hashes              # noqa: E402
    from cryptography.hazmat.primitives import serialization       # noqa: E402
    from cryptography.x509.oid import NameOID                      # noqa: E402
    have_cryptography = True
except ImportError:
    pass

if not have_cryptography:
    try:
        from OpenSSL import crypto  # noqa
        have_crypto = True
    except ImportError:
        pass

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

sys.path.append("/usr/lib/vmware/nsx-common/lib/python")
sys.path.append("/opt/vmware/nsx-monitoring/python")
sys.path.append("/opt/vmware/nsx-common/python")
have_nsx = False
try:
    from vmware.nsx.rpc import NsxRpcClient  # noqa
    from vmware.nsx.rpc import NsxRpcConnection  # noqa
    from vmware.nsx.messaging.applproxyinfo_pb2 import ApplProxyInfoService_Stub  # noqa
    from vmware.nsx.messaging.applproxyinfo_pb2 import ApplProxyInfoReqMsg  # noqa
    from vmware.nsx.certificate.certificate_service_pb2 import CertificateService_Stub  # noqa
    from vmware.nsx.certificate.certificate_service_pb2 import CheckTrustedRequestMsg  # noqa
    from vmware.nsx.certificate.certificate_service_pb2 import CheckTrustedResponseMsg  # noqa
    have_nsx = True
except ImportError:
    # This code path indicates that either NSX has not yet been installed
    # or we don't have the necessary version of NSX.
    pass

have_py3 = sys.version_info >= (3,)
httplib_module = "http.client" if have_py3 else "httplib"
httplib = importlib.import_module(httplib_module)
if have_py3:
    from subprocess import TimeoutExpired
    unicode = str


class LocalLogger:
    """Logging wrapper class so this script works on plain Ubuntu.
    """
    def __init__(self):
        """Constructor checks if nsx_logging library is available.
           If not then don't log to log files.
        """
        self.logger = None
        try:
            import nsx_logging  # pylint: disable=C0415
            self.logger = nsx_logging.getLogger(__name__)
            nsx_logging.basicConfig(syslog=True, subcomp="curl_wrapper")
            self.logger.setLevel(nsx_logging.INFO)
        except ImportError:
            self.logger = None
        if (not self.logger and exists("/etc/issue") and not
                exists(NSX_ISSUE)):
            # This is saying if this script is running on an Ubuntu system
            # (/etc/issue is present) but not on an NSX node (/etc/nsx_issue is
            # not present), then write logs to /var/log/syslog without using
            # NSX's nsx_logger and write logs to /var/log/syslog.log on ESX
            # system.
            try:
                # Copied from /usr/lib/vmware/vsan/bin/vsan-config.py on ESXi
                # 8.0.0 build 21203435.
                # Tested on ESXi 9.0.0 build 24957456.
                # Tested on ESXi 8.0.0 build 21203435.
                # Tested on ESXi 7.0.3 build 18644231.
                # Tested on ESXi 6.7.0 build 14320388.
                import logging  # pylint: disable=C0415
                import logging.handlers  # pylint: disable=C0415
                self.logger = logging.getLogger('curl_wrapper')
                self.logger.setLevel(logging.INFO)
                fmt = '%(asctime)s %(name)s[%(process)d]: %(message)s'
                datefmt = "%b %d %H:%M:%S"
                formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)
                handler = logging.handlers.SysLogHandler(address='/dev/log')
                handler.setFormatter(formatter)
                self.logger.addHandler(handler)
            except ImportError:
                self.logger = None

    def info(self, msg, *args):
        """Similar signature as info() in Lib/logging/__init__.py
           Don't include kwargs because this script doesn't use kwargs.
        """
        if self.logger:
            self.logger.info(msg, *args)


class NsxZeroize:
    """Wrapper class so that nsx_zeroize code doesn't fail in test environments
       like build servers.
    """
    def __init__(self):
        """Constructor checks if nsx_zeroize library is available.
           If not then don't zeroize the passed variable data.
        """
        self.nsx_zeroize = None
        if not exists(NSX_ISSUE) or not have_nsx:
            # On an Ubuntu dev system with source code, this script will import
            # nsx_zeroize, but it won't work properly because the associated
            # .so library is likely not installed. So don't attempt to import
            # nsx_zeroize in this case.
            return
        try:
            sys.path.append('/opt/vmware/nsx-common/python/nsx_utils')
            import nsx_zeroize  # pylint: disable=C0415
            self.nsx_zeroize = nsx_zeroize
        except ImportError:
            self.nsx_zeroize = None

    def zeroize(self, data):
        """If nsx_zeroize library is available try to
           zeroize the passed data.
        """
        if self.nsx_zeroize:
            self.nsx_zeroize.zeroize(data)


# curl error codes.
# See https://curl.se/libcurl/c/libcurl-errors.html.  The error codes with
# THIS_ preprended are deprecated in libcurl and being overloaded by this
# script.
CURLE_OK = 0
CURLE_FAILED_INIT = 2
CURLE_URL_MALFORMAT = 3
CURLM_INTERNAL_ERROR = 4
CURLE_COULDNT_CONNECT = 7
CURLE_PARTIAL_FILE = 18
CURLE_READ_ERROR = 26
CURLE_OPERATION_TIMEDOUT = 28
CURLE_SSL_CONNECT_ERROR = 35
THIS_WAS_REDIRECTED = 46
CURLE_TOO_MANY_REDIRECTS = 47
THIS_NO_ALTERNATIVE_CERTIFICATE_SUBJECT_NAME = 51
CURLE_GOT_NOTHING = 52
THIS_CRL_CHECK_FAILED = 53
CURLE_PEER_FAILED_VERIFICATION = 60
CURLE_USE_SSL_FAILED = 64
CURLE_SSL_CACERT_BADFILE = 77
THIS_KEYBOARD_INTERRUPT = 130

# Global variables
INVALID_HTTP_CODE = 0
ALGO_RSA = "RSA"
ALGO_EC = "EC"
ALGO_UNKNOWN = "unknown"
TYPE_EC = 408
NODE_TYPE_UNKNOWN = "unknown"
_NODE_TYPE = NODE_TYPE_UNKNOWN
DEFAULT_MAX_REDIRECTS = 50
MAX_EMPTY_READS = 3
NSX_ISSUE = "/etc/nsx_issue"
CURLE_URL_MALFORMAT_ERRSTR = ("URL using bad/illegal format or "
                              "missing URL")
CURLE_READ_ERRSTR = "Failed to open/read local data from file/application"
CURLE_OPTION_D_ERRSTR = "option -d: error encountered when reading a file"
REQUIRES_CRYPTOGRAPHY_ERRSTR = ("curl_wrapper requires the Python "
                                "cryptography or crypto package")
REQUIRES_OPENSSL_ERRSTR = "curl_wrapper requires openssl"
SERVER_VALIDATION_ERRSTR = ("SSL certificate problem: Server certificate "
                            "validation failed")
CACERT_BADFILE_ERRSTR = "error setting certificate file: "
CRL_CHECK_WARNING = "unable to perform CRL check"
CONNECTION_TIMED_OUT_MSG = 'Connection timed out'
CURL_WRAPPER_TAG = 'curl_wrapper'
NUM_WRITE_OUT_WORDS = 3
APPLIANCE_INFO = "/etc/vmware/nsx/appliance-info.xml"
EXCEPTION_TIMEDOUT_MSG = 'timedout'
DEFAULT_TIMEOUT = 10
REST_NODE_TYPES = ['global-manager']
NODE_TYPES_WITH_PROTON = ['nsx-manager', 'global-manager']
ESX_NODE_TYPES = ['nsx-esx', 'nsx-esxio']
VMWARE_CMD = "/bin/vmware"
OBFUSCATE_STR = "*****"
AUTHORIZATION_HDR = "Authorization"
UTF8 = 'utf-8'
ASCII = 'ascii'
CD_FORM_DATA = 'form-data'
CD_ATTACHMENT = 'attachment'
CT_MULTIPART_FORM_DATA = 'multipart/form-data'
HDR_CONTENT_TYPE = 'Content-Type'

lg = LocalLogger()
nz = NsxZeroize()


def get_node_type():
    """Retrieve node type.
       This is a stripped-down version of nsx_utils.node_utils.get_node_type.
       This function was added so that this script is self contained.
    """

    global _NODE_TYPE
    if _NODE_TYPE != NODE_TYPE_UNKNOWN:
        return _NODE_TYPE
    nsx_issue_node_type = NODE_TYPE_UNKNOWN
    try:
        with open(NSX_ISSUE) as fo:
            lines = fo.readlines()
        for line in lines:
            parts = line.split(":", 1)
            if parts[0].strip() == "node-type" and len(parts) > 1:
                nsx_issue_node_type = parts[1].strip()
                break
    except Exception:
        nsx_issue_node_type = NODE_TYPE_UNKNOWN
    _NODE_TYPE = nsx_issue_node_type
    return nsx_issue_node_type


def _is_esx():
    """Returns True if this script is running on ESXi and False
       otherwise. This function is designed to work in cases where the
       /etc/nsx_issue file is not yet present and in such cases get_node_type()
       returns NODE_TYPE_UNKNOWN.
    """
    node_type = get_node_type().split()[0]
    if node_type in REST_NODE_TYPES:
        return False
    if node_type in ESX_NODE_TYPES:
        return True
    # check if the /bin/vmware binary is present
    return os.path.exists(VMWARE_CMD)


def _get_esx_version():
    """Returns the esx version as provided by /bin/vmware -v.
       If either /bin/vmware is not present or the version can't
       be determined then return None. For example, on ESX 7.0.3,
       return "7.0.3".
    """
    if not _is_esx():
        return None
    cmd_args = [VMWARE_CMD, "-v"]
    try:
        df = subprocess.Popen(cmd_args, stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        output = df.communicate()[0]
        return str(output).split()[2]
    except Exception as ex:
        submsg = str(ex)
        errstr = "The command " + VMWARE_CMD + " -v failed: " + submsg
        _log_backtrace(None, errstr,
                       traceback.format_exc())
        return None


def _is_appliance_info_valid():
    """Function to determine whether this current node is registered with the
       NSX Manager.  Returns True if the current node is registered and False
       otherwise.
    """
    if not exists(APPLIANCE_INFO):
        return False
    try:
        doc = minidom.parse(APPLIANCE_INFO)
        applianceInfo = doc.getElementsByTagName('appliance-proxy')
        if not len(applianceInfo) > 0:
            return False
        return True
    except Exception:
        pass
    return False


class CmdLineOptions:
    """Command line options.
    """
    def __init__(self):
        self.host = None
        self.port = None
        self.url = None
        self.path = None  # Not an option but parsed from url
        self.cacert = None
        self.cacert_is_ca = None
        self.cert = None
        self.connect_timeout = None
        self.data = None
        self.form = None
        self.head = None
        self.header = None
        self.include = None
        self.insecure = None
        self.key = None
        self.location = None
        self.max_redirs = None
        self.max_time = None
        self.no_hostname_check = None
        self.output = None
        self.remote_name = None
        self.request = None
        self.retry = None
        self.retry_delay = None
        self.retry_max_time = None
        self.silent = None
        self.show_error = None
        self.thumbprint = None
        self.upload_file = None
        self.user = None
        self.validate_cert_from_file = None
        self.verbose = None
        self.write_out = None
        self.trust_store = {}       # Not an option. Store for leaf certs.
        self.num_retry_conns = -1   # Not an option. Count of connections.
        self.errcode = CURLE_OK     # Not an option. Store last errcode.
        self.errstr = ""            # Not an option. Store last errstr.
        self.prev_stdout_line = ""  # Not an option. Store last stdout line.
        self.last_curl_fin = False  # Not an option. Has the last curl command
        # finished.


def _get_host_opt(extra_args):
    """Extract host, port, path, url fields from the command line arguments.
       Return the tuple (errcode, errstr, host, port, path, url) where errcode
       and errstr mimic curl's response.
    """
    host = None
    port = None
    path = None
    url = None
    prev_s = None
    errcode = CURLE_URL_MALFORMAT  # assume this error until proven otherwise
    errstr = CURLE_URL_MALFORMAT_ERRSTR
    for s in extra_args:
        if s.lower().startswith('https://') or s.lower().startswith('http://'):
            if prev_s != '-x':
                try:
                    url_parts = urlparse(s)
                    host = url_parts.hostname
                    port = url_parts.port
                    path = url_parts.path
                    url = s
                except ValueError:
                    return (CURLE_URL_MALFORMAT, errstr, None, None, None,
                            None)
        prev_s = s
    if host:
        if not port:
            port = 80 if url_parts.scheme == "http" else 443
        return (CURLE_OK, None, host, port, path, url)
    if url:
        errcode = CURLE_URL_MALFORMAT
        errstr = CURLE_URL_MALFORMAT_ERRSTR
    else:
        errcode = CURLE_FAILED_INIT
        errstr = "no URL specified!"
    return (errcode, errstr, None, None, None, None)


def _get_cacert_opt(parsed_args):
    """Extract the --cacert option from the command line arguments.  The
       cacert option is the filename containing a PEM encoded cert.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'cacert') and v:
            return v
    return None


def _get_cacert_is_ca_opt(parsed_args):
    """Extract the --cacert_is_ca option from the command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'cacert_is_ca') and v:
            return v
    return None


def _get_cert_opt(parsed_args):
    """Extract the --cert option from the curl command line arguments.  The
       cert option is the filename containing a PEM encoded key.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'cert') and v:
            return v
    return None


def _get_connect_timeout_opt(parsed_args):
    """Extract the --connect-timeout option from the curl command line
       arguments.  Note that the code below uses connect_timeout with an
       underscore rather than a hyphen, but that is just how the argparse
       library works.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'connect_timeout') and v:
            return int(v)
    return 0


def _get_data_opt(parsed_args):
    """Extract the --data (-d) option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'd' or k == 'data') and v:
            return v
    return None


def _get_form_opt(parsed_args):
    """Extract the --form (-F) option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'F' or k == 'form') and v:
            # Returns a list
            return v
    return None


def _get_head_opt(parsed_args):
    """Extract the --head option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'I' or k == 'head') and v:
            return True
    return False


def _get_key_opt(parsed_args):
    """Extract the --key option from the curl command line arguments.  The
       key option is the filename containing a PEM encoded key.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'key') and v:
            return v
    return None


def _get_location_opt(parsed_args):
    """Extract the --location option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'L' or k == 'location') and v:
            return True
    return False


def _get_max_redirs_opt(parsed_args):
    """Extract the --max-redirs option from the curl command line arguments.
       Note that the code below uses max_redirs with an underscore rather than
       a hyphen, but that is just how the argparse library works.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'max_redirs') and v:
            return int(v)
    return None


def _get_max_time_opt(parsed_args):
    """Extract the --max-time option from the curl command line arguments.
       Note that the code below uses max_time with an underscore rather than a
       hyphen, but that is just how the argparse library works.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'm' or k == 'max_time') and v:
            return int(v)
    return 0


def _get_method_opt(parsed_args):
    """Extract the --request option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'X' or k == 'request') and v:
            return v
    return None


def _get_no_hostname_check_opt(parsed_args):
    """Extract the --no-hostname-check option from the curl command line
       arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'no_hostname_check') and v:
            return True
    return False


def _get_output_file_opt(parsed_args):
    """Extract the --output option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'o' or k == 'output') and v:
            return v
    return None


def _get_remote_name_opt(parsed_args):
    """Extract the --remote-name option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'O' or k == 'remote_name') and v:
            return True
    return False


def _get_request_headers_opt(parsed_args):
    """Extract the --header option from the curl command line arguments.
    """
    hdrs = {}
    for k1, v1 in parsed_args.__dict__.items():
        if (k1 == 'H' or k1 == 'header') and v1:
            for item in v1:
                k2, v2 = item.split(':')
                hdrs.update({k2.strip(): v2.strip()})
    return hdrs


def _get_response_headers_opt(parsed_args):
    """Extract the --include option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'i' or k == 'include') and v:
            return True
    return False


def _get_retry_opt(parsed_args):
    """Extract the --retry option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if k == 'retry' and v:
            return int(v)
    return 0


def _get_retry_delay_opt(parsed_args):
    """Extract the --retry-delay option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'retry_delay') and v:
            return int(v)
    return 0


def _get_retry_max_time_opt(parsed_args):
    """Extract the --retry-max-time option from the curl command line
       arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'retry_max_time') and v:
            return int(v)
    return 0


def _get_show_error_opt(parsed_args):
    """Extract the --show-error option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'S' or k == 'show_error') and v:
            return True
    return False


def _get_silent_opt(parsed_args):
    """Extract the --silent option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 's' or k == 'silent') and v:
            return True
    return False


def _get_thumbprint_opt(parsed_args):
    """Extract the --thumbprint option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if k == 'thumbprint' and v:
            return v
    return 0


def _get_upload_opt(parsed_args):
    """Extract the --upload-file (-T) from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'T' or k == 'upload_file') and v:
            return v
    return None


def _get_user_passwd_opt(parsed_args):
    """Extract the --user option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'u' or k == 'user') and v:
            return v
    return None


def _get_verbose_opt(parsed_args):
    for k, v in parsed_args.__dict__.items():
        if (k == 'v' or k == 'verbose') and v:
            return True
    return False


def _get_write_out_opt(parsed_args):
    """Extract the --write-out option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'w' or k == 'write_out') and v:
            return v
    return None


def _get_leaf_cert(cert_chain):
    """Return the leaf cert from the chain.  The leaf cert is the first cert in
       the chain.
    """
    return cert_chain[0]


def _get_last_cert(cert_chain):
    """Return the last cert from the chain.  The leaf cert is on one side of
       the chain and the last cert is on the other. The last cert may or may
       not be the logical root of the chain.
    """
    return cert_chain[-1]


def _log_trying_connection(options, host, port, with_httplib=True):
    """Curl prints the "Trying" message to stderr in verbose mode.
       Trying 10.185.21.158:443...
       On NSX the this script prints to stderr and to syslog.
    """
    with_str = "(with httplib)" if with_httplib else "(with curl)"
    logmsg = ("Trying " + with_str + " " + host + ":"
              + str(port) + "...")
    lg.info(logmsg)
    _print_msg_stderr(options, "*   " + logmsg)


def _log_command(options, cmd_args_in,  # pylint: disable=W0613
                 obfuscate_secrets=False):
    """Log commands invoked by this script.  Typically cmd_args will be an
       array passed to subprocess.Popen().  No secrets should be logged and in
       cases where there are secrets in command arguments, this function should
       be called with obfuscate_secrets=True.
    """
    cmd_args = cmd_args_in
    if obfuscate_secrets:
        cmd_args = list(cmd_args_in)
        # Shallow copy. Therefore don't zeroize items with secrets.
        # Credentials are stored in the -u command line option and in the
        # Authorization header.
        i = 0
        while i < len(cmd_args):
            if cmd_args[i] == '-u' and i < len(cmd_args)-1:
                if ':' in cmd_args[i+1]:
                    parts = cmd_args[i+1].split(':')
                    cmd_args[i+1] = parts[0] + ':' + OBFUSCATE_STR
            elif cmd_args[i] == '-H' and i < len(cmd_args)-1:
                if (AUTHORIZATION_HDR.lower() in cmd_args[i+1].lower() and
                        ':' in cmd_args[i+1]):
                    # The Authorization header looks like:
                    # Authorization: 'Basic %s' % username_passwd
                    parts = cmd_args[i+1].split(':')
                    parts[0] = parts[0].strip()
                    part1 = parts[1]
                    if 'basic' in part1.lower() or 'remote' in part1.lower():
                        words = part1.split()
                        cmd_args[i+1] = (parts[0] + ': ' + words[0] + ' ' +
                                         OBFUSCATE_STR)
                    else:
                        cmd_args[i+1] = parts[0] + ': ' + OBFUSCATE_STR
            i += 1
    cmd_str = str(cmd_args)
    if len(cmd_str) > 0 and cmd_str[0] == '[':
        cmd_str = cmd_str[1:]
    if len(cmd_str) > 0 and cmd_str[-1] == ']':
        cmd_str = cmd_str[:-1]
    if len(cmd_str) > 0:
        lg.info("Calling " + cmd_str)


def _log_exit_code(options, exitcode):  # pylint: disable=W0613
    lg.info(sys.argv[0] + " exit code " + str(exitcode))


def _log_connection_timed_out(options, logmsg):
    """Log curl's "Connection timed out" message
    """
    lg.info(logmsg)
    _print_msg_stderr(options, "* " + logmsg)


def _log_closing_connection(options):
    """Log curl's "Closing connection" stderr message.
    """
    logmsg = ("Closing connection "
              + str(options.num_retry_conns))
    lg.info(logmsg)
    _print_msg_stderr(options, "* " + logmsg)


def _log_transient_problem(options, sleep_time, retry):
    """Log curl's "Warning: Transient problem:" stderr message.
    """
    logmsg = ("Warning: Transient problem:  Will retry in " + str(sleep_time)
              + " seconds. " + str(retry) + " retries left.")
    lg.info(logmsg)
    _print_msg_stderr(options, logmsg)


def _log_backtrace(options, logmsg, backtrace):  # pylint: disable=W0613
    """Log backtrace for debugging purposes.  Curl doesn't log this message so
       this script writes it to the log file, not stderr.
    """
    logmsg = "Logging backtrace for analysis: " + logmsg + " " + backtrace
    lg.info(logmsg)


def _log_command_timedout(options, logmsg):  # pylint: disable=W0613
    """Log openssl timed out message.  Curl doesn't log this message so this
       script writes it to the log file, not stderr.
    """
    lg.info(logmsg)


def _log_cert_verification_result(options, logmsg):
    """Log result of cert verification.  Curl doesn't log this message but it
       is important so we also write it to stderr.
    """
    lg.info(logmsg)
    _print_msg_stderr(options, "* " + logmsg)


def _cert_has_expired(cert):
    """Check if the cert has expired.
    """
    if have_cryptography:
        try:
            # timezone not available in Python2
            from datetime import timezone   # pylint: disable=C0415
            return cert.not_valid_after_utc <= datetime.now(timezone.utc)
        except Exception:
            pass
        return cert.not_valid_after <= datetime.now()
    # There is a bug in the has_expired() function in the python
    # OpenSSL.crypto.X509 library used on ESX versions 6, 7, and 8 and
    # therefore we use our own logic in this function.  This function is
    # similar to OpenSSL.crypto.X509.has_expired() in that it returns True if
    # the cert has expired and False otherwise.  More recently the /bin/vmware
    # command has been deprecated so we have removed esx-specific logic.
    # Tested on VMware ESXi 6.7.0 build-14320388.
    # Tested on VMware ESXi 7.0.3 build-18644231.
    # Tested on VMware ESXi 8.0.0 build-21203435.
    if not exists('/usr/bin/curl'):
        # This is an approximate way of determine whether running on ESX.
        not_after = cert.get_notAfter().decode(UTF8)
        # get_notAfter returns time as ASN.1 TIME YYYYMMDDhhmmssZ
        not_after_date = datetime.strptime(not_after, '%Y%m%d%H%M%SZ')
        return not_after_date <= datetime.now()
    return cert.has_expired()


def _get_signature_algorithm(cert):
    """ Return the algorithm name.
    """
    if have_cryptography:
        algo = cert.signature_algorithm_oid
    else:
        algo = cert.get_signature_algorithm().decode(UTF8)
    if "RSA" in str(algo):
        return ALGO_RSA
    if "EC" in str(algo):
        return ALGO_EC
    return ALGO_UNKNOWN


def _get_pubkey_algorithm(cert):
    if have_cryptography:
        pubkey = cert.public_key()
        if isinstance(pubkey, rsa.RSAPublicKey):
            return ALGO_RSA
        if isinstance(pubkey, ec.EllipticCurvePublicKey):
            return ALGO_EC
        return ALGO_UNKNOWN
    pubkey = cert.get_pubkey()
    if pubkey.type() == crypto.TYPE_RSA:
        return ALGO_RSA
    if pubkey.type() == TYPE_EC:
        return ALGO_EC
    return ALGO_UNKNOWN


def _get_pubkey_size(cert):
    """Get size of public key in bits.
    """
    if have_cryptography:
        pubkey = cert.public_key()
        return pubkey.key_size
    pubkey = cert.get_pubkey()
    return pubkey.bits()


def _validate_common(cert):
    """Used to validate the leaf cert.  This function is called by both
       _validate_self_signed_cert and _validate_ca_signed_cert and handles all
       common validation logic for these two cert validation functions.
       Returns the tuple (errcode, errstr).  If the cert is validated
       successfully, this function returns (0, None).  Otherwise this function
       returns a non-zero errcode and a non-None errstr.  This function is
       used to validate the leaf cert only.
    """
    if _cert_has_expired(cert):
        return (CURLE_PEER_FAILED_VERIFICATION, "certificate has expired")
    algo = _get_pubkey_algorithm(cert)
    if algo == ALGO_RSA:
        if _get_pubkey_size(cert) < 2048:
            return (CURLE_PEER_FAILED_VERIFICATION,
                    "RSA certificate key length less than 2048")
    elif algo == ALGO_EC:
        if _get_pubkey_size(cert) < 256:
            return (CURLE_PEER_FAILED_VERIFICATION,
                    "EC certificate key length less than 256")
    else:
        return (CURLE_PEER_FAILED_VERIFICATION,
                "certificate neither RSA nor EC")
    return 0, None


def _canonical_thumbprint(thumbprint):
    """Converts to canonical form so that thumbprints can be compared.
    """
    return thumbprint.lower().replace(':', '')


def _get_thumbprint(cert):
    """Utility to extract thumbprint as a string from the given certificate.
    """
    if have_cryptography:
        return hexlify(cert.fingerprint(hashes.SHA256())).decode(UTF8)
    return cert.digest("sha256").decode(UTF8)


def _get_subject(cert):
    """Get the cert subject.
    """
    if have_cryptography:
        return cert.subject
    return cert.get_subject()


def _get_issuer(cert):
    """Get the cert issuer.
    """
    if have_cryptography:
        return cert.issuer
    return cert.get_issuer()


def _get_basic_constraints(cert):
    """Return (is_ca, path length) from the cert basic contraint extensions.
    """
    if have_cryptography:
        try:
            extns = cert.extensions
            bc_extn = extns.get_extension_for_class(BasicConstraints)
            basic_constraints = bc_extn.value
            is_ca = basic_constraints.ca
            path_length = basic_constraints.path_length
            return is_ca, path_length
        except x509.ExtensionNotFound:
            pass
        return None, None

    is_ca = None
    path_length = None
    for i in range(cert.get_extension_count()):
        ext = cert.get_extension(i)
        if ext.get_short_name() == b'basicConstraints':
            for entry in ext.__str__().split(","):
                if 'CA:TRUE' in entry:
                    is_ca = True
                elif 'CA:FALSE' in entry:
                    is_ca = False
                if 'pathlen' in entry:
                    attribs = entry.split(':')
                    if len(attribs) > 0:
                        try:
                            path_length = int(attribs[1])
                        except ValueError:
                            pass
    return is_ca, path_length


def _validate_cert_chain_eku(cert_chain, options):  # pylint: disable=W0613
    """Checks if the leaf certificate in the chain has SERVER_AUTH bit set in
       the EKU section.  Since curl_wrapper is used as a client mostly, we will
       check for SERVER_AUTH bit only.
    """
    leaf_cert = cert_chain[0]
    if have_cryptography:
        try:
            eku_extension = leaf_cert.extensions.get_extension_for_class(
                x509.ExtendedKeyUsage)
            server_auth_oid = ExtendedKeyUsageOID.SERVER_AUTH.dotted_string
            lg.info("EKU: server_auth: " + server_auth_oid)

            if eku_extension:
                for usage in eku_extension.value:
                    if usage.dotted_string == server_auth_oid:
                        return CURLE_OK, None
        except x509.ExtensionNotFound:
            pass
        return (CURLE_PEER_FAILED_VERIFICATION,
                SERVER_VALIDATION_ERRSTR + ": missing extendedKeyUsage "
                "in the certificate.")

    for i in range(leaf_cert.get_extension_count()):
        ext = leaf_cert.get_extension(i)
        if ext.get_short_name() == b'extendedKeyUsage':
            if 'Server Authentication' in ext.__str__():
                return CURLE_OK, None
    return (CURLE_PEER_FAILED_VERIFICATION,
            SERVER_VALIDATION_ERRSTR + ": missing extendedKeyUsage "
            "in the certificate.")


def _validate_basic_constraints_for_leaf(cert,
                                         options):  # pylint: disable=W0613
    """Validate certificate basic constraints for leaf cert.
    """
    if not options.cacert:
        return CURLM_INTERNAL_ERROR, None
    is_ca, _ = _get_basic_constraints(cert)
    if is_ca:
        lg.info("Basic Constraints extension CA is set to TRUE "
                "in the leaf certificate of the chain.")
        return (CURLE_PEER_FAILED_VERIFICATION,
                SERVER_VALIDATION_ERRSTR + ": CA:TRUE set in the leaf "
                "certificate.")
    return CURLE_OK, None


def _validate_basic_constraints_for_nonleaf(cert,
                                            options,  # pylint: disable=W0613
                                            cert_chain=None, index=None):
    """Validate certificate basic constraints for non-leaf cert.
    """
    if not options.cacert:
        return CURLM_INTERNAL_ERROR, None
    is_ca, path_length = _get_basic_constraints(cert)
    if is_ca is None:
        return (CURLE_PEER_FAILED_VERIFICATION,
                SERVER_VALIDATION_ERRSTR + ": missing basicConstraints "
                "in the certificate.")
    if not is_ca:
        return (CURLE_PEER_FAILED_VERIFICATION,
                SERVER_VALIDATION_ERRSTR + ": CA:FALSE set in the "
                "issuing CA certificate.")

    # cert_chain and index are None when checking basic constraints
    # for --cacert.
    if cert_chain is not None and index is not None:
        # No need to check path_length of root CA certificate.
        if path_length is not None and (index < len(cert_chain) - 1):
            # path_length should be 0 or greater for non-leaf certs.
            # path_length should be 0 for the intermediate CA cert which
            # is the issuer of leaf certificate. If some other cert in the
            # chain has path_length as 0 that means chain is not proper.
            if path_length < 0 or (path_length == 0 and index != 1):
                return (CURLE_PEER_FAILED_VERIFICATION,
                        "SSL certificate problem: Certificate chain "
                        "issue: validation failed: one of the certificate "
                        "in the chain has incorrect path_length set.")
    return CURLE_OK, None


def _validate_basic_constraints_for_nonleaf2(cert_found, options):
    """A wrapper over _validate_basic_contraints_for_nonleaf.
       The difference between nonleaf and nonleaf2 functions
       is that nonleaf is used to validate the remote service cert
       while nonleaf2 is used to validate the --cacert cert, and
       the latter requires a different error message.
    """
    errcode, errstr = _validate_basic_constraints_for_nonleaf(
        cert_found, options)
    if errcode:
        # Translate the errstr
        if "missing" in errstr:
            errstr = "missing basicConstraints in cacert."
        elif "CA:FALSE" in errstr:
            errstr = ("CA:FALSE was found in the cacert "
                      "whereas CA:TRUE is expected.")
        return CURLE_SSL_CACERT_BADFILE, errstr
    return CURLE_OK, None


def _validate_cert_chain_expiry(cert_chain):
    """Check expiry for each cert in the cert_chain.
    """
    for cert in cert_chain:
        if _cert_has_expired(cert):
            return (CURLE_PEER_FAILED_VERIFICATION,
                    ("The cert " + _get_subject(cert) +
                     " has expired"))
    return CURLE_OK, None


def _validate_cert_chain_basic_constraints(cert_chain, options):
    """Checks if the proper basic constraints are set for each certificate in
       the chain.  The leaf certificate should NOT have 'CA: True' bit set.
       The root and intermediate CA certificates must have the 'CA: True' bit
       set.
    """
    if not options.cacert:
        return CURLM_INTERNAL_ERROR, None
    for index in range(len(cert_chain)):
        cert = cert_chain[index]
        if index == 0:
            errcode, errstr = _validate_basic_constraints_for_leaf(cert,
                                                                   options)
        else:
            errcode, errstr = \
                _validate_basic_constraints_for_nonleaf(cert, options,
                                                        cert_chain, index)
        if errcode:
            return errcode, errstr
    return CURLE_OK, None


def _write_cert_to_tmp_file(cert, file_cache=None):
    """Write cert to a temporary file.
       The optional file_cache contains the mapping from cert to temporary file
       name.  The existance of an entry in the file_cache implies that the file
       already exists with the cert contents in PEM encoded data.  The cert
       parameter is a single cert, which this function will write as a PEM file
       to a temporary file.  The function returns the filename of this
       temporary file. The optional file_cache parameter is a dictionary where
       the key is a cert, and the value is the temporary filename. In case the
       same cert is being written to the file-system in subsequent calls, this
       function simply returns the temporary filename, without re-writing the
       cert to a new temporary file.
    """
    if not file_cache or cert not in file_cache:
        _, tmp_filename = tempfile.mkstemp()
        if file_cache is not None:
            file_cache[cert] = tmp_filename
        with open(tmp_filename, "w") as f:
            f.write(_dump_cert_as_pem(cert))
    else:
        tmp_filename = file_cache[cert]
    return tmp_filename


def _write_trusted_to_tmp_file(options, cert_chain, index):
    """Construct trusted file to contain all certs in cacert
       store plus all certs from index+1 to root in the cert_chain. If
       next_cert is in the cert chain then it is at position index+1.  If
       next_cert is not in the cert chain, then it is already in the cacert
       store and we don't need to add it in tmp_file_trusted.  If the cacert
       store is large this could be optimized by using file copy and append
       commands instead of reading in the cacerts and writing them out again.
"""
    if not options.cacert:
        return CURLM_INTERNAL_ERROR, None
    _, tmp_filename = tempfile.mkstemp()
    bundle = _read_certs_from_file(options.cacert)
    for i in range(index, len(cert_chain)):
        bundle.append(cert_chain[i])
    _write_certs_to_file(bundle, tmp_filename)
    return tmp_filename


def _verify_signature(index, current_cert, next_cert, cert_chain, options,
                      file_cache=None):
    """Verify certificate signature.
       The current_cert is at index in the cert chain.  next_cert is either
       towards the root of the cert chain, or it might one of the CA certs in
       the CA store (--cacert).
    """
    if not options.cacert:
        return CURLM_INTERNAL_ERROR, None
    if have_cryptography:
        # use cryptography library
        try:
            key_type = _get_signature_algorithm(current_cert)
            if key_type == ALGO_RSA:
                next_cert.public_key().verify(
                    current_cert.signature,
                    current_cert.tbs_certificate_bytes,
                    padding.PKCS1v15(),
                    current_cert.signature_hash_algorithm)
                lg.info("Certificate at index: " + str(index) +
                        " is signed by the next in the chain.")
            else:
                next_cert.public_key().verify(
                    current_cert.signature,
                    current_cert.tbs_certificate_bytes,
                    ec.ECDSA(current_cert.signature_hash_algorithm))
            return CURLE_OK, None
        except InvalidSignature:
            pass
        lg.info("Certificate at index: " + str(index) + " is NOT " +
                "verifiably signed by the next cert in the chain")
        return (CURLE_PEER_FAILED_VERIFICATION,
                "SSL certificate problem: Invalid certificate chain.")

    # Unable to have crypto library to verify signatures so use openssl
    # directly. One problem is that openssl behaves differently on different
    # ESXi versions.
    tmp_file_trusted = None
    file_untrusted = _write_cert_to_tmp_file(current_cert, file_cache)
    if _is_esx():
        esx_version = _get_esx_version()
        if int(esx_version.split('.')[0]) <= 7:
            # Least preferred compared to openssl commands below because it
            # requires that the real root of the cert chain be present in
            # cacert.
            tmp_file_trusted = _write_trusted_to_tmp_file(options, cert_chain,
                                                          index+1)
            cmd_args = ["openssl", "verify",  "-CAfile", tmp_file_trusted,
                        file_untrusted]
        else:
            # ESXi 8.X.X.
            # Note that ESXi 9.0.0 has the cryptography package and is handled
            # by the have_cryptography code block.
            file_trusted = _write_cert_to_tmp_file(next_cert, file_cache)
            cmd_args = ["openssl", "verify", "-no-CApath",
                        "-no-CAfile", "-trusted", file_trusted,
                        "-partial_chain", file_untrusted]
    else:
        file_trusted = _write_cert_to_tmp_file(next_cert, file_cache)
        cmd_args = ["openssl", "verify", "-no_check_time", "-no-CApath",
                    "-no-CAfile", "-trusted", file_trusted, "-partial_chain",
                    file_untrusted]
    try:
        returncode, _, stderr = _call_command(options, cmd_args,
                                              None, True)
        if returncode:
            if returncode == 127:
                return CURLE_FAILED_INIT, REQUIRES_OPENSSL_ERRSTR
            lines = stderr.splitlines()
            errmsg = lines[1] if len(lines) > 1 else lines[0]
            errmsg = errmsg.decode(ASCII)
            return (CURLE_PEER_FAILED_VERIFICATION,
                    "SSL certificate problem: Invalid certificate chain. " +
                    errmsg)
        return CURLE_OK, None
    finally:
        if tmp_file_trusted:
            os.remove(tmp_file_trusted)


def _validate_cert_against_cabundle(index, current_cert, cacert_bundle,
                                    cert_chain, options, file_cache):
    """Check if we can find any cert in the cabundle that has issued
       the current_cert.
       Returns (errcode, errstr, cert_found).
    """
    try:
        r_bundle = [cacert for cacert in cacert_bundle
                    if _get_subject(cacert) == _get_issuer(current_cert)]
        count = 0
        errcode = -1
        errstr = "unknown"
        for ca_cert in r_bundle:
            count += 1
            errcode, errstr = _verify_signature(index, current_cert, ca_cert,
                                                cert_chain, options,
                                                file_cache)
            if not errcode:
                return CURLE_OK, None, ca_cert
        if count:
            return errcode, errstr, None
    except Exception:
        pass
    return (CURLE_PEER_FAILED_VERIFICATION,
            "SSL certificate problem: Invalid certificate chain. "
            "Can't find trusted chain rooted in CA store.", None)


def _validate_cert_chain_order(cert_chain, options):
    """ Validate if the certificate chain is OK.  We do this by checking if
        each certificate is being signed by the next one in the chain.  The
        caller ensures that options.cacert is populated.  Returns (errcode,
        errstr, cert_found).
    """
    if not options.cacert:
        # Shouldn't have called without options.cacert.
        # We can't reliably perform this check unless provided with the
        # --cacert option.
        return CURLM_INTERNAL_ERROR, None, None
    file_cache = {}
    try:
        cacert_bundle = _read_certs_from_file(options.cacert)
        for index in range(len(cert_chain)-1):
            current_cert = cert_chain[index]
            next_cert = cert_chain[index+1]
            errcode, errstr = _verify_signature(index, current_cert, next_cert,
                                                cert_chain, options,
                                                file_cache)
            if errcode:
                return errcode, errstr, None

        # next_cert is the last cert and current_cert is the one before that.

        # Check if any cert in the CA cert store issued the current_cert.
        errcode, errstr, cert_found = \
            _validate_cert_against_cabundle(index, current_cert,
                                            cacert_bundle, cert_chain,
                                            options, file_cache)
        if not errcode:
            return errcode, None, cert_found

        # Advance to last cert in the chain and repeat.
        index += 1
        current_cert = cert_chain[index]
        # return errcode, errstr, cert_found
        return _validate_cert_against_cabundle(index, current_cert,
                                               cacert_bundle, cert_chain,
                                               options, file_cache)
    except Exception:
        pass
    finally:
        # Delete temp files
        for tmp_filename in file_cache.values():
            if exists(tmp_filename):
                os.remove(tmp_filename)
    return (CURLE_PEER_FAILED_VERIFICATION, "Internal error in verifying "
            "the certificate chain.", None)


def _validate_trust_with_thumbprint(cert_chain, options):
    """Establishes trust based on the cert chain received by the server
       and the given thumbprint from the --thumbprint option.  Returns the
       tuple (errcode, errstr).  If trust can be established, this function
       returns (0, None).  Otherwise this function returns a non-zero errcode
       and an non-None errstr.
    """
    if not options.thumbprint:
        return CURLM_INTERNAL_ERROR, None
    leaf_cert = _get_leaf_cert(cert_chain)
    digest = _get_thumbprint(leaf_cert)
    if (_canonical_thumbprint(options.thumbprint) !=
            _canonical_thumbprint(digest)):
        return (CURLE_PEER_FAILED_VERIFICATION,
                ("curl_wrapper failed to verify the legitimacy of the "
                 "server because the given thumbprint " +
                 options.thumbprint + " "
                 "didn't match the certificate's " +
                 _canonical_thumbprint(digest) + "."))
    return 0, None


def _validate_trust_with_cacerts_for_selfsigned(leaf_cert, options):
    if not options.cacert:
        return CURLM_INTERNAL_ERROR, None, None

    cacert_bundle = _read_certs_from_file(options.cacert)
    r_bundle = [cacert for cacert in cacert_bundle
                if _get_subject(cacert) == _get_subject(leaf_cert)]
    digest = _canonical_thumbprint(_get_thumbprint(leaf_cert))
    for cert in r_bundle:
        if _canonical_thumbprint(_get_thumbprint(cert)) == digest:
            return CURLE_OK, None, cert
    return (CURLE_PEER_FAILED_VERIFICATION,
            ("curl_wrapper failed to verify the legitimacy of the server "
             "because the server's certificate with thumbprint " + digest +
             " wasn't found in the CA cert store."), None)


def _validate_self_signed_cert(cert_chain, options):
    """Validate a self-signed cert. The cert_chain parameter is used for
       consistency, but only a single cert is expected (and enforced in this
       function) in the cert chain.  Returns the tuple (errcode, errstr).  If
       cert is validated successfully, this function returns (0, None).
       Otherwise this function returns a non-zero errcode and a non-None
       errstr.
    """
    leaf_cert = _get_leaf_cert(cert_chain)
    errcode, errstr = _validate_common(leaf_cert)
    if errcode:
        return errcode, errstr
    if len(cert_chain) > 1:
        return (CURLE_PEER_FAILED_VERIFICATION,
                "found self-signed certificate with certificate chain")
    if options.thumbprint:
        errcode, errstr = _validate_trust_with_thumbprint(cert_chain, options)
        if errcode:
            return errcode, errstr
    if options.cacert:
        errcode, errstr, cert_found = \
            _validate_trust_with_cacerts_for_selfsigned(leaf_cert, options)
        if errcode:
            return errcode, errstr
        if options.cacert_is_ca:
            # If cacert_is_ca option is enabled then do same basic constraint
            # checks as we would a root cert.
            errcode, errstr = \
                _validate_basic_constraints_for_nonleaf2(cert_found, options)
    return errcode, errstr


class SimpleParseContext:
    """Used by validate_hostname to validate hostnames. Use a "context" object
       to avoid global variables.
    """
    def __init__(self):
        self.errstr = None
        self.data = None
        self.ptr = 0
        self.parsed_data = []


def _read_octet(context):
    """Used by validate_hostname to validate hostnames.  Returns the next
       token in the ASN.1 string.  For a description of ASN.1 see
       https://luca.ntop.org/Teaching/Appunti/asn1.html
    """
    try:
        octet = context.data[context.ptr]
        # For Python 2 context.data is a string whereas for Python 3 it is a
        # buffer. Ensure that an integer is returned in both cases.
        if not have_py3:
            # Python 2
            octet = ord(octet)
        context.ptr += 1
        return octet
    except IndexError:
        return -1


def _read_length(context):
    """Used by validate_hostname to validate hostnames.  Returns the tuple
       (length of the ASN.1 object, length of the ASN.1 object including
       header).
    """
    octet1 = _read_octet(context)
    if octet1 < 0:
        return (-1, -1)
    if octet1 < 0x80:
        # + 1 because we read 1 octet: octet1
        return (octet1, octet1 + 1)
    # 0x80 exactly: means the length is "indefinite" (not supported here)
    if octet1 == 0x81:  # means the length is stored in one octet
        octet2 = _read_octet(context)
        if octet2 < 0:
            return (-1, -1)
        value = octet2
        # + 2 because we read 2 octets: octet1, octet2
        return (value, value + 2)
    if octet1 == 0x82:  # means the length is stored in two octets
        octet2 = _read_octet(context)
        if octet2 < 0:
            return (-1, -1)
        octet3 = _read_octet(context)
        if octet3 < 0:
            return (-1, -1)
        value = octet2 * 256 + octet3
        # + 3 because we read 3 octets: octet1, octet2, octet3
        return (value, value + 3)
    # unsupported
    return (-1, -1)


def _canonical_ipaddress(ip_addr):
    """Given an IP address as a string, typically an IPv6 address, return the
       canonical form of this IP address.  Note that IPv4 addressses like
       10.04.3.2 (with a leading 0) are not considered valid IPv4 addresses by
       the Python3 library ipaddress or by inet_pton.  For example,
       ipaddress.ip_address("10.04.3.2") will raise a ValueError.
       This function works in Python2 and Python3.
    """
    try:
        _str = socket.inet_pton(socket.AF_INET6, ip_addr)
        a, b = struct.unpack('!2Q', _str)
        return socket.inet_ntop(socket.AF_INET6, struct.pack('!2Q', a, b))
    except socket.error:
        pass
    try:
        _str = socket.inet_pton(socket.AF_INET, ip_addr)
        n = struct.unpack('!I', _str)[0]
        return socket.inet_ntop(socket.AF_INET, struct.pack('!I', n))
    except socket.error:
        pass
    return ip_addr


def _read_dns_name(context):
    """Used by validate_hostname to validate hostnames.
    """
    ALT_DNS = 130
    ALT_IP = 135
    name = ""
    total_length = 0
    ptype = _read_octet(context)  # read the type
    total_length += 1
    (length, length_with_hdr) = _read_length(context)
    if length < 0:
        context.errstr = ("Unexpected ASN.1 length at position " +
                          str(context.ptr))
        return -1
    total_length += length_with_hdr
    i = length
    while i > 0:
        char = _read_octet(context)
        if char < 0:
            context.errstr = ("Unexpected ASN.1 character at position " +
                              str(context.ptr))
            return -1
        if ptype == ALT_DNS:
            # build DNS name
            name += chr(char)
        elif ptype == ALT_IP:
            # build dotted decimal IP address
            if name:
                name += "."
            name += str(char)
        i -= 1
    if ptype == ALT_DNS or ptype == ALT_IP:
        if ptype == ALT_IP and name.count('.') > 4:
            octets = name.split('.')
            name = ""
            count = 0
            for octet in octets:
                if name and count % 2 == 0:
                    name += ":"
                # Remove leading "0x" with [2:]
                name += hex(int(octet))[2:].zfill(2)
                count += 1
            name = _canonical_ipaddress(name)
        context.parsed_data.append(name)
    return total_length


def _parse_subject_alt(context):
    """Used by validate_hostname to validate hostnames.  Writes output to
       context.parsed_data.
    """
    ptype = _read_octet(context)  # read the type
    if ptype != 48:
        context.errstr = ("Unexpected ASN.1 type.  Expected 48 but read " +
                          str(ptype) + " at position " + str(context.ptr))
        return -1
    (length, _) = _read_length(context)
    if length < 0:
        context.errstr = ("Unexpected ASN.1 length at position " +
                          str(context.ptr))
        return -1
    rem_len = length
    while rem_len > 0:
        length = _read_dns_name(context)
        if length < 0:
            context.errstr = ("Unexpected DNS name at position " +
                              str(context.ptr))
            return -1
        rem_len -= length
    return 0


def _is_valid_dns_entry(dnsname):
    """This function filters out incorrectly specified hostname in certs, to
       avoid matching the hostname in cases where the cert is improper.  This
       might not occur in the wild, but just being cautious.  Returns True if
       the DNS name in the cert is proper and False otherwise.
    """
    # See rules in https://en.wikipedia.org/wiki/Wildcard_certificate
    if not dnsname:
        return False
    if '*' not in dnsname:
        return True
    if dnsname.count('*') > 1:
        # A cert with multiple wildcards in a name is not allowed.
        # *.*.domain.com
        return False
    if dnsname.count('.') == 1:
        # A cert with * plus a top-level domain is not allowed.
        # *.com
        return False
    if dnsname == '*':
        # Too general and should not be allowed.
        # *
        return False
    if dnsname[0] != '*':
        # All major browsers have deliberately removed support for
        # partial-wildcard certificates. In other words, the '*' must be the
        # first character if the '*' is present.
        return False
    return True


def _set_err_stderr(options, errcode, errstr):
    """Sets variables to be used by the _print_final_msgs() function.
    """
    options.errcode = errcode
    options.errstr = errstr


def _print_final_msgs(options, stats, without_prefix=False):
    """Used to print final error message to stderr and the write-out message to
       stdout.  Once options.last_curl_fin is True don't print any more
       messages to stderr so that curl has the last say.
    """
    if not options.silent or options.verbose or options.show_error:
        # Final stderr message
        if options.errstr and not options.last_curl_fin:
            if not without_prefix:
                errmsg = ("curl_wrapper: (" + str(options.errcode) + ") " +
                          options.errstr)
            else:
                errmsg = options.errstr
            print(errmsg, file=sys.stderr)
            sys.stderr.flush()
    if stats:
        # Final stdout message
        _print_write_out(options, stats)


def _print_msg_stderr(options, msg):
    """Used to print verbose error message to stderr.
       Once options.last_curl_fin is True don't print any more messages
       to stderr so that curl has the last say.
    """
    # verbose trumps silent
    if options.verbose and not options.last_curl_fin:
        print(msg, file=sys.stderr)
        sys.stderr.flush()


def _print_headers_stderr(options, method, path, outbound_hdrs):
    """Used to print outbound HTTP headers to stderr.
    """
    # verbose trumps silent
    if options.verbose and not options.last_curl_fin:
        print("> " + method + " " + path + " HTTP/1.1", file=sys.stderr)
        for name, value in outbound_hdrs.items():
            print("> " + name + ": " + str(value), file=sys.stderr)
        sys.stderr.flush()


def _get_min_timeout(options, exclude_default=False):
    """Helper function to get the smaller of the two timeout options,
       options.max_time and options.connect_timeout. If neither options are
       specified, return either DEFAULT_TIMEOUT (in seconds) or None depending
       on the optional exclude_default parameter.  The default curl timeout is
       2 minutes, but this script uses a DEFAULT_TIMEOUT second timeout.
    """
    if options.max_time > 0:
        if options.connect_timeout > 0:
            if options.max_time < options.connect_timeout:
                return options.max_time
            return options.connect_timeout
        return options.max_time
    if options.connect_timeout > 0:
        return options.connect_timeout
    if exclude_default:
        return None
    return DEFAULT_TIMEOUT


def _hostname_check(dnsname, hostname):
    """Low-level hostname check that takes the 'dnsname' that comes from the
       cert, and 'hostname' which comes from the URL given as input to this
       script.  Returns True if there is a match and False otherwise.
    """
    dnsname = dnsname.lower()
    hostname = hostname.lower()
    if dnsname[0] == '*':
        # The wildcard applies only to one level of the domain name.
        return (dnsname.count('.') == hostname.count('.') and
                hostname.endswith(dnsname[1:]))
    if dnsname == hostname:
        return True
    if _is_ipv6_address(hostname):
        return dnsname == _canonical_ipaddress(hostname)
    return False


def _get_list_common_names(cert):
    """Extract all CNs from the given cert.
    """
    if have_cryptography:
        attrs = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)
        if attrs:
            return [cn.value for cn in attrs]
        return []
    else:
        cns = []
        for comp in cert.get_subject().get_components():
            if comp[0].decode(UTF8) == 'CN':
                cns.append(comp[1].decode(UTF8))
        return cns


def _get_first_common_name(cert):
    """Extract the first CN from the given cert.
       If no common name return ''.
    """
    cns = _get_list_common_names(cert)
    if len(cns) > 0:
        return cns[0]
    return ''


def _get_san_entries(cert, options):
    """Collect subject alternative names from the cert.
    """
    entries = []
    if have_cryptography:
        try:
            san_extension = cert.extensions.get_extension_for_class(
                x509.SubjectAlternativeName)
            san_values = san_extension.value
            entries.extend(san_values.get_values_for_type(x509.DNSName))
            entries.extend(str(san_values.get_values_for_type(x509.IPAddress)))
        except x509.ExtensionNotFound:
            pass
    else:
        ctx = SimpleParseContext()
        for i in range(cert.get_extension_count()):
            ext = cert.get_extension(i)
            if ext.get_short_name() == b'subjectAltName':
                # For Python 2 ext.get_data() is a string whereas for Python 3
                # it is a buffer.  This difference is handled by _read_octet().
                ctx.data = ext.get_data()
                ret = _parse_subject_alt(ctx)
                if ret == -1:
                    # Just log parse errors and proceed
                    _print_msg_stderr(options, ctx.errstr)
                entries = ctx.parsed_data
    return entries


def _validate_hostname(cert, hostname, options):
    """Perform the hostname check that curl does.  If the hostname in the
       cert's subject doesn't match then iterate through the cert's Subject
       Alternative Name fields.  It may be possible to use the httplib library
       to achieve this so we may be able to remove this code in future.
       Returns the tuple (errcode, errstr).  If the hostname is validated
       successfully, this function returns (0, None).  Otherwise this function
       returns a non-zero errcode and a non-None errstr.
    """
    if options.no_hostname_check:
        return 0, None
    for cn in _get_list_common_names(cert):
        if _is_valid_dns_entry(cn) and _hostname_check(cn, hostname):
            return 0, None
    for host in _get_san_entries(cert, options):
        if _is_valid_dns_entry(host) and _hostname_check(host, hostname):
            return 0, None

    errcode = THIS_NO_ALTERNATIVE_CERTIFICATE_SUBJECT_NAME
    errstr = ("SSL: no alternative certificate subject name matches " +
              "target host name '" +
              hostname + "'")
    return errcode, errstr


def _get_int_status(response):
    """Returns integer HTTP status from response object.
       Copied from backup_restore.py.

    Args:
        response: Either http.client.HTTPResponse or webob.response.Response
    Returns:
        webob.response.Response.status_int or http.client.HTTPResponse.status
    """
    if hasattr(response, "status_int"):
        return response.status_int
    return response.status


def _replace_newlines(text):
    """Replace the newline character with the two characters '\' and 'n' so
       that the resultant string can be included in a JSON body.
    """
    return '\\n'.join(text.splitlines()) + '\\n'


def _replace_slashn(text):
    """Replace the \\n with newline characters.
    """
    return '\n'.join(text.split('\\n')) + '\n'


def _dump_cert_as_pem(cert):
    """Write cert as a PEM string.
    """
    if have_cryptography:
        return cert.public_bytes(serialization.Encoding.PEM).decode(UTF8)
    return crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode(UTF8)


def _dump_certchain_as_pem(cert_chain):
    """Write cert chain as a PEM string.
    """
    pem_data = ''
    for cert in cert_chain:
        pem_data += _dump_cert_as_pem(cert)
    return pem_data


def _validate_crl_over_rest(cert_chain, options):
    """This function requests a CRL check over REST.  The REST interface is
    required only because the Global Manager doesn't support the RPC interface
    implemented by _validate_crl_over_nsxrpc.  If the CRL check passes, this
    function returns (0, None).  Otherwise this function returns a non-zero
    errcode and a non-None errstr.
    """
    errcode = CURLM_INTERNAL_ERROR
    errstr = "unknown"
    host = '127.0.0.1'
    port = 7440
    conn = None
    try:
        pem_data = _replace_newlines(_dump_certchain_as_pem(cert_chain))
        timeout = _get_min_timeout(options)
        method = 'POST'
        path = ('/nsxapi/api/v1/trust-management/certificates?'
                'action=check_trusted&crl_check=true')
        data = '{"pem_encoded": "' + pem_data + '", "cert_type": "SERVER"}'
        if options.verbose:
            _print_msg_stderr(options, 'Calling ' + path + ' with payload:')
            _print_msg_stderr(options, data)

        hdrs = {}
        hdrs.update({HDR_CONTENT_TYPE: 'application/json'})
        hdrs.update({'X-NSX-Username': 'admin'})

        conn = httplib.HTTPConnection(host, port, timeout=timeout)
        conn.request(method, path, data, hdrs)
        resp = conn.getresponse()
        resp_status = _get_int_status(resp)
        body = resp.read()
        if resp_status != httplib.OK:
            # Use CURLE_COULDNT_CONNECT to allow a retry
            errcode = CURLE_COULDNT_CONNECT
            errstr = ('Response for /nsxapi/api/v1/trust-management'
                      '/certificates?action=check_trusted&crl_check=true '
                      'was ' + str(resp_status) + ' but ' +
                      str(httplib.OK.value) + ' is expected')
        else:
            resp_obj = json.loads(body)
            app_status = resp_obj.get('status')
            if app_status not in ('OK', 'CRL_NOT_READY'):
                errcode = THIS_CRL_CHECK_FAILED
                errstr = resp_obj.get('error_message')
            else:
                errcode = CURLE_OK
                errstr = None
    except Exception as ex:
        errcode = CURLE_COULDNT_CONNECT
        submsg = str(ex)
        errstr = ("Failed to connect to " + host + " port " +
                  str(port) + ": " + submsg)
        _log_backtrace(options, errstr,
                       traceback.format_exc())
    finally:
        if conn:
            conn.close()
            conn = None
    return errcode, errstr


def _get_nsx_proxy_rpc_connection():
    """This function returns a connection object to the local RPC service.  It
       uses the RPC service tcp://127.0.0.1:9004" for ESX and
       "unix:///var/run/vmware/nsx-proxy/aphinfoservice.sock" for Edge and
       Windows. The function returns a connection object if successful and None
       otherwise.
    """
    if exists("/var/run/vmware/nsx-proxy/aphinfoservice.sock"):
        provider_conn = ("unix:///var/run/vmware/nsx-proxy/"
                         "aphinfoservice.sock")
    else:
        provider_conn = "tcp://127.0.0.1:9004"
    try:
        conn = NsxRpcConnection()
        conn.Connect(provider_conn)
        return conn
    except Exception:
        pass
    return None


def _bypass_crl_check_for_url(options):
    """There are several cases where RPC response will not be available and
       therefore the CRL check can't be performed. Bypass the CRL check for
       these APIs.
    """
    bypass = False
    bypass_list = ['/api/v1/transport-nodes?action=register_node',
                   '/api/v1/cluster/nodes/',
                   '/api/v1/messaging/clients/',
                   '/api/v1/fabric/nodes/']
    path = ''
    try:
        # Shouldn't fail here because this URL parsing has been done earlier.
        # If for some reason there is a URL parsing problem, then don't bypass
        # the CRL check.
        url_parts = urlparse(options.url)
        path = url_parts.path
        if url_parts.query:
            path = path + '?' + url_parts.query
    except ValueError:
        pass
    for api in bypass_list:
        if path.startswith(api):
            if api == '/api/v1/fabric/nodes/':
                if path.endswith('?action=register_node'):
                    bypass = True
            else:
                bypass = True
            break
    if bypass:
        logmsg = ("Skipping CRL check for special API " + path)
        _print_msg_stderr(options, logmsg)
    return bypass


def _validate_crl_over_nsxrpc(cert_chain, options):
    """This function requests a CRL check over NSXRPC.  The NSXRPC interface is
       supported by all node types except for the Global Manager.  The function
       forwards the CRL check to the registered Manager.  If the CRL check
       passes, this function returns (0, None).  This function needs to also
       work in the install use-case on ESX where the NSX vibs are not yet
       installed, and in this case the function returns success (0,
       CRL_CHECK_WARNING) to mimic a successful CRL check.  Otherwise this
       function returns a non-zero errcode and a non-None errstr.
    """
    if not _is_appliance_info_valid():
        node_type = get_node_type().split()[0]
        if node_type in NODE_TYPES_WITH_PROTON:
            return CURLE_FAILED_INIT, "Invalid appliance-info.xml file"
        # This node is not yet registered with a Manager and therefore
        # it is not possible to check the CRL. Return success to mimic
        # that the CRL check passed.
        reason = "the node isn't registered with the NSX Manager"
        logmsg = "Skipping CRL check because " + reason
        _print_msg_stderr(options, logmsg)
        return 0, CRL_CHECK_WARNING + " because " + reason
    if _bypass_crl_check_for_url(options):
        # this function also calls _print_msg_stderr()
        return 0, None

    if not have_nsx:
        node_type = get_node_type().split()[0]
        if node_type in NODE_TYPES_WITH_PROTON:
            # Should never happen on Manager and Global Manager
            return CURLE_FAILED_INIT, "NSX isn't installed"
        # NSX hasn't been installed yet and therefore it is not possible to
        # check the CRL. Return success to mimic that the CRL check passed.
        reason = "NSX isn't installed"
        logmsg = "Skipping CRL check because " + reason
        _print_msg_stderr(options, logmsg)
        return 0, CRL_CHECK_WARNING + " because " + reason

    conn = None
    response = None
    appl_info_param = ApplProxyInfoReqMsg()
    pem_data = _replace_newlines(_dump_certchain_as_pem(cert_chain))
    try:
        conn = _get_nsx_proxy_rpc_connection()
        if not conn:
            # Use CURLE_COULDNT_CONNECT to allow a retry
            errcode = CURLE_COULDNT_CONNECT
            errstr = "Unable to connect to RPC service"
            return errcode, errstr
        # pylint: disable=W0612
        with NsxRpcClient(ApplProxyInfoService_Stub,
                          connection=conn) as nsx_rpc_client:
            get_appl_info = eval("nsx_rpc_client.GetApplProxyInfo")
            response = get_appl_info(appl_info_param)
    except Exception:
        pass
    finally:
        if conn is not None:
            conn.Close()
    if not response:
        # Use CURLE_COULDNT_CONNECT to allow a retry
        errcode = CURLE_COULDNT_CONNECT
        errstr = "Unable to invoke GetApplProxyInfo RPC call"
        return errcode, errstr
    aph_id = 0
    for info in response.applproxy_info:
        if not info.HasField("id"):
            continue
        aph_id = uuid.UUID(int=(info.id.left << 64) + info.id.right)
        if info.master:
            break
    if not aph_id:
        # Use CURLE_COULDNT_CONNECT to allow a retry
        errcode = CURLE_COULDNT_CONNECT
        errstr = "No APH UUID found in CheckTrusted RPC response"
        return errcode, errstr

    check_trusted_param = CheckTrustedRequestMsg()
    check_trusted_param.crl_check = True
    _SERVER = CheckTrustedRequestMsg.CertificateType.Value("SERVER")
    check_trusted_param.cert_type = _SERVER
    s_ok = CheckTrustedResponseMsg.CertificateCheckStatus.Value("OK")
    check_trusted_param.pem_encoded = pem_data
    provider_connection = "tcp://127.0.0.1:4096"
    provider_endpoint = str(aph_id)
    response = None
    conn = None
    try:
        conn = NsxRpcConnection()
        conn.Connect(provider_connection)
        with NsxRpcClient(CertificateService_Stub, connection=conn,
                          destination=provider_endpoint) as nsx_rpc_client:
            check_trusted_rpc = eval("nsx_rpc_client.CheckTrusted")
            response = check_trusted_rpc(check_trusted_param)
    except Exception:
        pass
    finally:
        if conn is not None:
            conn.Close()
    if not response:
        # Use CURLE_COULDNT_CONNECT to allow a retry
        errcode = CURLE_COULDNT_CONNECT
        errstr = "Unable to invoke CheckTrusted RPC call"
        return errcode, errstr

    status = response.status
    s_ok = CheckTrustedResponseMsg.CertificateCheckStatus.Value("OK")
    s_crl_not_ready = CheckTrustedResponseMsg.CertificateCheckStatus.Value(
        "CRL_NOT_READY")
    if status not in (s_ok, s_crl_not_ready):
        errcode = THIS_CRL_CHECK_FAILED
        errstr = response.error_message
        return errcode, errstr
    return 0, None


def _validate_crl(cert_chain, options):
    """Perform a CRL check using the cert chain from the remote server as
       input. If the CRL check passes, this function returns (0, None).  This
       function needs to also work in the install use-case on ESX where the NSX
       vibs are not yet installed, and in this case the function returns
       success (0, CRL_CHECK_WARNING) to mimic a successful CRL check.
       Otherwise this function returns a non-zero errcode and a non-None
       errstr.
    """
    node_type = get_node_type().split()[0]
    if node_type in REST_NODE_TYPES:
        return _validate_crl_over_rest(cert_chain, options)
    return _validate_crl_over_nsxrpc(cert_chain, options)


def _validate_ca_signed_cert(cert_chain, hostname, options):
    """Validate a ca-signed cert.  Either the --cacert or --thumbprint option
       must be present otherwise trust will not be established and an error
       will be returned.  Returns the tuple (errcode, errstr).  If cert is
       validated successfully, this function returns (0, None).  This function
       needs to also work in the install use-case on ESX where the NSX vibs are
       not yet installed, and in this case the function returns success (0,
       CRL_CHECK_WARNING) to mimic a successful CRL check.  Otherwise this
       function returns a non-zero errcode and a non-None errstr.
    """
    leaf_cert = _get_leaf_cert(cert_chain)
    errcode, errstr = _validate_common(leaf_cert)
    if errcode:
        return errcode, errstr
    errcode, errstr = _validate_hostname(leaf_cert, hostname, options)
    if errcode:
        return errcode, errstr
    if options.thumbprint:
        errcode, errstr = _validate_trust_with_thumbprint(cert_chain, options)
        if errcode:
            return errcode, errstr
    if options.cacert:
        if not exists(options.cacert):
            # Doesn't get checked in validate_server_cert_from_file.
            errstr = CACERT_BADFILE_ERRSTR + options.cacert
            return CURLE_SSL_CACERT_BADFILE, errstr
        errcode, errstr, cacert_found = _validate_cert_chain_order(cert_chain,
                                                                   options)
        # If errcode is 0 then cert_found is non-None.
        if errcode:
            return errcode, errstr
        errcode, errstr = _validate_cert_chain_expiry(cert_chain)
        if errcode:
            return errcode, errstr
        if _cert_has_expired(cacert_found):
            return (CURLE_SSL_CACERT_BADFILE,
                    ("The cacert " + _get_subject(cacert_found) +
                     " in the CA store has expired"))
        if options.cacert_is_ca:
            errcode, errstr = \
                _validate_basic_constraints_for_nonleaf2(cacert_found, options)
            if errcode:
                return errcode, errstr
        errcode, errstr = _validate_cert_chain_basic_constraints(cert_chain,
                                                                 options)
        if errcode:
            return errcode, errstr
        errcode, errstr = _validate_cert_chain_eku(cert_chain, options)
        if errcode:
            return errcode, errstr
    return _validate_crl(cert_chain, options)


def _validate_cert(cert_chain, hostname, options):
    """Top-level function to validate a cert chain received from the remote
       server. The cert chain could be either a single self-signed cert or a
       CA-signed cert and its cert chain.  Either the --cacert or --thumbprint
       option must be present otherwise trust will not be established and an
       error will be returned.  Returns the tuple (is_self_signed, errcode,
       errstr).  If cert is validated successfully, this function returns
       (is_self_signed, 0, None).  Otherwise this function returns a non-zero
       errcode and a non-None errstr.
    """
    # Assume CA-signed until known otherwise.
    is_self_signed = False
    if len(cert_chain) == 0:
        errcode = CURLE_PEER_FAILED_VERIFICATION
        errstr = "No certificate provided"
        thumbprint = "(no thumbprint)"
    else:
        leaf_cert = _get_leaf_cert(cert_chain)
        is_self_signed = _get_subject(leaf_cert) == _get_issuer(leaf_cert)
        if is_self_signed:
            errcode, errstr = _validate_self_signed_cert(cert_chain,
                                                         options)
        else:
            errcode, errstr = _validate_ca_signed_cert(cert_chain, hostname,
                                                       options)
        thumbprint = _canonical_thumbprint(_get_thumbprint(leaf_cert))
    logmsg = ("certificate verification " + thumbprint + " from " +
              options.host)
    if options.port:
        logmsg += ":" + str(options.port)
    logmsg += " "
    if errcode == 0:
        logmsg += "passed"
        if errstr:
            # warning about unable to perform CRL check
            logmsg += " (" + errstr + ")"
    else:
        logmsg += "failed: " + errstr
    _log_cert_verification_result(options, logmsg)
    return is_self_signed, errcode, errstr


def _is_transient_error(errcode, httpcode):
    """Return True if the errorcode and httpcode are considered a transient
       error as documented by the curl man page.  The comment below about
       transient errors comes from the curl man page under the --retry option.
       Transient error means either: a timeout, an FTP 4xx response code or an
       HTTP 408, 429, 500, 502, 503 or 504 response code.
    """
    if errcode in [CURLE_COULDNT_CONNECT, CURLE_OPERATION_TIMEDOUT,
                   CURLE_PARTIAL_FILE]:
        return True
    if httpcode in [408, 429, 500, 502, 503, 504]:
        return True
    if httpcode >= 400 and httpcode < 500:
        return True
    return False


def _is_ipv6_address(host):
    """Return True if host is an IPv6 address.  Don't use ipaddress import
       because that package exists only for Python3.
    """
    return True if ':' in host else False


def _convert_to_standard_certs(cert_chain):
    """If cryptography package available, convert from "OpenSSL" certs to
       "cryptography" certs.  Otherwise leave as "OpenSSL" certs.
    """
    if not have_cryptography:
        return cert_chain
    return [cert.to_cryptography() for cert in cert_chain]


def _call_command(options, cmd_args, timeout=None, need_stderr=False):
    """Call command and return (returncode, stdout, stderr).  The optional
       timeout parameter is an integer in seconds.  This function is typically
       used to invoke openssl.  Throws Exception(EXCEPTION_TIMEDOUT_MSG) on
       timeout for both Python2 and Python3.  May throw other Exceptions as
       well.
    """
    returncode = -1
    output = ''
    stderr = ''
    if have_py3:
        # Python3
        try:
            if need_stderr:
                # subprocess.check_output is preferred but it doesn't
                # provide stderr.
                if timeout:
                    if exists("/usr/bin/timeout"):
                        if _is_esx():
                            cmd_args = (["/usr/bin/timeout", "-t",
                                         str(timeout)] + cmd_args)
                        else:
                            cmd_args = (["/usr/bin/timeout", str(timeout)] +
                                        cmd_args)
                _log_command(options, cmd_args)
                df = subprocess.Popen(cmd_args, stdin=subprocess.PIPE,
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.PIPE)
                output, stderr = df.communicate()
                if timeout and df.returncode == 124:
                    _log_command_timedout(options, cmd_args[0] + " timed out")
                    raise Exception(EXCEPTION_TIMEDOUT_MSG)
                return df.returncode, output, stderr
            # MacOS doesn't like DEVNULL for stdin, whereas PIPE appears to
            # work for MacOS and Ubuntu.  The advantage of check_output() over
            # Popen() is that check_output() in Python 3 supports a timeout
            # parameter.
            _log_command(options, cmd_args)
            if timeout:
                output = subprocess.check_output(cmd_args,
                                                 stdin=subprocess.PIPE,
                                                 stderr=subprocess.DEVNULL,
                                                 timeout=timeout)
            else:
                output = subprocess.check_output(cmd_args,
                                                 stdin=subprocess.PIPE,
                                                 stderr=subprocess.DEVNULL)
            returncode = 0
        except TimeoutExpired:
            _log_backtrace(options, cmd_args[0] + " timed out",
                           traceback.format_exc())
            # Our special exception used for both Python2 and Python3.
            raise Exception(EXCEPTION_TIMEDOUT_MSG)
        except subprocess.CalledProcessError as ex:
            returncode = ex.returncode
        return returncode, output, stderr
    # Python2
    if timeout:
        if exists("/usr/bin/timeout"):
            # No Python2 on ESXi 6.5.0 and later
            cmd_args = ["/usr/bin/timeout", str(timeout)] + cmd_args
    # TimeoutExpired is not defined in Python2 so resort to the timeout
    # command.  subprocess.DEVNULL doesn't exist in Python2.  check_output()
    # doesn't work with "timeout=timeout" and with the stdin= parameter when we
    # have connectivity to the remote server.  Conversely, check_output()
    # doesn't work without stdin= parameters when we don't have network
    # connectivity. Therefore check_output() doesn't work in all cases.
    # Fortunately, Popen works in both these cases.
    _log_command(options, cmd_args)
    df = subprocess.Popen(cmd_args, stdin=subprocess.PIPE,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE)
    output, stderr = df.communicate()
    returncode = df.returncode
    if timeout and df.returncode == 124:
        _log_command_timedout(options, cmd_args[0] + " timed out")
        raise Exception(EXCEPTION_TIMEDOUT_MSG)
    return returncode, output, stderr


def _noverify_callback(connection, cert, errnum,  # pylint: disable=W0613
                       errdepth, ok):             # pylint: disable=W0613
    """This function is required by set_verify in Python 3.5.6 and earlier.
    """
    return ok


def _get_peer_cert_chain_with_sock_type(options,  # pylint: disable=W0613
                                        host, port, socket_type):
    """Return the cert chain from the host:port and given socket_type (IPv4 or
       IPv6).  Returns the cert chain if a cert chain retrieved.  Otherwise
       throws an Exception.  One of the reasons for throwing an Exception is
       that the OpenSSL library is not available.  The leaf cert is the first
       cert in cert chain.
       When changing the method need to test on Manager and ESXi.
       Tested on NSX  9.1.0 build 25014552.
       Tested on ESXi 9.0.0 build 24957456.
       Tested on ESXi 7.0.3 build 18644231.
       Tested on ESXi 6.7.0 build 14320388.
    """
    conn = None
    try:
        # OpenSSL may not be available in Python2
        from OpenSSL import SSL     # pylint: disable=C0415
        context = SSL.Context(method=SSL.SSLv23_METHOD)
        # Can't seem to disable the hostname check, but it doesn't hurt.
        # The second argument is required in Python 3.5.6 and earlier.
        context.set_verify(SSL.VERIFY_NONE, _noverify_callback)

        sock = socket.socket(socket_type, socket.SOCK_STREAM)
        conn = SSL.Connection(context, sock)
        conn.connect((host, port))
        # The following command works for the most part except when the remote
        # cert is CA signed and host is an IP address.  When this does fail
        # because of the hostname check, even if we were able to retrieve the
        # cert chain, we would fail at a later time for the same reason (but
        # with a better error message). When this does fail, an exception
        # "ssl/tls alert handshake failure" will be raised by the
        # do_handshake() function.
        conn.set_tlsext_host_name(host.encode(UTF8))
        conn.setblocking(1)
        conn.do_handshake()
        cert_chain = conn.get_peer_cert_chain()
        return _convert_to_standard_certs(cert_chain)
    finally:
        if conn:
            conn.close()


def _read_certs_from_openssl_output(text_with_pem_data):
    """Read cert chain from openssl output.  The leaf cert is the first cert in
       cert chain.  Throws an Exception if unable to parse the PEM encoding.
    """
    cert_chain = []
    text = str(text_with_pem_data)
    start_line = '-----BEGIN CERTIFICATE-----'
    end_line = '-----END CERTIFICATE-----'
    cert_slots = text.split(start_line)
    for single in cert_slots[1:]:
        # The variable single will typically include the END CERTIFICATE header
        # and it may include other text after that (because of the way openssl
        # displays cert chains).
        single_pem_cert = single.split(end_line)[0]
        cert_pem = start_line + single_pem_cert + end_line
        cert_pem = _replace_slashn(cert_pem).encode(UTF8)
        if have_cryptography:
            cert = x509.load_pem_x509_certificate(cert_pem,
                                                  default_backend())
        else:
            cert = crypto.load_certificate(crypto.FILETYPE_PEM,
                                           cert_pem)
        cert_chain.append(cert)
    return cert_chain


def _get_peer_cert_chain_with_openssl(options, host, port):
    """There are two functions that attempt to retrieve the peer cert. This
       function is one of them.  The other function is
       _get_peer_cert_chain_with_sock_type().  The function returns the cert
       chain if successful, otherwise throws an Exception.  The function throws
       a Exception(EXCEPTION_TIMEDOUT_MSG) on a timeout.
    """
    # openssl doesn't have a parameter to specify the connection timeout.
    # So select the smaller of --max-time and --connect-timeout on the basis
    # that any time spent waiting for openssl to respond will likely be due
    # to the connection time.
    timeout = _get_min_timeout(options)
    cmd_args = ["openssl", "s_client", "-showcerts", "-servername", host,
                "-connect", host + ":" + str(port)]
    _, output, _ = _call_command(options, cmd_args, timeout, False)
    # output will be '' if there is any problem.
    # _read_certs_from_openssl_output throws an Exception if unable to parse
    # the PEM encoding.
    return _read_certs_from_openssl_output(output)


def _get_peer_cert_chain(options, host, port):
    """This function retrieves the cert chain from the remote host using
       a different connection that will be used after this to perform the curl
       operation.  This function returns a cert chain or throws an Exception.
       This function throws KeyboardInterrupt if user presses CONTROL-C.
    """
    try:
        cert_chain = _get_peer_cert_chain_with_openssl(options, host, port)
        if len(cert_chain) > 0:
            return cert_chain
    except KeyboardInterrupt as ex:
        raise ex
    except Exception as ex:
        if str(ex) == EXCEPTION_TIMEDOUT_MSG:
            raise ex
        else:
            pass
    socket_type = socket.AF_INET6 if _is_ipv6_address(host) else socket.AF_INET
    return _get_peer_cert_chain_with_sock_type(options, host, port,
                                               socket_type)


def _submsg_translate(submsg):
    """Translate between httplib errorcodes and curl errorcodes
    """
    if 'timed out' in submsg:
        return CONNECTION_TIMED_OUT_MSG
    if 'Connection refused' in submsg:
        return 'Connection refused'
    return submsg


def _add_cert_to_trust(options, host, port, leaf_cert):
    """Once the leaf certificate has been validated for a given host and port
       add it to our in-memory trust store. This trust store is then used when
       invoking curl on subsequent retries.  See _is_trust_established(). The
       trust store is located in the options structure as a convenience.
       The parameter port should be an integer.
    """
    options.trust_store[(host, port)] = leaf_cert


def _is_trust_established(options):
    """Check if trust has been established with the remote server.
       The trust was annotated with _add_cert_to_trust().  If using HTTP rather
       than HTTPS then return True because we don't need trust in this case.
       Returns True or False.
    """
    if options.url.lower().startswith("https:"):
        try:
            url_parts = urlparse(options.url)
            host = url_parts.hostname
            port = url_parts.port
            if not port:
                port = 443
            return (host, port) in options.trust_store
        except ValueError:
            return False
    return True


def _errcode_from_options(errcode, options):  # pylint: disable=W0613
    """Set the error code based on the context of the exception message.
       Previously, curl would return CURLE_COULDNT_CONNECT (7) for connection
       errors unless the --max-time option is set in which case curl returns
       CURLE_OPERATION_TIMEDOUT (28).  However, the more recent behavior is
       curl returns CURLE_OPERATION_TIMEDOUT (28) regardless of options used.
    """
    if errcode != CURLE_COULDNT_CONNECT:
        return errcode
    # Translate CURLE_COULDNT_CONNECT to CURLE_OPERATION_TIMEDOUT
    errcode = CURLE_OPERATION_TIMEDOUT
    return errcode


def _validate_peer_cert_chain(options, host, port):
    """Fetch the cert chain from the 'host' and validate it.  Validated leaf
       certs are placed in options.trust_store[].  Returns the tuple
       (errcode, errstr).  If cert is validated successfully,
       this function returns (0, None).  Otherwise this
       function returns a non-zero errcode and a non-None errstr.
    """
    # Assume the peer cert is CA-signed unless told otherwise.
    try:
        cert_chain = _get_peer_cert_chain(options, host, port)
        _, errcode, errstr = _validate_cert(cert_chain, host, options)
        if errcode:
            return errcode, errstr

        # If _validate_cert() succeeds then the cert is trusted.
        leaf_cert = _get_leaf_cert(cert_chain)
        _add_cert_to_trust(options, host, port, leaf_cert)
        return 0, None
    except (AttributeError, ValueError, NameError) as ex:
        # _get_peer_cert_chain() may throw an AttributeError or a ValueError.
        # We forget why AttributeError are thrown.  ValueErrors are thrown if
        # the PEM encoding is corrupt.  NameErrors are thrown when the
        # cryptography package isn't available.  When this happens, handle it
        # has a non-recoverable error (that is don't retry when the --retry
        # option is used). Specifically, don't return with a
        # CURLE_COULDNT_CONNECT error.
        errcode = CURLE_SSL_CONNECT_ERROR
        submsg = str(ex)
        errstr = ("Failed to retrieve cert chain from " + host + " port " +
                  str(port) + ": " + submsg)
        return errcode, errstr
    except KeyboardInterrupt as ex:
        # Whenever we catch Exception, always catch KeyboardInterrupt prior to
        # catching Exception and rethrow it.
        raise ex
    except Exception as ex:
        # Handles timeout and all other exceptions
        if str(ex) == EXCEPTION_TIMEDOUT_MSG:
            submsg = str(ex)
            errcode = _errcode_from_options(CURLE_COULDNT_CONNECT, options)
            index = submsg.find("timed out after")
            if index >= 0:
                errstr = "Connection " + submsg[index:]
            else:
                errstr = "Connection timed out"
            _log_backtrace(options, errstr,
                           traceback.format_exc())
            return errcode, errstr
        else:
            submsg = _submsg_translate(str(ex))
            errcode = _errcode_from_options(CURLE_COULDNT_CONNECT, options)
            errstr = ("Failed to connect to " + host + " port " +
                      str(port) + ": " + submsg)
            _log_backtrace(options, errstr,
                           traceback.format_exc())
            return errcode, errstr


def _read_name(context):
    """ Used by _parse_form_option() to read the name in name=value strings in
        curl's --form option.  Returns '' if no name is present.
    """
    name = ''
    while context.ptr < len(context.data):
        ch = context.data[context.ptr]
        if ch == '=':
            break
        name += ch
        context.ptr += 1
    return name


def _read_equals(context):
    """Used by _parse_form_option() to read the = character in name=value
       strings in curl's --form option.  Returns None if no name is present.
    """
    if context.ptr < len(context.data):
        ch = context.data[context.ptr]
        if ch == '=':
            context.ptr += 1
            return ch
    return None


def _read_string(context, quote_char):
    """Helper function used by _parse_form_option() to read strings where the
       string starts with the given quote_char. The function handles escaped
       characters such that if the quote_char is escaped with '\' then the
       function continues.  Returns '' if no value is present.
    """
    value = ''
    if len(quote_char) == 1:
        if context.ptr >= len(context.data):
            return None
        if context.data[context.ptr] != quote_char:
            return None
        ch = context.data[context.ptr]
        context.ptr += 1
        value += ch
        while context.ptr < len(context.data):
            ch = context.data[context.ptr]
            context.ptr += 1
            value += ch
            if ch == quote_char:
                return value
    elif len(quote_char) == 2:
        if context.ptr + 1 >= len(context.data):
            return None
        if quote_char[0] != '\\':
            return None
        if context.data[context.ptr] != quote_char[0]:
            return None
        if context.data[context.ptr + 1] != quote_char[1]:
            return None
        context.ptr += 2
        found_escape = False
        while context.ptr < len(context.data):
            ch = context.data[context.ptr]
            context.ptr += 1
            value += ch
            if found_escape:
                if context.ptr < len(context.data):
                    ch = context.data[context.ptr]
                    context.ptr += 1
                    value += ch
                    if ch == quote_char[1]:
                        return value
                    found_escape = False
                else:
                    return None
            else:
                if context.ptr < len(context.data):
                    ch = context.data[context.ptr]
                    context.ptr += 1
                    value += ch
                    if ch == quote_char[0]:
                        found_escape = True
                else:
                    return None
    return None


def _read_value(context):
    """ Used by _parse_form_option() to read the value in name=value strings in
        curl's --form option.  Returns '' if no value is present.
        Read up to first unescaped ; character.
        Returns None on error.
    """
    value = ''
    while context.ptr < len(context.data):
        ch = context.data[context.ptr]
        if ch == ';':
            context.ptr += 1
            break
        if ch in ['"', "'"]:
            value += _read_string(context, ch)
        elif ch == '\\':
            if context.ptr + 1 < len(context.data):
                if context.data[context.ptr + 1] in ['"', "'"]:
                    value += _read_string(context, ch +
                                          context.data[context.ptr + 1])
                else:
                    return None
            else:
                return None
        else:
            value += ch
            context.ptr += 1
    return value


def _read_remaining(context):
    """ Used by _parse_form_option() to read remaining part of the --form
        option after the initial name=value part.  Returns '' if no value is
        present.
    """
    remaining = ''
    while context.ptr < len(context.data):
        ch = context.data[context.ptr]
        remaining += ch
        context.ptr += 1
    return remaining


def _parse_form_option(context):
    """Parse curl's --form option and output the parsed strings in
       context.parsed_data list.
    """
    # The string will be empty for -F =@a.out
    name = _read_name(context)
    ch = _read_equals(context)
    if ch:
        value = _read_value(context)
        # The remaining part of the option after the first semi-colon.
        # For example ";type=image/jpeg"
        extra = _read_remaining(context)
        context.parsed_data.append((name, value, extra))
    else:
        context.parsed_data.append((name, '', ''))


def _get_http_method(options):
    """ Get the HTTP method name from options.
    """
    if options.request:
        return options.request
    if options.form:
        return 'POST'
    if options.upload_file:
        return 'PUT'
    return 'GET'


class ChunkedData:
    """Class to implement -d option.  Supports read() function used by httplib
       requests() to read the -d data to be sent to the server.  This approach
       supports large files by reading in chunks, and doesn't create any
       intermediate files. It does this by reading through the data twice so
       that the body size can be computed before sending any HTTP multipart
       data to the server.  The -d option is used for sending text, not binary
       data. If the -d option is used with binary data, then curl appears to
       send data up to the first '\0' character.
    """
    def __init__(self):
        """Constructor.
        """
        self._value = None
        self._do_read_file = False
        self._file_handle = None
        self._done = False
        self._size = None  # body content length of -d option
        self._is_dir = False

    def set_config(self, data_value):
        """This function is separate from the __init__ function so that parsing
           errors can be returned rather than throwing a constructor exception.
           The data_value is the value of the -d option, which is a string,
           where the first character may be an @, which means the data should
           be read from a file.  Returns (errcode, errstr).
        """
        if data_value.startswith('@'):
            self._value = data_value[1:]
            if os.path.isdir(self._value):
                self._is_dir = True
            else:
                try:
                    tmpfp = open(self._value, "rb")
                    tmpfp.close()
                    self._do_read_file = True
                except IOError:
                    return CURLE_READ_ERROR, CURLE_OPTION_D_ERRSTR
        else:
            self._value = data_value
            self._do_read_file = False
        return 0, None

    def read(self, blocksize=8192):
        """During the httplib requests() call, this read() function is called
           to read headers and data, to be sent to the server.
        """
        char_null = b'\0' if have_py3 else '\0'
        char_newline = b'\n' if have_py3 else '\n'
        char_carriage = b'\r' if have_py3 else '\r'
        char_empty = b'' if have_py3 else ''
        if self._done:
            return None
        if self._do_read_file:
            if not self._file_handle:
                # If this open throws an exception then we didn't
                # detect the problem in set_config().
                self._file_handle = open(self._value, "rb")
                # -d is used for sending text but we read data as binary and
                # process it as binary.
            chunk = self._file_handle.read(blocksize)
            if chunk:
                i = chunk.find(char_null)
                if i >= 0:
                    # mimic curl's truncation of binary data
                    chunk = chunk[:i]
                    self._done = True
                else:
                    # Remove newlines in -d data
                    chunk = chunk.replace(char_newline, char_empty)
                    chunk = chunk.replace(char_carriage, char_empty)
            return chunk
        if self._is_dir:
            # curl happily allows -d @/ and sends a body of 0 bytes
            return None
        self._done = True
        return self._value.encode(UTF8)

    def get_size(self):
        """Return the size of the full HTTP body.  This size is computed by
           reading through all the data to be sent (before it is sent).
           We can't just get the size of the file because curl removes
           newlines from the data before sending to the server.
        """
        if self._size:
            return self._size
        self.reset()
        total_size = 0
        while True:
            chunk = self.read()
            if not chunk:
                break
            total_size += len(chunk)
        self._size = total_size
        return self._size

    def reset(self):
        """Close the file handles and reset data structures, so that the read()
           can start from the beginning again.  Note that _file_handles and
           _body_stage_done arrays have the same index.
        """
        if self._file_handle:
            self._file_handle.close()
            self._file_handle = None
        self._done = False


class ChunkedMultiPart:
    """Class to implement -F option.  Supports read() function used by httplib
       requests() to read the -F data to be sent to the server.  The read()
       function includes a simple state machine to provide -F data in multipart
       HTTP format.  This approach supports large files by reading in chunks,
       and doesn't create any intermediate files. It does this by
       reading through the data twice so that the body size can be
       computed before sending any HTTP multipart data to the server.
    """

    def __init__(self, hdrs):
        """Constructor.
        """
        self._boundary = self.generate_boundary().encode(UTF8)
        ct = CT_MULTIPART_FORM_DATA
        if HDR_CONTENT_TYPE in hdrs:
            ct = hdrs[HDR_CONTENT_TYPE]
        self._content_type = (ct + '; boundary=' +
                              self._boundary.decode())
        self._index = 0    # iterate through each of the multiple -F options
        self._stage = 0    # state variable for each -F option
        self._size = None  # body content length of all -F options
        self._form_data = []        # indexed by _index
        self._file_handles = []     # indexed by _index
        self._body_stage_done = []  # indexed by _index
        self._is_any_dir = False    # any directories in any of the -F options

    def set_config(self, form_data):
        """This function is separate from the __init__ function so that parsing
           errors can be returned rather than throwing a constructor exception.
           Example form_data: [('username', 'JohnDoe'), ('email',
           'john.doe@example.com'), ('profile_picture', '@portrait.jpg')]
           Returns (errcode, errstr).
        """
        for (name, value, extra) in form_data:
            # Parse through each -F option returning an error if any parsing
            # problems are found.  Determine for each -F option if a file read
            # is required.  Return results in self._form_data, which is an
            # array of (name, value, do_read_file). This self._form_data is
            # then used by the read() function to actually read any files.

            do_read_file = False
            if value.startswith('@'):
                do_read_file = True
                value = value[1:]
                if os.path.isdir(value):
                    self._is_any_dir = True
                else:
                    try:
                        tmpfp = open(value, "rb")
                        tmpfp.close()
                    except IOError:
                        return CURLE_READ_ERROR, CURLE_READ_ERRSTR
            content_type = None
            for word in extra.split(';'):
                i = word.find('type=')
                if i >= 0:
                    content_type = word[i+5:]
            if not content_type:
                content_type = self.get_content_type(value)
            self._form_data.append((name, value, content_type, do_read_file))
            self._file_handles.append(None)
            self._body_stage_done.append(False)
        return 0, None

    def get_content_disposition(self, content_type_header):
        # Based on emperical tests of curl behavior.  The Content-Disposition
        # is 'form-data' when the Content-Type in the header is
        # 'multipart/form-data', otherwise the Content-Disposition is
        # 'attachment'.
        if CT_MULTIPART_FORM_DATA in content_type_header:
            return CD_FORM_DATA
        return CD_ATTACHMENT

    def get_pre_stage(self, index):
        """Return the multi-part header for the -F option at postion index.
        """
        crlf = b'\r\n'
        lines = []
        name, value, content_type, do_read_file = self._form_data[index]
        if index > 0:
            lines.append(b'')
        lines.append(b'--' + self._boundary)
        line = b'Content-Disposition: ' + \
            self.get_content_disposition(
                self.get_content_type_header()).encode(UTF8)
        if name:
            line += b'; name="' + name.encode(UTF8) + b'"'
        if do_read_file:
            line += b'; filename="' + value.encode(UTF8) + b'"'
        lines.append(line)
        if do_read_file:
            lines.append(b'Content-Type: ' + content_type.encode(UTF8))
        lines.append(b'')
        lines.append(b'')
        return crlf.join(lines)

    def get_body_stage(self, index, blocksize):
        """Return the multi-part body for the -F option at postion index.
        """
        _, value, _, do_read_file = self._form_data[index]
        if do_read_file:
            if os.path.isdir(value):
                # mimic curl for directories, eg -F @/
                return None
            if not self._file_handles[index]:
                self._file_handles[index] = open(value, "rb")
            return self._file_handles[index].read(blocksize)
        if self._body_stage_done[index]:
            return None
        self._body_stage_done[index] = True
        return value.encode(UTF8)

    def get_post_stage(self):
        """Return the final boundary.
        """
        crlf = b'\r\n'
        lines = []
        lines.append(b'')
        lines.append(b'--' + self._boundary + b'--')
        lines.append(b'')
        return crlf.join(lines)

    def read(self, blocksize=8192):
        """During the httplib requests() call, this read() function is called
           to read headers and data, to be sent to the server.
        """
        if self._is_any_dir:
            # Mimic curl. Send no data.
            return None
        if self._stage == 0:
            self._stage += 1
            return self.get_pre_stage(self._index)

        if self._stage == 1:
            chunk = self.get_body_stage(self._index, blocksize)
            if chunk:
                return chunk
            self._stage += 1

        if self._stage == 2:
            if self._index == len(self._form_data) - 1:
                self._stage += 1
                return self.get_post_stage()
            self._index += 1
            self._stage = 1
            return self.get_pre_stage(self._index)
        return None

    def get_size(self):
        """Return the size of the full HTTP body.  This size is computed by
           reading through all the data to be sent (before it is sent).
        """
        if self._size:
            return self._size
        self.reset()
        total_size = 0
        while True:
            chunk = self.read()
            if not chunk:
                break
            total_size += len(chunk)
        self._size = total_size
        return self._size

    def reset(self):
        """Close the file handles and reset data structures, so that the read()
           can start from the beginning again.  Note that _file_handles and
           _body_stage_done arrays have the same index.
        """
        for index in range(len(self._file_handles)):
            if self._file_handles[index]:
                self._file_handles[index].close()
                self._file_handles[index] = None
            self._body_stage_done[index] = False
        self._index = 0
        self._stage = 0

    def get_content_type(self, filename):
        """Guess the mime type based on the filename extension.
        """
        content_type, _ = mimetypes.guess_type(filename)
        if not content_type:
            content_type = "application/octet-stream"
        return content_type

    def get_content_type_header(self):
        """Return the HTTP content type.
        """
        return self._content_type

    def generate_boundary(self):
        """Generate "boundary" string for curl's --data and --form options.  In
           one PUT or POST API, one boundary string is used to separate
           multiple attachments.
        """
        boundary_chars = string.digits + string.ascii_letters
        return '------------------------' + \
               ''.join(random.choice(boundary_chars) for i in range(22))


def _prepare_form_data(options, hdrs):
    """Helper function to parse -F option parameters and construct
       ChunkedMultiPart object supporing a read() function for the httplib
       requests() to invoke.  Returns errcode, errstr, multi_body
    """
    form_data = []
    for form_option in options.form:
        # option.form is a list where each item is a single -F option.
        # In this for loop, do one -F option at a time.
        ctx = SimpleParseContext()
        ctx.data = form_option
        _parse_form_option(ctx)
        form_data.extend(ctx.parsed_data)
    multi_body = ChunkedMultiPart(hdrs)
    errcode, errstr = multi_body.set_config(form_data)
    return errcode, errstr, multi_body


def _httplib_no_follow(url, was_redirected, options, output_file):
    """Use httplib to simulate curl functionality and write the REST response
       to an output_file.  The output_file may be stdout or an actual file.
       Tested on ESXi 9.0.0 build 24957456.
       Tested on ESXi 7.0.3 build 18644231.
       Tested on ESXi 6.7.0 build 14320388.
       Return the tuple (errcode, httpcode).
    """
    try:
        # We use the url function parameter rather than options.url because we
        # may have been redirected by the server.
        url_parts = urlparse(url)
        host = url_parts.hostname
        port = url_parts.port
        path = url_parts.path
        if url_parts.query:
            path = path + '?' + url_parts.query
        if not port:
            port = 80 if url_parts.scheme == "http" else 443
    except ValueError:
        errcode = CURLE_URL_MALFORMAT
        if was_redirected:
            errstr = "Found bad URL in Location header: " + url
        else:
            errstr = CURLE_URL_MALFORMAT_ERRSTR
        _set_err_stderr(options, errcode, errstr)
        return errcode, INVALID_HTTP_CODE

    _log_trying_connection(options, host, port, True)

    # EAL4_Zeroize_Sensitive_Data_By_Caller
    # username_passwd zeroized by the calling call_curl() function.
    username_passwd = options.user
    hdrs = {}
    if username_passwd:
        username_passwd = b64encode(
            username_passwd.encode(UTF8)).decode(ASCII)
        # EAL4_Zeroize_Sensitive_Data
        # hdrs[AUTHORIZATION_HDR] zeroized in finally block
        hdrs.update({AUTHORIZATION_HDR: 'Basic %s' % username_passwd})

    conn = None
    errcode = CURLE_OK
    httpcode = -1

    # timeout is connection timeout and is set to the smaller of max_time and
    # connect_timeout.  The max_time is used for each retry. In other words, if
    # max_time is 30 seconds, and the retry is 6, then we will retry 6 times
    # waiting for 30 seconds on each try. The total wait time will be 6 * 30
    # seconds plus the sleep time between each retry.
    timeout = _get_min_timeout(options, True)

    # Note that the Authorization header in the -H option overrides the
    # Authorization header constructed by the -u username/password option (as
    # it does in curl).
    hdrs.update(options.header)
    if 'User-Agent' not in hdrs:
        hdrs.update({'User-Agent': CURL_WRAPPER_TAG})

    try:
        if url.lower().startswith("https:"):
            errcode, errstr = _validate_peer_cert_chain(options, host, port)
            if errcode:
                # Either unable to fetch cert or cert failed validation.
                _set_err_stderr(options, errcode, errstr)
                return errcode, INVALID_HTTP_CODE
            # In case of CA-signed certs it would be preferable to avoid
            # setting create_unverified_context(), however on ESX, Python
            # complains when validating some CA-signed certs: [SSL:
            # CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed
            # certificate in certificate chain (_ssl.c:1125)
            ctx = ssl._create_unverified_context()
            if options.key and options.cert:
                conn = httplib.HTTPSConnection(host, port, context=ctx,
                                               timeout=timeout,
                                               key_file=options.key,
                                               cert_file=options.cert)
            else:
                conn = httplib.HTTPSConnection(host, port, context=ctx,
                                               timeout=timeout)
        else:
            conn = httplib.HTTPConnection(host, port,
                                          timeout=timeout)

        http_method = _get_http_method(options)
        if options.data:
            # The --data option is supposed to be for text data.  The
            # --binary-data is for binary data (however curl_wrapper doesn't
            # support this option yet).
            data_body = ChunkedData()
            errcode, errstr = data_body.set_config(options.data)
            if errcode:
                _set_err_stderr(options, errcode, errstr)
                return errcode, INVALID_HTTP_CODE
            # Note: If -H 'Content-Type:' is not provided then curl emits
            # Content-Type: application/x-www-form-urlencoded
            if HDR_CONTENT_TYPE not in hdrs:
                hdrs.update({HDR_CONTENT_TYPE:
                             'application/x-www-form-urlencoded'})
            hdrs.update({'Content-Length': data_body.get_size()})
            data_body.reset()  # close files for re-opening/re-reading
            _print_headers_stderr(options, http_method, path, hdrs)
            conn.request(http_method, path, data_body, hdrs)
            data_body.reset()  # close files
        elif options.form:
            errcode, errstr, multi_body = _prepare_form_data(options, hdrs)
            if errcode:
                _set_err_stderr(options, errcode, errstr)
                return errcode, INVALID_HTTP_CODE
            if HDR_CONTENT_TYPE not in hdrs:
                hdrs.update({HDR_CONTENT_TYPE:
                             multi_body.get_content_type_header()})
            clength = multi_body.get_size()
            if clength == 0:
                # mimic curl
                hdrs.update({'Expect': '100-continue'})
                hdrs.update({'Transfer-Encoding': 'chunked'})
            else:
                hdrs.update({'Content-Length': clength})
            multi_body.reset()  # close files for re-opening/re-reading
            _print_headers_stderr(options, http_method, path, hdrs)
            conn.request(http_method, path, multi_body, hdrs)
            multi_body.reset()  # close files
        elif options.upload_file:
            upload_data = None
            upload_filename = options.upload_file[0]
            if not os.path.isdir(upload_filename):
                try:
                    upload_data = open(upload_filename, 'rb')
                except IOError:
                    errcode = CURLE_READ_ERROR
                    _set_err_stderr(options, errcode, CURLE_READ_ERRSTR)
                    return errcode, INVALID_HTTP_CODE
            if upload_data:
                hdrs.update({'Content-Length':
                             os.path.getsize(upload_filename)})
            else:
                # mimic curl
                hdrs.update({'Expect': '100-continue'})
                hdrs.update({'Transfer-Encoding': 'chunked'})
            _print_headers_stderr(options, http_method, path, hdrs)
            # curl doesn't include Content-Length when the file is a
            # directory.  Unfortunately, requests() adds 'Content-Length' if
            # not in hdrs.  A work-around would be to implement the ChunkedData
            # pattern for more control.
            conn.request(http_method, path, upload_data, hdrs)
            if upload_data:
                upload_data.close()
        else:
            _print_headers_stderr(options, http_method, path, hdrs)
            conn.request(http_method, path, None, hdrs)
        resp = conn.getresponse()

        content_length = -1  # unknown
        for key, value in resp.getheaders():
            if key == 'content-length':
                content_length = int(value.encode(UTF8))
                break
        if options.include or options.head:
            version = str(resp.version)
            version = "HTTP/%s.%s" % (version[0], version[1])
            output_file.write(("%s %s %s\n" % (version, resp.status,
                                               resp.reason)).encode(UTF8))
            for key, value in resp.getheaders():
                output_file.write(("%s: %s\n" % (key, value)).encode(UTF8))
        if options.location:
            # Auto-redirection
            resp_headers = dict(resp.getheaders())
            if 'Location' in resp_headers and resp_headers['Location'] != url:
                return THIS_WAS_REDIRECTED, resp_headers['Location']
        if not options.head:
            recv_length = 0
            empty_reads_consecutive = 0
            while True:
                buffer = resp.read(65536)
                recv_length += len(buffer)
                if len(buffer) > 0:
                    empty_reads_consecutive = 0
                    output_file.write(buffer)
                else:
                    empty_reads_consecutive += 1
                    # content_length is -1 if unknown
                    if content_length == recv_length:
                        break
                    # resp.closed defined only in Python 3.
                    if ((hasattr(resp, 'closed') and resp.closed)
                            or empty_reads_consecutive >= MAX_EMPTY_READS):
                        if content_length > 0:
                            errcode = CURLE_PARTIAL_FILE
                        break
                    time.sleep(1)
        httpcode = resp.status
        return errcode, httpcode
    except AttributeError as ex:
        # _get_peer_cert_chain() may throw an AttributeError exception
        # and when it does, handle it has a non-recoverable error (that is
        # don't retry when the --retry option is used). Specifically, don't
        # return with a CURLE_COULDNT_CONNECT error.
        submsg = str(ex)
        errcode = CURLE_SSL_CONNECT_ERROR
        errstr = ("Failed to retrieve cert chain from " + host + " port " +
                  str(port) + ": " + submsg)
        _set_err_stderr(options, errcode, errstr)
        return errcode, INVALID_HTTP_CODE
    # Whenever we catch Exception, always catch KeyboardInterrupt prior to
    # catching Exception and throw it.
    except KeyboardInterrupt as ex:
        errcode = THIS_KEYBOARD_INTERRUPT
        raise ex
    except Exception as ex:
        submsg = _submsg_translate(str(ex))
        errcode = _errcode_from_options(CURLE_COULDNT_CONNECT, options)
        errstr = ("Failed to connect to " + host + " port " +
                  str(port) + ": " + submsg)
        _log_backtrace(options, errstr,
                       traceback.format_exc())
        _set_err_stderr(options, errcode, errstr)
        return errcode, INVALID_HTTP_CODE
    finally:
        if errcode != THIS_KEYBOARD_INTERRUPT:
            _log_closing_connection(options)
        if conn:
            conn.close()
        if AUTHORIZATION_HDR in hdrs:
            nz.zeroize(hdrs[AUTHORIZATION_HDR])


class Stats:
    """Used by _print_write_out to print the --write-out format string.
    """
    def __init__(self):
        self.url_effective = ''
        self.http_code = INVALID_HTTP_CODE


class WriteOutParseContext:
    """Used by _print_write_out to print the --write-out format string.
    """
    def __init__(self):
        self.errstr = ''
        self.input = ''
        self.ptr = 0


def _read_char(context):
    """Used by _print_write_out to print the --write-out format string.
    """
    if context.ptr >= len(context.input):
        return None
    char = context.input[context.ptr]
    context.ptr += 1
    return char


def _print_write_out(options, stats):
    """Print the --write-out format string to stdout.  Returns no error code.
       curl doesn't return an error code if there are formatting errors in the
       format string, nor are the error messages silenced with the -s option.
    """
    if options.errcode == THIS_KEYBOARD_INTERRUPT:
        return
    output = ''
    context = WriteOutParseContext()
    errstrfmt = "curl_wrapper: unknown --write-out variable: '{0}'"
    context.input = options.write_out

    if not context.input:
        # Nothing to write out.
        return
    TEXT_MODE = 0
    VARIABLE_MODE = 1
    mode = TEXT_MODE
    variable = ''
    while True:
        char = _read_char(context)
        if not char:
            break
        if mode == TEXT_MODE:
            if char == '%':
                char = _read_char(context)
                if not char:
                    break
                if char == '{':
                    mode = VARIABLE_MODE
                    variable = ''
                elif char == '%':
                    output += char
                else:
                    output += '%' + char
            elif char == '\\':
                char = _read_char(context)
                if not char:
                    break
                elif char == 'n':
                    output += '\n'
                elif char == 'r':
                    output += '\r'
                elif char == 't':
                    output += '\t'
                else:
                    output += char
            else:
                output += char
        else:
            # Parsing a variable in this block, eg %{url_effective}
            if char == '}':
                if variable == 'url_effective':
                    output += stats.url_effective
                elif variable == 'http_code':
                    if stats.http_code == INVALID_HTTP_CODE:
                        output += '000'
                    else:
                        output += str(stats.http_code)
                else:
                    errstr = errstrfmt.format(variable)
                    print(errstr, file=sys.stderr)
                    return
                mode = TEXT_MODE
                variable = ''
            else:
                variable += char
    if variable:
        errstr = errstrfmt.format(variable)
        print(errstr, file=sys.stderr)
        return
    print(output, end='')
    return


def _httplib_to_file(options, output_file, stats):
    """Use httplib to simulate curl functionality and write the REST response
       to an output_file.  The output_file may be stdout or an actual file.
       Tested on ESXi 7.0.3 build 18644231.
       Tested on ESXi 6.7.0 build 14320388.
       Return the tuple (errcode, httpcode).
       """

    url = options.url
    stats.url_effective = options.url

    if not options.location:
        errcode, httpcode = _httplib_no_follow(options.url, False, options,
                                               output_file)
        stats.http_code = httpcode
        return errcode, httpcode

    if options.max_redirs is None or options.max_redirs < 0:
        redirections = DEFAULT_MAX_REDIRECTS
    else:
        redirections = options.max_redirs

    was_redirected = False
    for _ in range(redirections + 1):
        errcode, httpcode = _httplib_no_follow(url, was_redirected,
                                               options, output_file)
        stats.http_code = httpcode
        if errcode != THIS_WAS_REDIRECTED:
            return errcode, httpcode
        # When errcode is THIS_WAS_REDIRECTED then httpcode is the new url.
        url = httpcode
        stats.url_effective = options.url
        was_redirected = True
    errcode = CURLE_TOO_MANY_REDIRECTS
    errstr = "Maximum (" + str(redirections) + ") redirects followed"
    _set_err_stderr(options, errcode, errstr)
    return errcode, INVALID_HTTP_CODE


def _append_curl_headers(cmd, headers):
    """ Append -H headers for curl command.
    """
    for key, value in headers.items():
        cmd.append('-H')
        cmd.append(key + ": " + value)


def build_curl_cmd(options, last_try, output_file):
    """Build curl command line options in a list and return the list.
       The assumption is that this function is never called unless the
       server's cert has already been validated. Therefore we always
       append the -k option to any https curl calls.
    """
    cmd = []
    cmd.append('/usr/bin/curl')
    if options.url.lower().startswith("https:"):
        cmd.append('-k')
    if options.connect_timeout:
        cmd.append('--connect-timeout')
        cmd.append(str(options.connect_timeout))
    if options.data:
        cmd.append('-d')
        cmd.append(options.data)
    if options.form:
        for option in options.form:
            cmd.append('-F')
            cmd.append(option)
    if options.head:
        cmd.append('-I')
    if len(options.header) > 0:
        _append_curl_headers(cmd, options.header)
    if options.include:
        cmd.append('-i')
    # options.insecure (-k) is handled above
    if options.location:
        cmd.append('-L')
    if options.max_redirs:
        cmd.append('--max-redirs')
        cmd.append(str(options.max_redirs))
    if options.max_time > 0:
        cmd.append('-m')
        cmd.append(str(options.max_time))
    # skip output because we must use output_file instead
    if output_file:
        cmd.append('-o')
        cmd.append(output_file)
    # skip remote_name because we must use output_file instead
    if options.upload_file:
        for option in options.upload_file:
            cmd.append('-T')
            cmd.append(option)
    if options.request:
        cmd.append('-X')
        cmd.append(options.request)
    # skip retry because it is handled by _call_http()
    # skip retry_delay because it is handled by _call_http()
    # skip retry_max_time because it is handled by _call_http()
    # always use silent mode
    cmd.append('-s')
    if options.show_error or not options.silent:
        if last_try:
            cmd.append('-S')
    # skip thumbprint because curl doesn't support this parameter
    if options.user:
        cmd.append('-u')
        cmd.append(options.user)
    if options.verbose:
        cmd.append('-v')
    # curl to write out all variables supported by this script
    if options.write_out:
        cmd.append('-w')
        # The number of words in this string must match NUM_WRITE_OUT_WORDS
        cmd.append(CURL_WRAPPER_TAG + " %{http_code} %{url_effective}")
    cmd.append(options.url)
    return cmd


def _collect_stats(stats, line):
    """Extract the --write-out values from the stdout output emitted by curl.
       This function is only called if line has NUM_WRITE_OUT_WORDS words.
    """
    words = line.split()
    stats.http_code = int(words[1])
    stats.url_effective = words[2].decode(UTF8)


def _write_to_stdout(options, stats, tag, line):
    """Function to delay the writing out of curl's stdout so we can detect the
       last line having curl's --write-out results
    """
    if options.prev_stdout_line:
        if (not line and options.write_out and
                tag in options.prev_stdout_line and
                len(options.prev_stdout_line.split()) == NUM_WRITE_OUT_WORDS):
            # last line has been read and the special tag is present
            _collect_stats(stats, options.prev_stdout_line)
            # remove tag and write-out results
            options.prev_stdout_line = \
                options.prev_stdout_line[0: options.prev_stdout_line.find(tag)]
        if hasattr(sys.stdout, "buffer"):
            # Python3
            sys.stdout.buffer.write(options.prev_stdout_line)
        else:
            # Python2
            sys.stdout.write(options.prev_stdout_line)
    options.prev_stdout_line = line


def _curl_to_file(options, last_try, output_file, stats):
    """Call curl to write the REST response to an output_file.
       Return the tuple (errcode, httpcode).  This function is not invoked
       unless _is_trust_established() returns True. See the caller
       _http_to_file().
    """
    cmd_args = build_curl_cmd(options, last_try, output_file)
    df = None
    _ex = None
    try:
        host = options.host
        port = options.port
        _log_trying_connection(options, host, port, False)
        # Important to call sys.stderr.flush() before invoking a subprocess. In
        # our case, the log_trying_connection calls the flush() function.
        _log_command(options, cmd_args, True)
        df = subprocess.Popen(cmd_args, stdout=subprocess.PIPE)
        if have_py3:
            tag = bytes(CURL_WRAPPER_TAG, UTF8)
        else:
            tag = CURL_WRAPPER_TAG
        for line in df.stdout:
            _write_to_stdout(options, stats, tag, line)
        _write_to_stdout(options, stats, tag, None)
        df.communicate()[0]
    except Exception as ex:
        _ex = ex
        _log_backtrace(options, "curl error",
                       traceback.format_exc())
    if last_try:
        options.last_curl_fin = True
    errcode = df.returncode if df else CURLM_INTERNAL_ERROR
    if not _ex or str(_ex) != EXCEPTION_TIMEDOUT_MSG:
        _log_closing_connection(options)
    # The cmd_args includes secrets but because it includes a shallow copy
    # of secrets in 'options' we don't zeroize the cmd_args.
    return errcode, stats.http_code


def _http_to_file(first_try, last_try, options, output_filename, stats):
    """On first attempt use httplib to simulate curl functionality and write
       the REST response to an output_file.  On subsequent attempts use curl if
       it is available.  Return the tuple (errcode, httpcode).
    """
    # curl_to_file doesn't validate the peer cert so until we have completed
    # the step of validating the peer cert we continue using _httplib_to_file.
    if (first_try or not exists('/usr/bin/curl') or not
            _is_trust_established(options)):
        if not output_filename:
            if hasattr(sys.stdout, "buffer"):
                # Python3
                file = sys.stdout.buffer
            else:
                # Python2
                file = sys.stdout
            return _httplib_to_file(options, file, stats)
        else:
            with open(output_filename, 'wb') as file:
                return _httplib_to_file(options, file, stats)
    else:
        # output_filename may be None which means stdout
        return _curl_to_file(options, last_try, output_filename, stats)


def _call_http(options):
    """Use httplib to simulate curl functionality and write the REST response
       to an output_file.  The function simulates curl's retry options.
       This function returns a single error code, where 0 indicates success.
    """
    stats = Stats()
    output_filename = None
    if options.remote_name:
        output_filename = options.url.split('/')[-1]
    else:
        output_filename = options.output

    if options.retry_max_time > 0:
        stop_time = datetime.now() + timedelta(seconds=options.retry_max_time)
    sleep_time = 1 if options.retry_delay <= 0 else options.retry_delay
    errcode = CURLM_INTERNAL_ERROR
    httpcode = -1

    retry = options.retry
    first_try = True
    try:
        while retry >= 0:
            # options.num_retry_conns is initialized to -1 and here
            # we increment to 0.
            options.num_retry_conns += 1
            if (output_filename and exists(output_filename) and
                    output_filename != '/dev/null'):
                os.remove(output_filename)
            last_try = retry == 0
            (errcode, httpcode) = _http_to_file(first_try, last_try, options,
                                                output_filename, stats)
            # Repeat if transient error.
            if not _is_transient_error(errcode, httpcode):
                break
            retry -= 1
            first_try = False
            if retry >= 0:
                # mimic curl's --retry
                if options.retry_max_time > 0:
                    now = datetime.now()
                    if now > stop_time:
                        errcode = CURLE_COULDNT_CONNECT  # timeout
                        break
                    if now + timedelta(seconds=sleep_time) > stop_time:
                        time.sleep((stop_time - now).total_seconds())
                        errcode = CURLE_COULDNT_CONNECT  # timeout
                        break
                _log_transient_problem(options, sleep_time, retry+1)
                time.sleep(sleep_time)
                if options.retry_delay <= 0:
                    # curl man page says maximum sleep time is 10 minutes,
                    # which is 600 seconds.
                    sleep_time = min(sleep_time * 2, 600)
                else:
                    sleep_time = options.retry_delay
    except KeyboardInterrupt:
        errcode = THIS_KEYBOARD_INTERRUPT
        _set_err_stderr(options, errcode, "")
    finally:
        # when writing to stdout output_filename will be None
        if errcode and output_filename and output_filename != '/dev/null':
            if exists(output_filename):
                os.remove(output_filename)
        if not errcode:
            # It is possible for options.errcode to have been set earlier but
            # for errcode to be zero at this point.  If this is the case
            # errcode should be used, and we must set options.errcode
            # accordingly before calling _print_final_msgs().
            _set_err_stderr(options, errcode, "")
        _print_final_msgs(options, stats)
    return errcode


def additional_option_checks(options):
    """Check if mutually exclusive options have been provided.
       Return the tuple (errcode, errstr) where errcode and errstr mimic curl's
       response.
    """
    cnt = 0
    elist = []
    if options.data:
        cnt += 1
        elist.append("POST (-d, --data)")
    if options.form:
        cnt += 1
        elist.append("multipart formpost (-F, --form)")
    if options.upload_file:
        cnt += 1
        elist.append("PUT (-T, --upload-file)")
    if options.head:
        cnt += 1
        elist.append("HEAD (-I, --head)")
    if cnt > 1:
        # curl treats -d as POST even when user gives a different method for -X
        # curl treats -T as PUT even when user gives a different method for -X
        first_line = elist[0]
        second_line = elist[1]
        search_words = ["PUT", "POST"]

        if ((not any(word in first_line for word in search_words))
                and any(word in second_line for word in search_words)):
            # The first line doesn't have either PUT or POST but the the
            # second line does, so swap them to mimic curl.
            first_line = elist[1]
            second_line = elist[0]

        for method in search_words:
            if method in first_line:
                first_line = first_line.replace(method, method + "\nWarning:")
        errstr = ("Warning: You can only select one HTTP request method! "
                  "You asked for both " + first_line + " and " +
                  second_line + ".")
        return CURLE_FAILED_INIT, errstr
    if options.thumbprint and options.cacert:
        errstr = "Options --thumbprint and --cacert are mutually exclusive"
        return CURLE_FAILED_INIT, errstr
    if options.cacert and not exists(options.cacert):
        errstr = CACERT_BADFILE_ERRSTR + options.cacert
        return CURLE_SSL_CACERT_BADFILE, errstr
    if options.upload_file and len(options.upload_file) > 1:
        errstr = "curl_wrapper supports at most one -T option"
        return CURLE_FAILED_INIT, errstr
    if options.cacert_is_ca and not options.cacert:
        errstr = "Option --cacert-is-ca requires option --cacert"
        return CURLE_FAILED_INIT, errstr
    return 0, None


def call_curl(args):
    """Top-level function that can be called from another python program.
       The function name is a bit misleading because this function used to
       invoke the curl binary but the curl binary was dropped because curl
       doesn't support checking trust with a thumbprint.  Instead this script
       uses the python library httplib to simulate curl.
       This function returns a single error code, where 0 indicates success.

       The call_curl function can be called as follows:
       import imp
       curl_wrapper = imp.load_source('curl_wrapper',
           '/opt/vmware/nsx-common/python/nsx_utils/curl_wrapper')
       args = ("-u admin:password -i "
               "https://10.192.193.86/api/v1/node/aaa/providers/vidm").split()
       sys.exit(curl_wrapper.call_curl(args))
    """
    # Step 1: Use argparse to parse arguments into a list
    parser = argparse.ArgumentParser()
    parser.add_argument('--cacert', required=False)
    parser.add_argument('--cacert-is-ca', action='store_true', required=False)
    parser.add_argument('--cert', required=False)
    parser.add_argument('--connect-timeout', required=False)
    parser.add_argument('-d', '--data', required=False)
    parser.add_argument('-F', '--form', action='append', required=False)
    parser.add_argument('-H', '--header', action='append', required=False)
    parser.add_argument('-i', '--include', action='store_true', required=False)
    parser.add_argument('-I', '--head', action='store_true', required=False)
    parser.add_argument('-k', '--insecure', action='store_true',
                        required=False)
    parser.add_argument('--key', required=False)
    parser.add_argument('-L', '--location', action='store_true',
                        required=False)
    parser.add_argument('--max-redirs', required=False)
    parser.add_argument('-m', '--max-time', required=False)
    parser.add_argument('--no-hostname-check', action='store_true',
                        required=False)
    parser.add_argument('-o', '--output', required=False)
    parser.add_argument('-O', '--remote-name', action='store_true',
                        required=False)
    parser.add_argument('--retry', required=False)
    parser.add_argument('--retry-delay', required=False)
    parser.add_argument('--retry-max-time', required=False)
    # This script does --silent always
    parser.add_argument('-s', '--silent', action='store_true', required=False)
    parser.add_argument('-S', '--show-error', action='store_true',
                        required=False)
    parser.add_argument('-T', '--upload-file', action='append', required=False)
    parser.add_argument('--thumbprint', required=False)
    parser.add_argument('-u', '--user', required=False)
    parser.add_argument('-v', '--verbose', action='store_true',
                        required=False)
    parser.add_argument('--validate-cert-from-file', required=False)
    parser.add_argument('-w', '--write-out', required=False)
    parser.add_argument('-X', '--request', required=False)
    parsed_args, extra_args = parser.parse_known_args(args)

    # Step 2: Convert argparse list to a compact structure
    options = CmdLineOptions()
    # Process the silent and show_error first because we need these if there
    # are problems parsing the other arguments
    try:
        options.silent = _get_silent_opt(parsed_args)
        options.show_error = _get_show_error_opt(parsed_args)

        (errcode, errstr, options.host, options.port, options.path,
            options.url) = _get_host_opt(extra_args)

        if parsed_args.validate_cert_from_file:
            # Test code.
            # Caller may optionally provide URL to set hostname.
            # Ignore "(2) no URL specified!".
            if errcode and errcode != CURLE_FAILED_INIT:
                _set_err_stderr(options, errcode, errstr)
                _print_final_msgs(options, None)
                return errcode
            if not have_cryptography and not have_crypto:
                errcode = CURLE_FAILED_INIT
                errstr = REQUIRES_CRYPTOGRAPHY_ERRSTR
                _set_err_stderr(options, errcode, errstr)
                _print_final_msgs(options, None)
                return errcode
            if not exists(parsed_args.validate_cert_from_file):
                errcode = CURLE_SSL_CACERT_BADFILE
                errstr = (CACERT_BADFILE_ERRSTR +
                          parsed_args.validate_cert_from_file)
                _set_err_stderr(options, errcode, errstr)
                _print_final_msgs(options, None)
                return errcode
            return _validate_server_cert_from_file(
                parsed_args.validate_cert_from_file,
                options.host, parsed_args.cacert, parsed_args.cacert_is_ca)

        if errcode:
            _set_err_stderr(options, errcode, errstr)
            _print_final_msgs(options, None)
            return errcode

        options.cacert = _get_cacert_opt(parsed_args)
        options.cacert_is_ca = _get_cacert_is_ca_opt(parsed_args)
        options.cert = _get_cert_opt(parsed_args)
        options.connect_timeout = _get_connect_timeout_opt(parsed_args)
        options.data = _get_data_opt(parsed_args)
        options.form = _get_form_opt(parsed_args)
        options.head = _get_head_opt(parsed_args)
        options.header = _get_request_headers_opt(parsed_args)
        options.include = _get_response_headers_opt(parsed_args)
        # options.insecure is not used
        options.key = _get_key_opt(parsed_args)
        options.location = _get_location_opt(parsed_args)
        options.max_redirs = _get_max_redirs_opt(parsed_args)
        options.max_time = _get_max_time_opt(parsed_args)
        options.no_hostname_check = _get_no_hostname_check_opt(parsed_args)
        options.output = _get_output_file_opt(parsed_args)
        options.remote_name = _get_remote_name_opt(parsed_args)
        options.request = _get_method_opt(parsed_args)
        options.retry = _get_retry_opt(parsed_args)
        options.retry_delay = _get_retry_delay_opt(parsed_args)
        options.retry_max_time = _get_retry_max_time_opt(parsed_args)
        # options.silent is handled above
        # options.show_error is handled above
        options.thumbprint = _get_thumbprint_opt(parsed_args)
        options.upload_file = _get_upload_opt(parsed_args)
        # EAL4_Zeroize_Sensitive_Data
        # options.user zeroized in finally block
        options.user = _get_user_passwd_opt(parsed_args)
        options.verbose = _get_verbose_opt(parsed_args)
        options.write_out = _get_write_out_opt(parsed_args)
        errcode, errstr = additional_option_checks(options)
        if errcode:
            _set_err_stderr(options, errcode, errstr)
            without_prefix = True
            if errcode == CURLE_SSL_CACERT_BADFILE:
                without_prefix = False
            _print_final_msgs(options, None, without_prefix)
            return errcode

        # Step 3: Do curl equivalent using compact option structure
        return _call_http(options)
    finally:
        nz.zeroize(options.user)


def _read_certs_from_file(filename):
    """Read collection of certs from a PEM encoded file.  When used to read a
       cert chain, the order is important (the leaf comes first).  When use to
       read certs from CA cert store, the order is not important.  Assumes the
       caller will check the file exists.  Throws an Exception if unable to
       parse the PEM encoding.
    """
    # 'encoding' supported only in Python3
    with open(filename, "r") as f:
        return _read_certs_from_openssl_output(f.read())


def _write_certs_to_file(certs, filename):
    # 'encoding' supported only in Python3
    with open(filename, "w") as f:
        return f.write(_dump_certchain_as_pem(certs))


def _validate_server_cert_from_file(filename, hostname=None, cacert_file=None,
                                    cacert_is_ca=None):
    """Test function to read a cert from a file and check it passes validation.
    """
    cert_chain = []
    try:
        cert_chain = _read_certs_from_file(filename)
    except ValueError:
        errstr = CACERT_BADFILE_ERRSTR + filename
        print(errstr, file=sys.stderr)
        print("Failed certificate validation")
        return CURLE_SSL_CACERT_BADFILE

    options = CmdLineOptions()
    options.silent = False
    options.show_error = True
    options.max_time = DEFAULT_TIMEOUT
    options.connect_timeout = DEFAULT_TIMEOUT
    options.cacert = cacert_file
    options.cacert_is_ca = cacert_is_ca

    # For testing purposes get the thumbprint from the cert.
    leaf_cert = _get_leaf_cert(cert_chain)
    options.thumbprint = _get_thumbprint(leaf_cert)
    if hostname:
        options.host = hostname
    else:
        cn = _get_first_common_name(leaf_cert)
        if cn.startswith('*.'):
            options.host = 'xxx' + cn[2:]
        else:
            options.host = cn

    _, errcode, errstr = _validate_cert(cert_chain, options.host, options)
    if errcode:
        # No check for silent and show_error here because this is a test
        # function.
        print(errstr, file=sys.stderr)
        print("Failed certificate validation")
    else:
        print("Passed certificate validation")
    return errcode


if __name__ == '__main__':
    # Main entry point.
    _log_command(None, sys.argv, True)
    merrcode = call_curl(sys.argv[1:])
    _log_exit_code(None, merrcode)
    sys.exit(merrcode)
