# -*- coding: utf-8 -*-
'''
A salt module for SSL/TLS.
Can create a Certificate Authority (CA)
or use Self-Signed certificates.

:depends:   - PyOpenSSL Python module (0.10 or later, 0.14 or later for
    X509 extension support)
:configuration: Add the following values in /etc/salt/minion for the CA module
    to function properly::

        ca.cert_base_path: '/etc/pki'

CLI Example #1
Creating a CA, a server request and its signed certificate:

.. code-block:: bash

    # salt-call tls.create_ca my_little \
    days=5 \
    CN='My Little CA' \
    C=US \
    ST=Utah \
    L=Salt Lake City \
    O=Saltstack \
    emailAddress=pleasedontemail@thisisnot.coms

    Created Private Key: "/etc/pki/my_little/my_little_ca_cert.key"
    Created CA "my_little_ca": "/etc/pki/my_little_ca/my_little_ca_cert.crt"

    # salt-call tls.create_csr my_little CN=www.thisisnot.coms
    Created Private Key: "/etc/pki/my_little/certs/www.thisisnot.coms.key
    Created CSR for "www.thisisnot.coms": "/etc/pki/my_little/certs/www.thisisnot.coms.csr"

    # salt-call tls.create_ca_signed_cert my_little CN=www.thisisnot.coms
    Created Certificate for "www.thisisnot.coms": /etc/pki/my_little/certs/www.thisisnot.coms.crt"

CLI Example #2:
Creating a client request and its signed certificate

.. code-block:: bash

    # salt-call tls.create_csr my_little CN=DBReplica_No.1 cert_type=client
    Created Private Key: "/etc/pki/my_little/certs//DBReplica_No.1.key."
    Created CSR for "DBReplica_No.1": "/etc/pki/my_little/certs/DBReplica_No.1.csr."

    # salt-call tls.create_ca_signed_cert my_little CN=DBReplica_No.1
    Created Certificate for "DBReplica_No.1": "/etc/pki/my_little/certs/DBReplica_No.1.crt"

CLI Example #3:
Creating both a server and client req + cert for the same CN

.. code-block:: bash

    # salt-call tls.create_csr my_little CN=MasterDBReplica_No.2  \
        cert_type=client
    Created Private Key: "/etc/pki/my_little/certs/MasterDBReplica_No.2.key."
    Created CSR for "DBReplica_No.1": "/etc/pki/my_little/certs/MasterDBReplica_No.2.csr."

    # salt-call tls.create_ca_signed_cert my_little CN=MasterDBReplica_No.2
    Created Certificate for "DBReplica_No.1": "/etc/pki/my_little/certs/DBReplica_No.1.crt"

    # salt-call tls.create_csr my_little CN=MasterDBReplica_No.2 \
        cert_type=server
    Certificate "MasterDBReplica_No.2" already exists

    (doh!)

    # salt-call tls.create_csr my_little CN=MasterDBReplica_No.2 \
        cert_type=server type_ext=True
    Created Private Key: "/etc/pki/my_little/certs/DBReplica_No.1_client.key."
    Created CSR for "DBReplica_No.1": "/etc/pki/my_little/certs/DBReplica_No.1_client.csr."

    # salt-call tls.create_ca_signed_cert my_little CN=MasterDBReplica_No.2
    Certificate "MasterDBReplica_No.2" already exists

    (DOH!)

    # salt-call tls.create_ca_signed_cert my_little CN=MasterDBReplica_No.2 \
        cert_type=server type_ext=True
    Created Certificate for "MasterDBReplica_No.2": "/etc/pki/my_little/certs/MasterDBReplica_No.2_server.crt"

CLI Example #4:
Create a server req + cert with non-CN filename for the cert

.. code-block:: bash

    # salt-call tls.create_csr my_little CN=www.anothersometh.ing \
        cert_type=server type_ext=True
    Created Private Key: "/etc/pki/my_little/certs/www.anothersometh.ing_server.key."
    Created CSR for "DBReplica_No.1": "/etc/pki/my_little/certs/www.anothersometh.ing_server.csr."

    # salt-call tls_create_ca_signed_cert my_little CN=www.anothersometh.ing \
        cert_type=server cert_filename="something_completely_different"
    Created Certificate for "www.anothersometh.ing": /etc/pki/my_little/certs/something_completely_different.crt
'''
from __future__ import absolute_import
# pylint: disable=C0103

# Import python libs
import os
import time
import calendar
import logging
import hashlib
import salt.utils
from salt._compat import string_types
from salt.ext.six.moves import range as _range
from datetime import datetime
from distutils.version import LooseVersion

import re

HAS_SSL = False
X509_EXT_ENABLED = True
try:
    import OpenSSL
    HAS_SSL = True
    OpenSSL_version = LooseVersion(OpenSSL.__dict__.get('__version__', '0.0'))
except ImportError:
    pass

# Import salt libs

log = logging.getLogger(__name__)

two_digit_year_fmt = "%y%m%d%H%M%SZ"
four_digit_year_fmt = "%Y%m%d%H%M%SZ"

def __virtual__():
    '''
    Only load this module if the ca config options are set
    '''
    global X509_EXT_ENABLED
    if HAS_SSL and OpenSSL_version >= LooseVersion('0.10'):
        if OpenSSL_version < LooseVersion('0.14'):
            X509_EXT_ENABLED = False
            log.error('You should upgrade pyOpenSSL to at least 0.14.1 '
                     'to enable the use of X509 extensions')
        elif OpenSSL_version <= LooseVersion('0.15'):
            log.warn('You should upgrade pyOpenSSL to at least 0.15.1 '
                     'to enable the full use of X509 extensions')
        # never EVER reactivate this code, this has been done too many times.
        # not having configured a cert path in the configuration does not
        # mean that users cant use this module as we provide methods
        # to configure it afterwards.
        # if __opts__.get('ca.cert_base_path', None):
        #     return True
        # else:
        #     log.error('tls module not loaded: ca.cert_base_path not set')
        #     return False
        return True
    else:
        X509_EXT_ENABLED = False
        return False, ['PyOpenSSL version 0.10 or later must be installed '
                       'before this module can be used.']

def cert_base_path(cacert_path=None):
    '''
    Return the base path for certs from CLI or from options

    cacert_path
        absolute path to ca certificates root directory

    CLI Example:

    .. code-block:: bash

        salt '*' tls.cert_base_path
    '''
    if not cacert_path:
        cacert_path = __context__.get(
            'ca.contextual_cert_base_path',
            __salt__['config.option']('ca.contextual_cert_base_path'))
    if not cacert_path:
        cacert_path = __context__.get(
            'ca.cert_base_path',
            __salt__['config.option']('ca.cert_base_path'))
    return cacert_path

def _cert_base_path(cacert_path=None):
    '''
    Retrocompatible wrapper
    '''
    return cert_base_path(cacert_path)

def set_ca_path(cacert_path):
    '''
    If wanted, store the aforementioned cacert_path in context
    to be used as the basepath for further operations

    CLI Example:

    .. code-block:: bash

        salt '*' tls.set_ca_path /etc/certs
    '''
    if cacert_path:
        __context__['ca.contextual_cert_base_path'] = cacert_path
    return cert_base_path()

def _new_serial(ca_name, CN):
    '''
    Return a serial number in hex using md5sum, based upon the ca_name and
    CN values

    ca_name
        name of the CA
    CN
        common name in the request
    '''
    opts_hash_type = __opts__.get('hash_type', 'md5')
    hashtype = getattr(hashlib, opts_hash_type)
    hashnum = int(
        hashtype(
            '{0}_{1}_{2}'.format(
                ca_name,
                CN,
                int(calendar.timegm(time.gmtime())))
        ).hexdigest(),
        16
    )
    log.debug('Hashnum: {0}'.format(hashnum))

    # record the hash somewhere
    cachedir = __opts__['cachedir']
    log.debug('cachedir: {0}'.format(cachedir))
    serial_file = '{0}/{1}.serial'.format(cachedir, ca_name)
    if not os.path.exists(cachedir):
        os.makedirs(cachedir)
    if not os.path.exists(serial_file):
        fd = salt.utils.fopen(serial_file, 'w')
    else:
        fd = salt.utils.fopen(serial_file, 'a+')
    with fd as ofile:
        ofile.write(str(hashnum))

    return hashnum

def _four_digit_year_to_two_digit(datetimeObj):
    return datetimeObj.strftime(two_digit_year_fmt)

def _get_basic_info(ca_name, cert, ca_dir=None):
    '''
    Get basic info to write out to the index.txt
    '''
    if ca_dir is None:
        ca_dir = '{0}/{1}'.format(_cert_base_path(), ca_name)

    index_file = "{0}/index.txt".format(ca_dir)

    expire_date = _four_digit_year_to_two_digit(
        datetime.strptime(
            cert.get_notAfter(),
            four_digit_year_fmt)
    )
    serial_number = format(cert.get_serial_number(), 'X')

    # gotta prepend a /
    subject = '/'

    # then we can add the rest of the subject
    subject += '/'.join(
        ['{0}={1}'.format(
            x, y
        ) for x, y in cert.get_subject().get_components()]
    )
    subject += '\n'

    return (index_file, expire_date, serial_number, subject)

def _write_cert_to_database(ca_name, cert, cacert_path=None, status='V'):
    '''
    write out the index.txt database file in the appropriate directory to
    track certificates

    ca_name
        name of the CA
    cert
        certificate to be recorded
    '''
    set_ca_path(cacert_path)
    ca_dir = '{0}/{1}'.format(cert_base_path(), ca_name)
    index_file, expire_date, serial_number, subject = _get_basic_info(
        ca_name,
        cert,
        ca_dir)

    index_data = '{0}\t{1}\t\t{2}\tunknown\t{3}'.format(
        status,
        expire_date,
        serial_number,
        subject
    )

    with salt.utils.fopen(index_file, 'a+') as ofile:
        ofile.write(index_data)

def maybe_fix_ssl_version(ca_name, cacert_path=None, ca_filename=None):
    '''
    Check that the X509 version is correct
    (was incorrectly set in previous salt versions).
    This will fix the version if needed.

    ca_name
        ca authority name
    cacert_path
        absolute path to ca certificates root directory
    ca_filename
        alternative filename for the CA

        .. versionadded:: 2015.5.3

    CLI Example:

    .. code-block:: bash

        salt '*' tls.maybe_fix_ssl_version test_ca /etc/certs
    '''
    set_ca_path(cacert_path)
    if not ca_filename:
        ca_filename = '{0}_ca_cert'.format(ca_name)
    certp = '{0}/{1}/{2}.crt'.format(
            cert_base_path(),
            ca_name,
            ca_filename)
    ca_keyp = '{0}/{1}/{2}.key'.format(
        cert_base_path(),
        ca_name,
        ca_filename)
    with salt.utils.fopen(certp) as fic:
        cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
                                               fic.read())
        if cert.get_version() == 3:
            log.info(
                'Regenerating wrong x509 version '
                'for certificate {0}'.format(certp))
            with salt.utils.fopen(ca_keyp) as fic2:
                try:
                    # try to determine the key bits
                    key = OpenSSL.crypto.load_privatekey(
                        OpenSSL.crypto.FILETYPE_PEM, fic2.read())
                    bits = key.bits()
                except Exception:
                    bits = 2048
                try:
                    days = (datetime.strptime(
                        cert.get_notAfter(),
                        '%Y%m%d%H%M%SZ') - datetime.utcnow()).days
                except (ValueError, TypeError):
                    days = 365
                subj = cert.get_subject()
                create_ca(
                    ca_name,
                    bits=bits,
                    days=days,
                    CN=subj.CN,
                    C=subj.C,
                    ST=subj.ST,
                    L=subj.L,
                    O=subj.O,
                    OU=subj.OU,
                    emailAddress=subj.emailAddress,
                    fixmode=True)

def ca_exists(ca_name, cacert_path=None, ca_filename=None):
    '''
    Verify whether a Certificate Authority (CA) already exists

    ca_name
        name of the CA
    cacert_path
        absolute path to ca certificates root directory
    ca_filename
        alternative filename for the CA

        .. versionadded:: 2015.5.3

    CLI Example:

    .. code-block:: bash

        salt '*' tls.ca_exists test_ca /etc/certs
    '''
    set_ca_path(cacert_path)
    if not ca_filename:
        ca_filename = '{0}_ca_cert'.format(ca_name)
    certp = '{0}/{1}/{2}.crt'.format(
            cert_base_path(),
            ca_name,
            ca_filename)
    if os.path.exists(certp):
        maybe_fix_ssl_version(ca_name,
                              cacert_path=cacert_path,
                              ca_filename=ca_filename)
        return True
    return False

def _ca_exists(ca_name, cacert_path=None):
    '''Retrocompatible wrapper'''
    return ca_exists(ca_name, cacert_path)

def get_ca(ca_name, as_text=False, cacert_path=None):
    '''
    Get the certificate path or content

    ca_name
        name of the CA
    as_text
        if true, return the certificate content instead of the path
    cacert_path
        absolute path to ca certificates root directory

    CLI Example:

    .. code-block:: bash

        salt '*' tls.get_ca test_ca as_text=False cacert_path=/etc/certs
    '''
    set_ca_path(cacert_path)
    certp = '{0}/{1}/{2}_ca_cert.crt'.format(
            cert_base_path(),
            ca_name,
            ca_name)
    if not os.path.exists(certp):
        raise ValueError('Certificate does not exist for {0}'.format(ca_name))
    else:
        if as_text:
            with salt.utils.fopen(certp) as fic:
                certp = fic.read()
    return certp

def get_ca_signed_cert(ca_name,
                       CN='localhost',
                       as_text=False,
                       cacert_path=None,
                       cert_filename=None):
    '''
    Get the certificate path or content

    ca_name
        name of the CA
    CN
        common name of the certificate
    as_text
        if true, return the certificate content instead of the path
    cacert_path
        absolute path to certificates root directory
    cert_filename
        alternative filename for the certificate, useful when using special characters in the CN

        .. versionadded:: 2015.5.3

    CLI Example:

    .. code-block:: bash

        salt '*' tls.get_ca_signed_cert test_ca CN=localhost as_text=False cacert_path=/etc/certs
    '''
    set_ca_path(cacert_path)
    if not cert_filename:
        cert_filename = CN

    certp = '{0}/{1}/certs/{2}.crt'.format(
            cert_base_path(),
            ca_name,
            cert_filename)
    if not os.path.exists(certp):
        raise ValueError('Certificate does not exists for {0}'.format(CN))
    else:
        if as_text:
            with salt.utils.fopen(certp) as fic:
                certp = fic.read()
    return certp

def get_ca_signed_key(ca_name,
                      CN='localhost',
                      as_text=False,
                      cacert_path=None,
                      key_filename=None):
    '''
    Get the certificate path or content

    ca_name
        name of the CA
    CN
        common name of the certificate
    as_text
        if true, return the certificate content instead of the path
    cacert_path
        absolute path to certificates root directory
    key_filename
        alternative filename for the key, useful when using special characters

        .. versionadded:: 2015.5.3

        in the CN

    CLI Example:

    .. code-block:: bash

        salt '*' tls.get_ca_signed_key \
                test_ca CN=localhost \
                as_text=False \
                cacert_path=/etc/certs
    '''
    set_ca_path(cacert_path)
    if not key_filename:
        key_filename = CN

    keyp = '{0}/{1}/certs/{2}.key'.format(
        cert_base_path(),
        ca_name,
        key_filename)
    if not os.path.exists(keyp):
        raise ValueError('Certificate does not exists for {0}'.format(CN))
    else:
        if as_text:
            with salt.utils.fopen(keyp) as fic:
                keyp = fic.read()
    return keyp

def _check_onlyif_unless(onlyif, unless):
    ret = None
    retcode = __salt__['cmd.retcode']
    if onlyif is not None:
        if not isinstance(onlyif, string_types):
            if not onlyif:
                ret = {'comment': 'onlyif execution failed', 'result': True}
        elif isinstance(onlyif, string_types):
            if retcode(onlyif) != 0:
                ret = {'comment': 'onlyif execution failed', 'result': True}
                log.debug('onlyif execution failed')
    if unless is not None:
        if not isinstance(unless, string_types):
            if unless:
                ret = {'comment': 'unless execution succeeded', 'result': True}
        elif isinstance(unless, string_types):
            if retcode(unless) == 0:
                ret = {'comment': 'unless execution succeeded', 'result': True}
                log.debug('unless execution succeeded')
    return ret

def create_ca(ca_name,
              bits=2048,
              days=365,
              CN='localhost',
              C='US',
              ST='Utah',
              L='Salt Lake City',
              O='SaltStack',
              OU=None,
              emailAddress='xyz@pdq.net',
              fixmode=False,
              cacert_path=None,
              ca_filename=None,
              digest='sha256',
              onlyif=None,
              unless=None,
              replace=False):
    '''
    Create a Certificate Authority (CA)

    ca_name
        name of the CA
    bits
        number of RSA key bits, default is 2048
    days
        number of days the CA will be valid, default is 365
    CN
        common name in the request, default is "localhost"
    C
        country, default is "US"
    ST
        state, default is "Utah"
    L
        locality, default is "Centerville", the city where SaltStack originated
    O
        organization, default is "SaltStack"
    OU
        organizational unit, default is None
    emailAddress
        email address for the CA owner, default is 'xyz@pdq.net'
    cacert_path
        absolute path to ca certificates root directory
    ca_filename
        alternative filename for the CA

        .. versionadded:: 2015.5.3

    digest
        The message digest algorithm. Must be a string describing a digest
        algorithm supported by OpenSSL (by EVP_get_digestbyname, specifically).
        For example, "md5" or "sha1". Default: 'sha256'
    replace
        Replace this certificate even if it exists

        .. versionadded:: 2015.5.1

    Writes out a CA certificate based upon defined config values. If the file
    already exists, the function just returns assuming the CA certificate
    already exists.

    If the following values were set::

        ca.cert_base_path='/etc/pki'
        ca_name='koji'

    the resulting CA, and corresponding key, would be written in the following
    location::

        /etc/pki/koji/koji_ca_cert.crt
        /etc/pki/koji/koji_ca_cert.key

    CLI Example:

    .. code-block:: bash

        salt '*' tls.create_ca test_ca
    '''
    status = _check_onlyif_unless(onlyif, unless)
    if status is not None:
        return None

    set_ca_path(cacert_path)

    if not ca_filename:
        ca_filename = '{0}_ca_cert'.format(ca_name)

    certp = '{0}/{1}/{2}.crt'.format(
        cert_base_path(), ca_name, ca_filename)
    ca_keyp = '{0}/{1}/{2}.key'.format(
        cert_base_path(), ca_name, ca_filename)
    if not replace and not fixmode and ca_exists(ca_name, ca_filename=ca_filename):
        return 'Certificate for CA named "{0}" already exists'.format(ca_name)

    if fixmode and not os.path.exists(certp):
        raise ValueError('{0} does not exists, can\'t fix'.format(certp))

    if not os.path.exists('{0}/{1}'.format(
        cert_base_path(), ca_name)
    ):
        os.makedirs('{0}/{1}'.format(cert_base_path(),
                                     ca_name))

    # try to reuse existing ssl key
    key = None
    if os.path.exists(ca_keyp):
        with salt.utils.fopen(ca_keyp) as fic2:
            # try to determine the key bits
            key = OpenSSL.crypto.load_privatekey(
                OpenSSL.crypto.FILETYPE_PEM, fic2.read())
    if not key:
        key = OpenSSL.crypto.PKey()
        key.generate_key(OpenSSL.crypto.TYPE_RSA, bits)

    ca = OpenSSL.crypto.X509()
    ca.set_version(2)
    ca.set_serial_number(_new_serial(ca_name, CN))
    ca.get_subject().C = C
    ca.get_subject().ST = ST
    ca.get_subject().L = L
    ca.get_subject().O = O
    if OU:
        ca.get_subject().OU = OU
    ca.get_subject().CN = CN
    ca.get_subject().emailAddress = emailAddress

    ca.gmtime_adj_notBefore(0)
    ca.gmtime_adj_notAfter(int(days) * 24 * 60 * 60)
    ca.set_issuer(ca.get_subject())
    ca.set_pubkey(key)

    if X509_EXT_ENABLED:
        ca.add_extensions([
            OpenSSL.crypto.X509Extension('basicConstraints', True,
                                         'CA:TRUE, pathlen:0'),
            OpenSSL.crypto.X509Extension('keyUsage', True,
                                         'keyCertSign, cRLSign'),
            OpenSSL.crypto.X509Extension('subjectKeyIdentifier', False,
                                         'hash', subject=ca)])

        ca.add_extensions([
            OpenSSL.crypto.X509Extension(
                'authorityKeyIdentifier',
                False,
                'issuer:always,keyid:always',
                issuer=ca)])
    ca.sign(key, digest)

    # alway backup existing keys in case
    keycontent = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM,
                                                key)
    write_key = True
    if os.path.exists(ca_keyp):
        bck = "{0}.{1}".format(ca_keyp, datetime.utcnow().strftime(
            "%Y%m%d%H%M%S"))
        with salt.utils.fopen(ca_keyp) as fic:
            old_key = fic.read().strip()
            if old_key.strip() == keycontent.strip():
                write_key = False
            else:
                log.info('Saving old CA ssl key in {0}'.format(bck))
                with salt.utils.fopen(bck, 'w') as bckf:
                    bckf.write(old_key)
                    os.chmod(bck, 0o600)
    if write_key:
        with salt.utils.fopen(ca_keyp, 'w') as ca_key:
            ca_key.write(keycontent)

    with salt.utils.fopen(certp, 'w') as ca_crt:
        ca_crt.write(
            OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, ca))

    _write_cert_to_database(ca_name, ca)

    ret = ('Created Private Key: "{0}/{1}/{2}.key." ').format(
        cert_base_path(), ca_name, ca_filename)
    ret += ('Created CA "{0}": "{1}/{2}/{3}.crt."').format(
        ca_name, cert_base_path(), ca_name, ca_filename)

    return ret

def get_extensions(cert_type):
    '''
    Fetch X509 and CSR extension definitions from tls:extensions:
    (common|server|client) or set them to standard defaults.

    .. versionadded:: Beryllium

    cert_type:
        The type of certificate such as ``server`` or ``client``.

    CLI Example:

    .. code-block:: bash

        salt '*' tls.get_extensions client

    '''

    assert X509_EXT_ENABLED, ('X509 extensions are not supported in '
                              'pyOpenSSL prior to version 0.15.1. Your '
                              'version: {0}'.format(OpenSSL_version))

    ext = {}
    if cert_type == '':
        log.error('cert_type set to empty in tls_ca.get_extensions(); '
                  'defaulting to ``server``')
        cert_type = 'server'

    try:
        ext['common'] = __salt__['pillar.get']('tls.extensions:common', False)
    except NameError as err:
        log.debug(err)

    if not ext['common'] or ext['common'] == '':
        ext['common'] = {
            'csr': {
                'basicConstraints': 'CA:FALSE',
            },
            'cert': {
                'authorityKeyIdentifier': 'keyid,issuer:always',
                'subjectKeyIdentifier': 'hash',
            },
        }

    try:
        ext['server'] = __salt__['pillar.get']('tls.extensions:server', False)
    except NameError as err:
        log.debug(err)

    if not ext['server'] or ext['server'] == '':
        ext['server'] = {
            'csr': {
                'extendedKeyUsage': 'serverAuth',
                'keyUsage': 'digitalSignature, keyEncipherment',
            },
            'cert': {},
        }

    try:
        ext['client'] = __salt__['pillar.get']('tls.extensions:client', False)
    except NameError as err:
        log.debug(err)

    if not ext['client'] or ext['client'] == '':
        ext['client'] = {
            'csr': {
                'extendedKeyUsage': 'clientAuth',
                'keyUsage': 'nonRepudiation, digitalSignature, keyEncipherment',
            },
            'cert': {},
        }

    # possible user-defined profile or a typo
    if cert_type not in ext:
        try:
            ext[cert_type] = __salt__['pillar.get'](
                'tls.extensions:{0}'.format(cert_type))
        except NameError as e:
            log.debug(
                'pillar, tls:extensions:{0} not available or '
                'not operating in a salt context\n{1}'.format(cert_type, e))

    retval = ext['common']

    for Use in retval:
        retval[Use].update(ext[cert_type][Use])

    return retval

def create_csr(ca_name,
               bits=2048,
               CN='localhost',
               C='US',
               ST='Utah',
               L='Salt Lake City',
               O='SaltStack',
               OU=None,
               emailAddress='xyz@pdq.net',
               subjectAltName=None,
               cacert_path=None,
               ca_filename=None,
               csr_path=None,
               csr_filename=None,
               digest='sha256',
               type_ext=False,
               cert_type='server',
               replace=False):
    '''
    Create a Certificate Signing Request (CSR) for a
    particular Certificate Authority (CA)

    ca_name
        name of the CA
    bits
        number of RSA key bits, default is 2048
    CN
        common name in the request, default is "localhost"
    C
        country, default is "US"
    ST
        state, default is "Utah"
    L
        locality, default is "Centerville", the city where SaltStack originated
    O
        organization, default is "SaltStack"
        NOTE: Must the same as CA certificate or an error will be raised
    OU
        organizational unit, default is None
    emailAddress
        email address for the request, default is 'xyz@pdq.net'
    subjectAltName
        valid subjectAltNames in full form, e.g. to add DNS entry you would call
        this function with this value:

        examples: ['DNS:somednsname.com',
                'DNS:1.2.3.4',
                'IP:1.2.3.4',
                'IP:2001:4801:7821:77:be76:4eff:fe11:e51',
                'email:me@i.like.pie.com']

    .. note::
        some libraries do not properly query IP: prefixes, instead looking
        for the given req. source with a DNS: prefix. To be thorough, you
        may want to include both DNS: and IP: entries if you are using
        subjectAltNames for destinations for your TLS connections.
            e.g.:
                requests to https://1.2.3.4 will fail from python's
                requests library w/out the second entry in the above list

    .. versionadded:: Beryllium

    cert_type
        Specify the general certificate type. Can be either `server` or
        `client`. Indicates the set of common extensions added to the CSR.

        server: {
           'basicConstraints': 'CA:FALSE',
           'extendedKeyUsage': 'serverAuth',
           'keyUsage': 'digitalSignature, keyEncipherment'
        }

        client: {
           'basicConstraints': 'CA:FALSE',
           'extendedKeyUsage': 'clientAuth',
           'keyUsage': 'nonRepudiation, digitalSignature, keyEncipherment'
        }

    type_ext
        boolean.  Whether or not to extend the filename with CN_[cert_type]
        This can be useful if a server and client certificate are needed for
        the same CN. Defaults to False to avoid introducing an unexpected file
        naming pattern

        The files normally named some_subject_CN.csr and some_subject_CN.key
        will then be saved

    replace
        Replace this signing request even if it exists

        .. versionadded:: 2015.5.1

    Writes out a Certificate Signing Request (CSR) If the file already
    exists, the function just returns assuming the CSR already exists.

    If the following values were set::

        ca.cert_base_path='/etc/pki'
        ca_name='koji'
        CN='test.egavas.org'

    the resulting CSR, and corresponding key, would be written in the
    following location::

        /etc/pki/koji/certs/test.egavas.org.csr
        /etc/pki/koji/certs/test.egavas.org.key

    CLI Example:

    .. code-block:: bash

        salt '*' tls.create_csr test
    '''
    set_ca_path(cacert_path)

    if not ca_filename:
        ca_filename = '{0}_ca_cert'.format(ca_name)

    if not ca_exists(ca_name, ca_filename=ca_filename):
        return ('Certificate for CA named "{0}" does not exist, please create '
                'it first.').format(ca_name)

    if not csr_path:
        csr_path = '{0}/{1}/certs/'.format(cert_base_path(), ca_name)

    if not os.path.exists(csr_path):
        os.makedirs(csr_path)

    CN_ext = '_{0}'.format(cert_type) if type_ext else ''

    if not csr_filename:
        csr_filename = '{0}{1}'.format(CN, CN_ext)

    csr_f = '{0}/{1}.csr'.format(csr_path, csr_filename)

    if not replace and os.path.exists(csr_f):
        return 'Certificate Request "{0}" already exists'.format(csr_f)

    key = OpenSSL.crypto.PKey()
    key.generate_key(OpenSSL.crypto.TYPE_RSA, bits)

    req = OpenSSL.crypto.X509Req()

    req.get_subject().C = C
    req.get_subject().ST = ST
    req.get_subject().L = L
    req.get_subject().O = O
    if OU:
        req.get_subject().OU = OU
    req.get_subject().CN = CN
    req.get_subject().emailAddress = emailAddress

    try:
        extensions = get_extensions(cert_type)['csr']

        extension_adds = []

        for ext, value in extensions.items():
            extension_adds.append(OpenSSL.crypto.X509Extension(ext, False,
                                                               value))

    except AssertionError as err:
        log.error(err)
        extensions = []

    if subjectAltName:
        if X509_EXT_ENABLED:
            if isinstance(subjectAltName, str):
                subjectAltName = [subjectAltName]

            extension_adds.append(
                OpenSSL.crypto.X509Extension(
                    'subjectAltName', False, ", ".join(subjectAltName)))
        else:
            raise ValueError('subjectAltName cannot be set as X509 '
                             'extensions are not supported in pyOpenSSL '
                             'prior to version 0.15.1. Your '
                             'version: {0}.'.format(OpenSSL_version))

    if X509_EXT_ENABLED:
        req.add_extensions(extension_adds)

    req.set_pubkey(key)
    req.sign(key, digest)

    # Write private key and request
    with salt.utils.fopen('{0}/{1}.key'.format(csr_path,
                                               csr_filename), 'w+') as priv_key:
        priv_key.write(
            OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)
        )

    with salt.utils.fopen(csr_f, 'w+') as csr:
        csr.write(
            OpenSSL.crypto.dump_certificate_request(
                OpenSSL.crypto.FILETYPE_PEM,
                req
            )
        )

    ret = 'Created Private Key: "{0}{1}.key." '.format(
        csr_path,
        csr_filename
    )
    ret += 'Created CSR for "{0}": "{1}{2}.csr."'.format(
        CN,
        csr_path,
        csr_filename
    )

    return ret

def create_self_signed_cert(tls_dir='tls',
                            bits=2048,
                            days=365,
                            CN='localhost',
                            C='US',
                            ST='Utah',
                            L='Salt Lake City',
                            O='SaltStack',
                            OU=None,
                            emailAddress='xyz@pdq.net',
                            cacert_path=None,
                            cert_filename=None,
                            digest='sha256',
                            replace=False):
    '''
    Create a Self-Signed Certificate (CERT)

    tls_dir
        location appended to the ca.cert_base_path, default is 'tls'
    bits
        number of RSA key bits, default is 2048
    CN
        common name in the request, default is "localhost"
    C
        country, default is "US"
    ST
        state, default is "Utah"
    L
        locality, default is "Centerville", the city where SaltStack originated
    O
        organization, default is "SaltStack"
        NOTE: Must the same as CA certificate or an error will be raised
    OU
        organizational unit, default is None
    emailAddress
        email address for the request, default is 'xyz@pdq.net'
    cacert_path
        absolute path to ca certificates root directory
    digest
        The message digest algorithm. Must be a string describing a digest
        algorithm supported by OpenSSL (by EVP_get_digestbyname, specifically).
        For example, "md5" or "sha1". Default: 'sha256'
    replace
        Replace this certificate even if it exists

        .. versionadded:: 2015.5.1

    Writes out a Self-Signed Certificate (CERT). If the file already
    exists, the function just returns.

    If the following values were set::

        ca.cert_base_path='/etc/pki'
        tls_dir='koji'
        CN='test.egavas.org'

    the resulting CERT, and corresponding key, would be written in the
    following location::

        /etc/pki/koji/certs/test.egavas.org.crt
        /etc/pki/koji/certs/test.egavas.org.key

    CLI Example:

    .. code-block:: bash

        salt '*' tls.create_self_signed_cert

    Passing options from the command line:

    .. code-block:: bash

        salt 'minion' tls.create_self_signed_cert CN='test.mysite.org'
    '''
    set_ca_path(cacert_path)

    if not os.path.exists('{0}/{1}/certs/'.format(cert_base_path(), tls_dir)):
        os.makedirs("{0}/{1}/certs/".format(cert_base_path(),
                                            tls_dir))

    if not cert_filename:
        cert_filename = CN

    if not replace and os.path.exists(
            '{0}/{1}/certs/{2}.crt'.format(cert_base_path(),
                                           tls_dir, cert_filename)
    ):
        return 'Certificate "{0}" already exists'.format(cert_filename)

    key = OpenSSL.crypto.PKey()
    key.generate_key(OpenSSL.crypto.TYPE_RSA, bits)

    # create certificate
    cert = OpenSSL.crypto.X509()
    cert.set_version(2)

    cert.gmtime_adj_notBefore(0)
    cert.gmtime_adj_notAfter(int(days) * 24 * 60 * 60)

    cert.get_subject().C = C
    cert.get_subject().ST = ST
    cert.get_subject().L = L
    cert.get_subject().O = O
    if OU:
        cert.get_subject().OU = OU
    cert.get_subject().CN = CN
    cert.get_subject().emailAddress = emailAddress

    cert.set_serial_number(_new_serial(tls_dir, CN))
    cert.set_issuer(cert.get_subject())
    cert.set_pubkey(key)
    cert.sign(key, digest)

    # Write private key and cert
    with salt.utils.fopen(
        '{0}/{1}/certs/{2}.key'.format(cert_base_path(),
                                       tls_dir, cert_filename),
        'w+'
    ) as priv_key:
        priv_key.write(
            OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)
        )

    with salt.utils.fopen('{0}/{1}/certs/{2}.crt'.format(cert_base_path(),
                                                         tls_dir,
                                                         cert_filename
                                                         ), 'w+') as crt:
        crt.write(
            OpenSSL.crypto.dump_certificate(
                OpenSSL.crypto.FILETYPE_PEM,
                cert
            )
        )

    _write_cert_to_database(tls_dir, cert)

    ret = 'Created Private Key: "{0}/{1}/certs/{2}.key." '.format(
        cert_base_path(),
        tls_dir,
        cert_filename
    )
    ret += 'Created Certificate: "{0}/{1}/certs/{2}.crt."'.format(
        cert_base_path(),
        tls_dir,
        cert_filename
    )

    return ret

def create_ca_signed_cert(ca_name,
                          CN,
                          days=365,
                          cacert_path=None,
                          ca_filename=None,
                          cert_path=None,
                          cert_filename=None,
                          digest='sha256',
                          cert_type=None,
                          type_ext=False,
                          replace=False):
    '''
    Create a Certificate (CERT) signed by a named Certificate Authority (CA)

    If the certificate file already exists, the function just returns assuming
    the CERT already exists.

    The CN *must* match an existing CSR generated by create_csr. If it
    does not, this method does nothing.

    ca_name
        name of the CA
    CN
        common name matching the certificate signing request
    days
        number of days certificate is valid, default is 365 (1 year)

    cacert_path
        absolute path to ca certificates root directory

    ca_filename
        alternative filename for the CA

        .. versionadded:: 2015.5.3

    cert_path
        full path to the certificates directory

    cert_filename
        alternative filename for the certificate, useful when using special
        characters in the CN. If this option is set it will override
        the certificate filename output effects of ``cert_type``.
        ``type_ext`` will be completely overridden.

        .. versionadded:: 2015.5.3

    digest
        The message digest algorithm. Must be a string describing a digest
        algorithm supported by OpenSSL (by EVP_get_digestbyname, specifically).
        For example, "md5" or "sha1". Default: 'sha256'
    replace
        Replace this certificate even if it exists

        .. versionadded:: 2015.5.1

    cert_type
        string. Either 'server' or 'client' (see create_csr() for details).

        If create_csr(type_ext=True) this function **must** be called with the
        same cert_type so it can find the CSR file.

    .. note::
        create_csr() defaults to cert_type='server'; therefore, if it was also
        called with type_ext, cert_type becomes a required argument for
        create_ca_signed_cert()

    type_ext
        bool. If set True, use ``cert_type`` as an extension to the CN when
        formatting the filename.

        e.g.: some_subject_CN_server.crt or some_subject_CN_client.crt

        This facilitates the context where both types are required for the same
        subject

        If ``cert_filename`` is `not None`, setting ``type_ext`` has no
        effect

    If the following values were set:

    .. code-block:: text

        ca.cert_base_path='/etc/pki'
        ca_name='koji'
        CN='test.egavas.org'

    the resulting signed certificate would be written in the following
    location:

    .. code-block:: text

        /etc/pki/koji/certs/test.egavas.org.crt

    CLI Example:

    .. code-block:: bash

        salt '*' tls.create_ca_signed_cert test localhost
    '''
    ret = {}

    set_ca_path(cacert_path)

    if not ca_filename:
        ca_filename = '{0}_ca_cert'.format(ca_name)

    if not cert_path:
        cert_path = '{0}/{1}/certs'.format(cert_base_path(), ca_name)

    if type_ext:
        if not cert_type:
            log.error('type_ext = True but cert_type is unset. '
                      'Certificate not written.')
            return ret
        elif cert_type:
            CN_ext = '_{0}'.format(cert_type)
    else:
        CN_ext = ''

    csr_filename = '{0}{1}'.format(CN, CN_ext)

    if not cert_filename:
        cert_filename = '{0}{1}'.format(CN, CN_ext)

    if not replace and os.path.exists(
            os.path.join(
                os.path.sep.join('{0}/{1}/certs/{2}.crt'.format(
                    cert_base_path(),
                    ca_name,
                    cert_filename).split('/')
                )
            )
    ):
        return 'Certificate "{0}" already exists'.format(cert_filename)

    try:
        maybe_fix_ssl_version(ca_name,
                              cacert_path=cacert_path,
                              ca_filename=ca_filename)
        with salt.utils.fopen('{0}/{1}/{2}.crt'.format(cert_base_path(),
                                                       ca_name,
                                                       ca_filename)) as fhr:
            ca_cert = OpenSSL.crypto.load_certificate(
                OpenSSL.crypto.FILETYPE_PEM, fhr.read()
            )
        with salt.utils.fopen('{0}/{1}/{2}.key'.format(cert_base_path(),
                                                       ca_name,
                                                       ca_filename)) as fhr:
            ca_key = OpenSSL.crypto.load_privatekey(
                OpenSSL.crypto.FILETYPE_PEM,
                fhr.read()
            )
    except IOError:
        ret['retcode'] = 1
        ret['comment'] = 'There is no CA named "{0}"'.format(ca_name)
        return ret

    try:
        csr_path = '{0}/{1}.csr'.format(cert_path, csr_filename)
        with salt.utils.fopen(csr_path) as fhr:
            req = OpenSSL.crypto.load_certificate_request(
                OpenSSL.crypto.FILETYPE_PEM,
                fhr.read())
    except IOError:
        ret['retcode'] = 1
        ret['comment'] = 'There is no CSR that matches the CN "{0}"'.format(
            cert_filename)
        return ret

    exts = []
    try:
        exts.extend(req.get_extensions())
    except AttributeError:
        try:
            # see: http://bazaar.launchpad.net/~exarkun/pyopenssl/master/revision/189
            # support is there from quite a long time, but without API
            # so we mimic the newly get_extensions method present in ultra
            # recent pyopenssl distros
            log.info('req.get_extensions() not supported in pyOpenSSL versions '
                     'prior to 0.15. Processing extensions internally. '
                     ' Your version: {0}'.format(
                         OpenSSL_version))

            native_exts_obj = OpenSSL._util.lib.X509_REQ_get_extensions(
                req._req)
            for i in _range(OpenSSL._util.lib.sk_X509_EXTENSION_num(
                    native_exts_obj)):
                ext = OpenSSL.crypto.X509Extension.__new__(
                    OpenSSL.crypto.X509Extension)
                ext._extension = OpenSSL._util.lib.sk_X509_EXTENSION_value(
                    native_exts_obj,
                    i)
                exts.append(ext)
        except Exception:
            log.error('X509 extensions are unsupported in pyOpenSSL '
                      'versions prior to 0.14. Upgrade required to '
                      'use extensions. Current version: {0}'.format(
                          OpenSSL_version))

    cert = OpenSSL.crypto.X509()
    cert.set_version(2)
    cert.set_subject(req.get_subject())
    cert.gmtime_adj_notBefore(0)
    cert.gmtime_adj_notAfter(int(days) * 24 * 60 * 60)
    cert.set_serial_number(_new_serial(ca_name, CN))
    cert.set_issuer(ca_cert.get_subject())
    cert.set_pubkey(req.get_pubkey())

    cert.add_extensions(exts)

    cert.sign(ca_key, digest)

    cert_full_path = '{0}/{1}.crt'.format(cert_path, cert_filename)

    with salt.utils.fopen(cert_full_path, 'w+') as crt:
        crt.write(
            OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert))

    _write_cert_to_database(ca_name, cert)

    return ('Created Certificate for "{0}": '
            '"{1}/{2}.crt"').format(
        CN,
        cert_path,
        cert_filename
    )

def create_pkcs12(ca_name, CN, passphrase='', cacert_path=None, replace=False):
    '''
    Create a PKCS#12 browser certificate for a particular Certificate (CN)

    ca_name
        name of the CA
    CN
        common name matching the certificate signing request
    passphrase
        used to unlock the PKCS#12 certificate when loaded into the browser
    cacert_path
        absolute path to ca certificates root directory
    replace
        Replace this certificate even if it exists

        .. versionadded:: 2015.5.1

    If the following values were set::

        ca.cert_base_path='/etc/pki'
        ca_name='koji'
        CN='test.egavas.org'

    the resulting signed certificate would be written in the
    following location::

        /etc/pki/koji/certs/test.egavas.org.p12

    CLI Example:

    .. code-block:: bash

        salt '*' tls.create_pkcs12 test localhost
    '''
    set_ca_path(cacert_path)
    if not replace and os.path.exists(
            '{0}/{1}/certs/{2}.p12'.format(
                cert_base_path(),
                ca_name,
                CN)
    ):
        return 'Certificate "{0}" already exists'.format(CN)

    try:
        with salt.utils.fopen('{0}/{1}/{2}_ca_cert.crt'.format(cert_base_path(),
                                                               ca_name,
                                                               ca_name)) as fhr:
            ca_cert = OpenSSL.crypto.load_certificate(
                OpenSSL.crypto.FILETYPE_PEM,
                fhr.read()
            )
    except IOError:
        return 'There is no CA named "{0}"'.format(ca_name)

    try:
        with salt.utils.fopen('{0}/{1}/certs/{2}.crt'.format(cert_base_path(),
                                                             ca_name,
                                                             CN)) as fhr:
            cert = OpenSSL.crypto.load_certificate(
                OpenSSL.crypto.FILETYPE_PEM,
                fhr.read()
            )
        with salt.utils.fopen('{0}/{1}/certs/{2}.key'.format(cert_base_path(),
                                                             ca_name,
                                                             CN)) as fhr:
            key = OpenSSL.crypto.load_privatekey(
                OpenSSL.crypto.FILETYPE_PEM,
                fhr.read()
            )
    except IOError:
        return 'There is no certificate that matches the CN "{0}"'.format(CN)

    pkcs12 = OpenSSL.crypto.PKCS12()

    pkcs12.set_certificate(cert)
    pkcs12.set_ca_certificates([ca_cert])
    pkcs12.set_privatekey(key)

    with salt.utils.fopen('{0}/{1}/certs/{2}.p12'.format(cert_base_path(),
                                                         ca_name,
                                                         CN), 'w') as ofile:
        ofile.write(pkcs12.export(passphrase=passphrase))

    return ('Created PKCS#12 Certificate for "{0}": '
            '"{1}/{2}/certs/{3}.p12"').format(
        CN,
        cert_base_path(),
        ca_name,
        CN
    )

def cert_info(cert_path, digest='sha256'):
    '''
    Return information for a particular certificate

    cert_path
        path to the cert file
    digest
        what digest to use for fingerprinting

    CLI Example:

    .. code-block:: bash

        salt '*' tls.cert_info /dir/for/certs/cert.pem
    '''
    # format that OpenSSL returns dates in
    date_fmt = '%Y%m%d%H%M%SZ'

    with salt.utils.fopen(cert_path) as cert_file:
        cert = OpenSSL.crypto.load_certificate(
            OpenSSL.crypto.FILETYPE_PEM,
            cert_file.read()
        )
    ret = {
        'fingerprint': cert.digest(digest),
        'subject': dict(cert.get_subject().get_components()),
        'issuer': dict(cert.get_issuer().get_components()),
        'serial_number': cert.get_serial_number(),
        'not_before': calendar.timegm(time.strptime(
            cert.get_notBefore(),
            date_fmt)),
        'not_after': calendar.timegm(time.strptime(
            cert.get_notAfter(),
            date_fmt)),
    }

    # add additional info if your version of pyOpenSSL supports it
    if hasattr(cert, 'get_extension_count'):
        ret['extensions'] = {}
        for i in _range(cert.get_extension_count()):
            ext = cert.get_extension(i)
            ret['extensions'][ext.get_short_name()] = ext

    if 'subjectAltName' in ret.get('extensions', {}):
        valid_names = set()
        for name in ret['extensions']['subjectAltName'] \
                ._subjectAltNameString().split(", "):
            if not name.startswith('DNS:'):
                log.error('Cert {0} has an entry ({1}) which does not start '
                          'with DNS:'.format(cert_path, name))
            else:
                valid_names.add(name[4:])
        ret['subject_alt_names'] = valid_names

    if hasattr(cert, 'get_signature_algorithm'):
        ret['signature_algorithm'] = cert.get_signature_algorithm()

    return ret

def create_empty_crl(
        ca_name,
        cacert_path=None,
        ca_filename=None,
        crl_file=None):
    '''
    Create an empty Certificate Revocation List.

    .. versionadded:: Beryllium

    ca_name
        name of the CA
    cacert_path
        absolute path to ca certificates root directory
    ca_filename
        alternative filename for the CA

        .. versionadded:: 2015.5.3

    crl_file
        full path to the CRL file

    CLI Example:

    .. code-block:: bash

        salt '*' tls.create_empty_crl ca_name='koji' \
                ca_filename='ca' \
                crl_file='/etc/openvpn/team1/crl.pem'
    '''

    set_ca_path(cacert_path)

    if not ca_filename:
        ca_filename = '{0}_ca_cert'.format(ca_name)

    if not crl_file:
        crl_file = '{0}/{1}/crl.pem'.format(
            _cert_base_path(),
            ca_name
        )

    if os.path.exists('{0}'.format(crl_file)):
        return 'CRL "{0}" already exists'.format(crl_file)

    try:
        ca_cert = OpenSSL.crypto.load_certificate(
            OpenSSL.crypto.FILETYPE_PEM,
            salt.utils.fopen('{0}/{1}/{2}.crt'.format(
                cert_base_path(),
                ca_name,
                ca_filename
            )).read()
        )
        ca_key = OpenSSL.crypto.load_privatekey(
            OpenSSL.crypto.FILETYPE_PEM,
            salt.utils.fopen('{0}/{1}/{2}.key'.format(
                cert_base_path(),
                ca_name,
                ca_filename)).read()
        )
    except IOError:
        return 'There is no CA named "{0}"'.format(ca_name)

    crl = OpenSSL.crypto.CRL()
    crl_text = crl.export(ca_cert, ca_key)

    with salt.utils.fopen(crl_file, 'w') as f:
        f.write(crl_text)

    return 'Created an empty CRL: "{0}"'.format(crl_file)

def revoke_cert(
        ca_name,
        CN,
        cacert_path=None,
        ca_filename=None,
        cert_path=None,
        cert_filename=None,
        crl_file=None):
    '''
    Revoke a certificate.

    .. versionadded:: Beryllium

    ca_name
        Name of the CA.

    CN
        Common name matching the certificate signing request.

    cacert_path
        Absolute path to ca certificates root directory.

    ca_filename
        Alternative filename for the CA.

    cert_path
        Path to the cert file.

    cert_filename
        Alternative filename for the certificate, useful when using special
        characters in the CN.

    crl_file
        Full path to the CRL file.

    CLI Example:

    .. code-block:: bash

        salt '*' tls.revoke_cert ca_name='koji' \
                ca_filename='ca' \
                crl_file='/etc/openvpn/team1/crl.pem'

    '''

    set_ca_path(cacert_path)
    ca_dir = '{0}/{1}'.format(cert_base_path(), ca_name)

    if ca_filename is None:
        ca_filename = '{0}_ca_cert'.format(ca_name)

    if cert_path is None:
        cert_path = '{0}/{1}/certs'.format(_cert_base_path(), ca_name)

    if cert_filename is None:
        cert_filename = '{0}'.format(CN)

    try:
        ca_cert = OpenSSL.crypto.load_certificate(
            OpenSSL.crypto.FILETYPE_PEM,
            salt.utils.fopen('{0}/{1}/{2}.crt'.format(
                cert_base_path(),
                ca_name,
                ca_filename
            )).read()
        )
        ca_key = OpenSSL.crypto.load_privatekey(
            OpenSSL.crypto.FILETYPE_PEM,
            salt.utils.fopen('{0}/{1}/{2}.key'.format(
                cert_base_path(),
                ca_name,
                ca_filename)).read()
        )
    except IOError:
        return 'There is no CA named "{0}"'.format(ca_name)

    try:
        client_cert = OpenSSL.crypto.load_certificate(
            OpenSSL.crypto.FILETYPE_PEM,
            salt.utils.fopen('{0}/{1}.crt'.format(
                cert_path,
                cert_filename)).read()
        )
    except IOError:
        return 'There is no client certificate named "{0}"'.format(CN)

    index_file, expire_date, serial_number, subject = _get_basic_info(
        ca_name,
        client_cert,
        ca_dir)

    index_serial_subject = '{0}\tunknown\t{1}'.format(
        serial_number,
        subject)
    index_v_data = 'V\t{0}\t\t{1}'.format(
        expire_date,
        index_serial_subject)
    index_r_data_pattern = re.compile(
        r"R\t" +
        expire_date +
        r"\t\d{12}Z\t" +
        re.escape(index_serial_subject))
    index_r_data = 'R\t{0}\t{1}\t{2}'.format(
        expire_date,
        _four_digit_year_to_two_digit(datetime.utcnow()),
        index_serial_subject)

    ret = {}
    with salt.utils.fopen(index_file) as f:
        for line in f:
            if index_r_data_pattern.match(line):
                revoke_date = line.split('\t')[2]
                try:
                    datetime.strptime(revoke_date, two_digit_year_fmt)
                    return ('"{0}/{1}.crt" was already revoked, '
                            'serial number: {2}').format(
                        cert_path,
                        cert_filename,
                        serial_number
                    )
                except ValueError:
                    ret['retcode'] = 1
                    ret['comment'] = ("Revocation date '{0}' does not match"
                                      "format '{1}'".format(
                                          revoke_date,
                                          two_digit_year_fmt))
                    return ret
            elif index_serial_subject in line:
                __salt__['file.replace'](
                    index_file,
                    index_v_data,
                    index_r_data,
                    backup=False)
                break

    crl = OpenSSL.crypto.CRL()

    with salt.utils.fopen(index_file) as f:
        for line in f:
            if line.startswith('R'):
                fields = line.split('\t')
                revoked = OpenSSL.crypto.Revoked()
                revoked.set_serial(fields[3])
                revoke_date_2_digit = datetime.strptime(fields[2],
                                                        two_digit_year_fmt)
                revoked.set_rev_date(revoke_date_2_digit.strftime(
                    four_digit_year_fmt))
                crl.add_revoked(revoked)

    crl_text = crl.export(ca_cert, ca_key)

    if crl_file is None:
        crl_file = '{0}/{1}/crl.pem'.format(
            _cert_base_path(),
            ca_name
        )

    if os.path.isdir(crl_file):
        ret['retcode'] = 1
        ret['comment'] = 'crl_file "{0}" is an existing directory'.format(
            crl_file)
        return ret

    with salt.utils.fopen(crl_file, 'w') as f:
        f.write(crl_text)

    return ('Revoked Certificate: "{0}/{1}.crt", '
            'serial number: {2}').format(
        cert_path,
        cert_filename,
        serial_number
    )

if __name__ == '__main__':
    # create_ca('koji', days=365, **cert_sample_meta)
    create_csr(
        'koji',
        CN='test_system',
        C="US",
        ST="Utah",
        L="Centerville",
        O="SaltStack",
        OU=None,
        emailAddress='test_system@saltstack.org'
    )
    create_ca_signed_cert('koji', 'test_system')
    create_pkcs12('koji', 'test_system', passphrase='test')


import sys
import glob
import os
import math
import matplotlib.pyplot as plt

def get_number( file_name):
    TEMPLATE = "gal_"
    file_name = file_name.split("/")[-1]
    number_index = file_name.find(TEMPLATE) + len(TEMPLATE)
    if number_index == len(TEMPLATE) -1:
        return file_name.split(".")[0]
    number = 0
    while file_name[number_index].isdigit():
        number*=10
        number+= int(file_name[number_index])
        number_index +=1
    return str(number)

if len(sys.argv) != 6:
    sys.error("WRONG NUMBER OF ARGUMENTS")

results_folder_name = sys.argv[1]

if results_folder_name[-1] != "/":
    results_folder_name += "/"

fout_file_name = sys.argv[2]
sim_masses_name = sys.argv[3]
data_folder = sys.argv[4]
if data_folder[-1] != "/":
    data_folder += "/"

output_file = sys.argv[5]

data_extracted = { 'num': [], 'red': [], 'ms': [], 'mf': []}

masses_fast = {}

fout_file = open(fout_file_name, "r")
for line in fout_file:
    if line[0] == '#':
        continue
    prevalues = line.split(" ")
    values = []
    for i in range(len(prevalues)):
        if not prevalues[i] == '':
            values.append(float(prevalues[i]))
    masses_fast[ str(int(values[0]))] = float(values[4])
fout_file.close()

print(masses_fast)

masses_sim = {}

sim_mass_file = open(sim_mass_name, "r")

for line in sim_mass_file:
    values = line.split(" ")
    masses_sim[values[0]] = float(values[1])

sim_mass_file.close()

print(masses_sim)

results_files = glob.glob(results_folder_name +"gal_*.out")

for result in results_files:
    print(result)
    number = get_number(result)
    data_extracted['num'].append(number)
    redshift_file = open(data_folder + "gal_"+str(number)+".txt", "r")
    for line in redshift_file:
        data_extracted['red'].append(float(line.split(" ")[0]))
        break

    data_extracted['mf'].append(masses_fast[number])
    result_file = open(result, "r")

    sim_num = -1
    for line in result_file:
        sim_num =str(int(line.split(" ")[0]))
        break

    data_extracted['ms'].append(masses_sim[sim_num])

print("%d %d %d %d" %( len(data_extracted['num']), len(data_extracted['red']), len(data_extracted['ms']), len(data_extracted['mf'])))

output = open(output_file, "w")

for i in range(len(data_extracted['num'])):
    output.write("%d %.6e %.6e %.6e\n" % (int(data_extracted['num'][i]), data_extracted['red'][i], data_extracted['ms'][i], data_extracted['mf'][i]))

output.close()

###################################################################
#
#           LET'S PLAY WITH THE DATA
#
###################################################################


import csv
import matplotlib.pyplot as plt

'''
Get a list of keys from dictionary which has value that matches with any value in given list of values
'''
def getKeysByValues(dictOfElements, value):
    listOfKeys = list()
    listOfItems = dictOfElements.items()
    for item  in listOfItems:
        if item[1] == value:
            listOfKeys.append(item[0])
    return  listOfKeys

sem_li = []

with open('temperaturas_2021.csv') as f:
    csv_reader = csv.reader(f, delimiter=';')
    for row in csv_reader:
        sem_li.append(row)

days = {}

for day in sem_li[1:]:
    days[day[0]] = (sum([float(temp) for temp in day[1:]])/len(day[1:]))

plt.plot(*zip(*days.items()))
plt.ylabel('Average Temperature')
plt.xlabel('Days')

all_values = days.values()
max_value = max(all_values)
min_value = min(all_values)

for x in getKeysByValues(days , max_value):
    plt.plot(x , max_value, marker = 'o')
    plt.text(x, max_value , round(max_value , 2) , fontsize=7)

for x in getKeysByValues(days, min_value):
    plt.plot(x , min_value, marker = 'o')
    plt.text(x, min_value , round(min_value , 2), fontsize=7)

plt.show()

print(days)


import solution
import numpy as np

class Solution(solution.Solution):
    def solve(self, test_input=None):
        return self.checkRecord(test_input)

    def checkRecord(self, n):
        """
        :type n: int
        :rtype: int
        """
        # mod = 10 ** 9 + 7
        # alast, alastL, alastLL, last, lastL, lastLL = 1, 0, 0, 1, 1, 0
        # for i in range(2, n + 2):
        #     alast, alastL, alastLL, last, lastL, lastLL = (alastLL + alastL + alast + last + lastL + lastLL) % mod,alast,alastL,(last + lastL + lastLL) % mod,last,lastL
        # return alast

        mod = 10 ** 9 + 7
        cell = np.diag([1] * 6)
        mul = np.array(
            [[1, 1, 1, 0, 0, 0], [1, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0], [1, 0, 0, 1, 1, 1], [1, 0, 0, 1, 0, 0],
             [0, 0, 0, 0, 1, 0]])
        n -= 1
        while n:
            if n & 1:
                cell = np.mod(np.dot(cell, mul), mod)
            mul = np.mod(np.dot(mul, mul), mod)
            n >>= 1
        ans = np.mod(np.dot(cell, np.array([1, 1, 0, 0, 0, 0])), mod)
        return int((sum(ans) + ans[0]) % mod)


"""Bolted Entity Manager"""
from collections.abc import Mapping, MutableMapping
import logging
from typing import Any, Optional

import homeassistant.helpers.device_registry as hass_device_registry
import homeassistant.helpers.entity_registry as hass_entity_registry
from homeassistant.helpers.restore_state import (
    ExtraStoredData,
    RestoredExtraData,
    RestoreEntity,
)

from .helpers import ObservableVariable

_LOGGER: logging.Logger = logging.getLogger(__package__)

class EntityManager:
    """Entity Manager."""

    hass = None

    platform_adders = {}
    platform_classes = {}
    registered_entities = {}

    @classmethod
    def init(cls, hass):
        """Initialize Class Variables"""
        cls.hass = hass
        cls.entity_registry = hass_entity_registry.async_get(hass)
        cls.device_registry = hass_device_registry.async_get(hass)

    @classmethod
    def register_platform(cls, platform, adder, entity_class):
        """Register platform from Home Assistant"""
        _LOGGER.debug(
            "Platform %s Registered",
            platform,
        )
        cls.platform_adders[platform] = adder
        cls.platform_classes[platform] = entity_class
        cls.registered_entities[platform] = {}

    @classmethod
    async def get(cls, bolted, platform, name, restore=False):
        """Get an Entity from Bolted"""
        unique_id = f"{bolted.__class__.__module__}::{bolted.name}::{name}"
        await cls.wait_platform_registered(platform)
        if (
            platform not in cls.registered_entities
            or unique_id not in cls.registered_entities[platform]
        ):
            await cls.create(bolted, platform, name, restore=restore)

        return cls.registered_entities[platform][unique_id]

    @classmethod
    async def create(cls, bolted, platform, name, restore=False):
        """Create entity from Bolted."""
        unique_id = f"{bolted.__class__.__module__}::{bolted.name}::{name}"
        await cls.wait_platform_registered(platform)
        _LOGGER.debug("Created New Entity %s %s", platform, unique_id)
        new_entity = cls.platform_classes[platform](
            cls.hass, bolted, unique_id, restore=restore
        )
        new_entity.entity_id = f"{platform}.{bolted.name}_{name}"
        cls.platform_adders[platform]([new_entity])
        await new_entity.wait_for_added()
        cls.registered_entities[platform][unique_id] = new_entity

    @classmethod
    def remove(cls, entity):
        entity.set(None, {})
        # entity IDs don't stick with this. commenting for now.
        # entity_id = entity.entity_id
        # unique_id = entity.unique_id
        # entity_platform, _ = entity_id.split('.', 1)
        # _LOGGER.debug('Removing Entity %s', entity_id)
        # cls.entity_registry.async_remove(entity_id)
        # del cls.registered_entities[entity_platform][unique_id]

    @classmethod
    async def wait_platform_registered(cls, platform):
        """Wait for platform registration."""
        if platform not in cls.platform_classes:
            raise KeyError(f"Platform {platform} not registered.")

        return True

    @classmethod
    def get_by_entity_id(cls, entity_id):
        return cls.entity_registry.async_get(entity_id)

    @classmethod
    def get_device_id(cls, entity_id):
        this_entity = cls.get_by_entity_id(entity_id)
        if this_entity is None:
            return None
        return this_entity.device_id

    @classmethod
    def get_device_by_entity_id(cls, entity_id):
        device_id = cls.get_device_id(entity_id)
        if device_id is None:
            return None
        return cls.device_registry.async_get(device_id)

class BoltedEntity(RestoreEntity):
    """Base Class for all Bolted Entities"""

    _attr_unique_id: Optional[str] = None
    _attr_should_poll = False
    _attr_extra_state_attributes: MutableMapping[str, Any]
    _attr_bolted_state_attributes: MutableMapping[str, Any]
    _restorable_attributes = None

    def __init__(self, hass, bolted, unique_id, restore=False):
        self.hass = hass
        self.bolted = bolted
        self._added = ObservableVariable(False)
        self._should_restore = restore

        self._attr_unique_id = unique_id

        self._attr_bolted_state_attributes = {
            "bolted_app": self.bolted.__class__.__module__,
            "bolted_app_name": self.bolted.name,
        }

        _LOGGER.debug(
            "Entity Initialized %s",
            self.unique_id,
        )

    async def wait_for_added(self):
        while self._added.value is not True:
            _LOGGER.debug("Waiting for Entity to be added %s", self.unique_id)
            await self._added.wait()

    @property
    def extra_state_attributes(self) -> Optional[Mapping[str, Any]]:
        """Return entity specific state attributes."""
        attrs = {}
        if hasattr(self, "_attr_extra_state_attributes"):
            attrs.update(self._attr_extra_state_attributes)
        if hasattr(self, "_attr_bolted_state_attributes"):
            attrs.update(self._attr_bolted_state_attributes)
        return attrs

    @property
    def extra_restore_state_data(self) -> Optional[ExtraStoredData]:
        """Return entity specific state data to be restored.
        Implemented by platform classes.
        """
        if self._restorable_attributes is None:
            return None

        restore = dict()
        for key in self._restorable_attributes:
            restore[key] = getattr(self, key)
        return RestoredExtraData(restore)

    async def async_added_to_hass(self):
        """Called when Home Assistant adds the entity to the registry"""
        await super().async_added_to_hass()

        if self._should_restore:
            last_data = await self.async_get_last_extra_data()
            self.bolted.logger.debug(
                "Last Data %s: %s", self.unique_id, last_data
            )
            if last_data is not None:
                for key, value in last_data.as_dict().items():
                    self.bolted.logger.debug(
                        "Restore %s = %s value", key, value
                    )
                    setattr(self, key, value)

        self._added.value = True
        self.async_update()

        _LOGGER.debug(
            "Entity %s Added to Hass as %s",
            self.unique_id,
            self.entity_id,
        )

    # USED INTERNALLY
    #####################################

    def async_update(self):
        """Request an entity update from Home Assistant"""
        if self._added.value is True:
            self.async_write_ha_state()


import os
import sys
import unittest

sys.path.insert(0, os.getcwd())

from pynewtonmath import core, wrapper

class TestWrapper (unittest.TestCase):
    def test_expose_endpoints (self):
        for op in core.ENDPOINTS:
            self.assertTrue(op in dir(wrapper))
            print('.', end='', flush=True)

    def test_endpoints (self):
        # operation: (expression, result)
        tests = {
            'simplify': ('x^2 + 2x', 'x^2 + 2 x'),
            'factor': ('x^2 + 2x', 'x (x + 2)'),
            'derive': ('x^2+2x', '2 x + 2'),
            'integrate': ('x^2+2x', '1/3 x^3 + x^2'), # + C missing
            'zeroes': ('x^2+2x', [-2, 0]),
            'tangent': ('2|x^3', '12 x + -16'),
            'area': ('2:4|x^3', 60),
            'cos': ('pi', -1),
            'sin': ('0', 0),
            'tan': ('0', 0),
            'arccos': ('1', 0),
            'arcsin': ('0', 0),
            'arctan': ('0', 0),
            'abs': ('-1', 1),
            'log': ('2|8', 3),
        }

        for op, test in tests.items():
            exp, result = test
            self.assertEqual(getattr(wrapper, op)(exp), result)
            print('.', end='', flush=True)

    def test_extended_endpoints (self):
        self.assertEqual(wrapper.tangent('x^3', 2), '12 x + -16')
        self.assertEqual(wrapper.area('x^3', 2, 4), 60)
        self.assertEqual(wrapper.log(8, 2), 3)

if __name__ == '__main__':
    unittest.main()


import os
import sys

from CLS.Vokker import Vokker

class VokkerConsole:
    _menue_: list = list()

    def __init__(self, vokker: Vokker):
        self._vokker_ = vokker
        self._menue_()

    def _menue_(self):
        self._menue_ = [
            '',
            '',
            '   Vokabeltrainer',
            '--------------------',
            '[filename]',
            '(1) öffnen',
            '(2) eingeben',
            '(3) auswerten',
            '(4) speichern',
            '(5) beenden',
            '--------------------',
        ]

    def show_menu(self):

        try:
            s = sys.winver
            os.system("cls")
        except:
            os.system("clear")

        for line in self._menue_:
            if line == '[filename]':
                print('\t\t', str(self._vokker_.datafile).center(20), '\n')
            else:
                print('\t\t',line)

    def imput_menue(self):
        pass


def subsetsum(l,n):
    s_max = sum(l)//2
    s = {0}
    for x in l:
        for y in s.copy():
            if (x+y)<=s_max:
                s.add(x+y)
    return s

n = int(input())
l = [int(it) for it in input().split(" ")]
l_sum = sum(l)

s = subsetsum(l,n)
diff= l_sum//2-max(s)

if l_sum%2==0:
    print(2*diff)
else:
    print(2*diff+1)


N = int(input())

lst = []
for i in range(N):
    L, H = map(int,input().split())
    lst.append([L,H])

# x축을 기준으로 정렬
lst.sort(key=lambda x: x[0])

# 가장 높은 기둥의 면적을 구하고 미리 더해주기
value = 0
for i in range(len(lst)) :
    if lst[i][1] > value :
        value = lst[i][1]
        idx = i

# 처음 높이는 첫번째 기둥의 높이
high = lst[0][1]

# 가장 높은 기둥 전까지 돌면서 면적을 추가해주기
# value에 면적을 계산해서 더해주고 높이를 다음 기둥으로 갱신
for i in range(idx) :
    if high < lst[i+1][1] :
        value += high * (lst[i+1][0]-lst[i][0])
        high = lst[i+1][1]
    # 높이가 낮다면 이전 높이를 그대로하고 현재 면적을 더하기
    else :
        value += high * (lst[i+1][0] - lst[i][0])

# 뒤에서도 가장 큰 기둥까지 면적을 추가
high = lst[-1][1]

for i in range(N-1, idx, -1) :
    if high < lst[i-1][1] :
        value += high * (lst[i][0]-lst[i-1][0])
        high = lst[i-1][1]
    else :
        value += high * (lst[i][0] - lst[i-1][0])

print(value)


#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
    author comger@gmail.com
"""
import sys
import json

sys.path.append('../../')

from datetime import datetime
from pprint import pprint
from kpages import run

def callback(app):

    print("Start time: {0}".format(datetime.now().isoformat(" ")))
    print("Config Params")

    for k in sorted(app.settings.keys()):
        if k.startswith("__"):
            continue
        print("{0:<40}:{1}".format(k, app.settings[k]))

    print("Router Handlers")
    for h in app.handlers:
        print('{0:<50}:{1}'.format(str(h[1]),str(h[0])))

if __name__ == "__main__":
    try:
        run(callback)
    except KeyboardInterrupt:
        print('exit server ')

# vim: ts=4 sw=4 sts=4 expandtab


import difflib
import re
from typing import Literal, TypedDict

import nltk

def segment_sentences_with_nltk(text: str) -> list[str]:
    sent_detector = nltk.data.load("tokenizers/punkt/german.pickle")
    return list(sent_detector.tokenize(text.strip()))

_PUNCTUATION_IN_MIDDLE_RE = re.compile("([\\w\\s,;:\\-]*)([?!.])([\\w\\s,:;\\-]*)")
_END_PUNCTIONATION_FROM_MIDDLE_RE = re.compile(r"([\w\s,;:\-]+)([?!]+)([\w\s,:;\-]+)")
_END_PUNCTIONATION_RE = re.compile(r'([\w\s,;:"\-()]+)[.?!]+(»|\"|\')*\Z')
_UPPERCASE_START_RE = re.compile(r"\A(«|\"|\')*[A-ZÄÖÜ]")
_LOWERCASE_START_RE = re.compile(r"\A[a-zäöü]")
_NON_WHITESPACE_RE = re.compile(r"\S+")

def contains_end_punctuation_in_the_middle(seq: str) -> bool:
    return _PUNCTUATION_IN_MIDDLE_RE.search(seq) is not None

def retrieve_end_punctuation_from_the_middle(seq: str) -> str | None:
    end_punctuation = _END_PUNCTIONATION_FROM_MIDDLE_RE.search(seq)
    if end_punctuation:
        return end_punctuation.group(2)
    return None

def ends_with_end_punctuation(seq: str) -> bool:
    return _END_PUNCTIONATION_RE.search(seq) is not None

def is_short_sequence(seq: str, length: int) -> bool:
    return len(seq) < length

def starts_with_uppercase_letter(seq: str) -> bool:
    uppercase_at_beginning = _UPPERCASE_START_RE.search(seq)
    return uppercase_at_beginning is not None

def starts_with_lowercase_letter(seq: str) -> bool:
    lowercase_at_beginning = _LOWERCASE_START_RE.search(seq)
    return lowercase_at_beginning is not None

def calculate_sequence_similarity(a: str, b: str) -> float:
    return difflib.SequenceMatcher(None, a, b).ratio()

def retrieve_match_range(seq1: str, seq2: str) -> list[difflib.Match]:
    seq_match = difflib.SequenceMatcher(None, seq1.strip(), seq2.strip())
    return seq_match.get_matching_blocks()

def retrieve_mismatch_ranges(
    seq1: str, seq2: str
) -> list[tuple[str, int, int, int, int]]:
    seq_match = difflib.SequenceMatcher(None, seq1, seq2)
    return seq_match.get_opcodes()

def check_overlap_with_seq_beginning(
    s1: str, s2: str
) -> tuple[Literal[True], str, int] | tuple[Literal[False], str, None]:
    # s1 is modifying seq and s2 is sen
    index = 1
    s1_beginning = s1[:index]
    if s1_beginning not in s2:
        return False, "", None

    while s1_beginning in s2 and len(s1_beginning) < len(s1):
        index += 1
        s1_beginning = s1[:index]
    return True, s1_beginning, index

def check_overlap_with_seq_end(
    s1: str, s2: str
) -> tuple[Literal[True], str, int] | tuple[Literal[False], str, None]:
    # s1 is modifying seq and s2 is sen
    index = len(s1)
    s1_end = s1[index:]
    if s1_end not in s2:
        return False, "", None

    while s1_end in s2 and len(s1_end) < len(s1):
        index -= 1
        s1_end = s1[index:]
    return True, s1_end, index

def retrieve_token_indices(
    prev_sen: str, cur_sen: str
) -> tuple[list[tuple[str, int, int]], list[tuple[str, int, int]]]:
    prev_toks_with_indices = [
        (match.group(0), match.start(), match.end() - 1)
        for match in _NON_WHITESPACE_RE.finditer(prev_sen)
    ]
    cur_toks_with_indices = [
        (match.group(0), match.start(), match.end() - 1)
        for match in _NON_WHITESPACE_RE.finditer(cur_sen)
    ]
    return prev_toks_with_indices, cur_toks_with_indices

# def identify_affected_tokens(
#     prev_sen_toks: list, cur_sen_toks: list, edit_type: str
# ) -> list:
#     affected_tokens = []
#     if prev_sen_toks and cur_sen_toks:
#         for pt, ct in itertools.zip_longest(prev_sen_toks, cur_sen_toks):
#             print(ct)
#             if pt["text"] != ct["text"]:
#                 if edit_type == "insertion":
#                     affected_tokens.append({"prev": None, "cur": ct})
#                 elif edit_type == "deletion":
#                     affected_tokens.append({"prev": pt, "cur": None})
#     elif not prev_sen_toks and cur_sen_toks:
#         for ct in cur_sen_toks:
#             affected_tokens.append({"prev": None, "cur": ct})
#     elif prev_sen_toks and not cur_sen_toks:
#         for pt in prev_sen_toks:
#             affected_tokens.append({"prev": pt, "cur": None})
#     print("AFFECTED TOKENS:")
#     print(affected_tokens)
#     return affected_tokens

# def retrieve_affected_tokens(prev_sen, cur_sen) -> list:
#     # affected_tokens is a list of tuples: previous word, current word with their indices
#     affected_tokens = []
#     prev_toks_with_indices, cur_toks_with_indices = retrieve_token_indices(
#         prev_sen, cur_sen
#     )
#     _, mismatch_range, _ = retrieve_mismatch_range_for_sentence_pair(prev_sen, cur_sen)
#     if mismatch_range:
#         # as there is only one edit per TPSF (one sequence gets changed), there can always be only one mismatch range
#         # if more than one mismatch range exists, merge all consecutive mismatch ranges together
#         # multiple mismatch ranges occur if the inserted or deleted sequence is very short
#         # and can be found at another position in the sentence which hasn't been edited
#         mismatch_range = range(mismatch_range[0][0], mismatch_range[-1][-1] + 1)
#         for pt, ct in itertools.zip_longest(
#             prev_toks_with_indices, cur_toks_with_indices
#         ):
#             if pt is not None and ct is not None:
#                 if (
#                     mismatch_range[0] <= pt[2]
#                     and pt[1] <= mismatch_range[-1]
#                     or mismatch_range[0] <= ct[2]
#                     and ct[1] <= mismatch_range[-1]
#                 ):  # TODO to test
#                     affected_token_pair = {"prev_tok": pt, "cur_tok": ct}
#                     affected_tokens.append(affected_token_pair)
#             elif pt is None:
#                 pt = ("", None, None)
#                 if mismatch_range[0] <= ct[2] and ct[1] <= mismatch_range[-1]:
#                     affected_token_pair = {"prev_tok": pt, "cur_tok": ct}
#                     affected_tokens.append(affected_token_pair)
#             elif ct is None:
#                 ct = ("", None, None)
#                 if mismatch_range[0] <= pt[2] and pt[1] <= mismatch_range[-1]:
#                     affected_token_pair = {"prev_tok": pt, "cur_tok": ct}
#                     affected_tokens.append(affected_token_pair)
#     else:
#         for ct in cur_toks_with_indices:
#             pt = ("", None, None)
#             affected_token_pair = {"prev_tok": pt, "cur_tok": ct}
#             affected_tokens.append(affected_token_pair)
#     return affected_tokens

# def filter_out_irrelevant_tokens(tokens: list) -> list:
#     relevant_tokens = []
#     for tok in tokens:
#         if tokens['prev'][1] is not None and tokens['cur'][1] is not None:
#             relevant_tokens.append(tok)
#     return relevant_tokens

class TokenDict(TypedDict):
    text: str

class TokensDict(TypedDict):
    prev: TokenDict | None
    cur: TokenDict | None

def check_edit_distance(tokens: TokensDict) -> int:
    prev_tok = "" if tokens["prev"] is None else tokens["prev"]["text"]
    cur_tok = "" if tokens["cur"] is None else tokens["cur"]["text"]
    return nltk.edit_distance(prev_tok, cur_tok)

def retrieve_mismatch_range_for_sentence_pair(
    prev_sen: str, cur_sen: str
) -> tuple[str | None, list[range], str]:
    seq_match = difflib.SequenceMatcher(None, prev_sen, cur_sen)
    prev_cur_match = seq_match.get_opcodes()
    mismatch_range = []
    edit = None
    relevant = ""
    # there is always one mismatch range, as TPSF capturing happens upon each edit,
    # two separate edits on the same TPSF are not possible
    # TODO make it more generic
    for m in prev_cur_match:
        if m[0] == "delete":
            edit = m[0]
            mismatch_range.append(range(m[1], m[2] + 1))
            relevant = "prev"
        elif m[0] == "insert":
            edit = m[0]
            mismatch_range.append(range(m[3], m[4] + 1))
            relevant = "cur"
        elif m[0] == "replace":
            edit = m[0]
            mismatch_range.append(range(max(m[1], m[3]), max(m[2] + 1, m[4] + 1)))
            relevant = (
                "prev"
                if m[1] in mismatch_range[0] and m[2] in mismatch_range[-1]
                else "cur"
            )
    return edit, mismatch_range, relevant


#!/usr/bin/env python3

import os
import click
import hashlib
import imageio
from .jpeg import *
import subprocess as sp
from pathlib import Path
from contextlib import suppress
from multiprocessing import cpu_count

import concurrent.futures

cache_dir = Path.home() / '.cache/glitch-art-display'
cache_dir.mkdir(exist_ok=True)

frame_timings = [5, 4, 3, 2, 2, 1, 1, 1, 1]

def is_cached(filename):

    if (cache_dir / filename).is_file():
        return True
    return False

def find_images(directory):

    valid_suffixes = ['.jpg', '.jpeg']
    convert_suffixes = ['.png']

    for root,dirs,files in os.walk(directory):
        for file in files:
            filename = Path(root) / file

            # if image is a JPEG
            if any([filename.suffix.lower().endswith(s) for s in valid_suffixes]):
                yield filename

            # if image is an unsupported image type
            elif any([filename.suffix.lower().endswith(s) for s in convert_suffixes]):
                # try to convert it with imagemagick
                try:
                    new_filename = cache_dir / (filename.stem + '.jpg')
                    sp.run(['convert', str(filename), str(new_filename)], check=True)
                    yield new_filename
                except (FileNotFoundError, sp.CalledProcessError) as e:
                    print(f'[!] Unsupported file: {filename.name}')
                    print(f'[!]  - please install imagemagick in order to use {filename.suffix} files')

def gen_frames(image_dir, output, glitch_amount=100, fps=25, num_image_frames=25, num_transition_frames=30, shuffle=False):

    frame_groups = []
    frames_output = Path(output).resolve()
    frames_output.parent.mkdir(parents=True, exist_ok=True)
    frame_number = 0

    transition_frames = int(num_transition_frames/2)
    found_images = list(find_images(image_dir))
    if shuffle:
        random.shuffle(found_images)

    with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count()) as pool:
        for image in found_images:
            try:
                print(f'[+] Generating frames for {image.name}')

                glitched_frames = []

                image_bytes = bytearray(image.read_bytes())
                image_hash = hashlib.md5(image_bytes).hexdigest()
                png_filename = cache_dir / f'{image_hash}.png'
                if not png_filename.exists():
                    pool.submit(sp.run, ['convert', str(image), str(png_filename)], check=True)

                # generate glitch frames
                glitched_futures = []
                for i in range(transition_frames):
                    amount = int(glitch_amount * ((i+1)/transition_frames))
                    glitched_futures.append(pool.submit(glitch, image, amount, i))

                concurrent.futures.wait(glitched_futures)
                for glitched_future in glitched_futures:
                    glitched_frame = glitched_future.result()
                    if glitched_frame:
                        glitched_frames.append(glitched_frame)
            except KeyboardInterrupt:
                pool.shutdown(wait=False, cancel_futures=True)
                raise

            glitch_in = []
            random.shuffle(frame_timings)
            for i, glitched_image in enumerate(glitched_frames[::-1]):
                glitch_group = []
                for j in range(frame_timings[i % len(frame_timings)]):
                    glitch_group.append(glitched_image)
                glitch_in.append(glitch_group)

            normal = []
            for i in range(num_image_frames):
                normal.append(png_filename)

            glitch_out = []
            random.shuffle(frame_timings)
            for i, glitched_image in enumerate(glitched_frames):
                glitch_group = []
                for j in range(frame_timings[i % len(frame_timings)]):
                    glitch_group.append(glitched_image)
                glitch_out.append(glitch_group)

            frame_groups.append([glitch_in, normal, glitch_out])

    frames = []
    interlace_frames = max(2, min(int(transition_frames/4), 1))
    for i, [glitch_in, normal, glitch_out] in enumerate(frame_groups):
        for g in glitch_in:
            frames += g
        frames += normal
        if i == len(frame_groups)-1:
            for g in glitch_out:
                frames += g
        else:
            # truncate end of this glitch group
            glitch_out, glitch_interlaced = glitch_out[:-interlace_frames], glitch_out[-interlace_frames:]

            # truncate beginning of next glitch group
            frame_groups[i+1][0], glitch_next_interlaced = frame_groups[i+1][0][interlace_frames:], frame_groups[i+1][0][:interlace_frames]

            for g in glitch_out:
                frames += g
            for i in range(min(interlace_frames, len(glitch_interlaced))):
                frames += glitch_next_interlaced[i]
                frames += glitch_interlaced[i]

    w = imageio.get_writer(str(frames_output), format='FFMPEG', mode='I', fps=fps, macro_block_size=1)
    try:
        prev_frame = ''
        for i, frame in enumerate(frames):
            print(f'\r[+] Rendering frame {i:,}/{len(frames):,} ({i/len(frames)*100:.1f}%)', end='')
            frame = str(frame)
            if frame != prev_frame:
                try:
                    read_frame = imageio.imread(frame, pilmode='RGB')
                except Exception as e:
                    print(f'[!] Error reading {frame}: {e}')
                    continue
            try:
                w.append_data(read_frame)
            except ValueError:
                print(f'[!] {frame}')
                raise
            prev_frame = str(frame)
    finally:
        with suppress(Exception):
            w.close()

    print(f'[+] Video saved to {frames_output}')

def glitch(image, amount=None, sequence=''):

    print(f'[+] Glitching {image.name} by {amount}')

    if amount is None:
        amount = random.randint(0,99)
    else:
        amount = max(0, min(99, int(amount)-1))

    image = Path(image)

    image_bytes = bytearray(image.read_bytes())
    image_hash = hashlib.md5(image_bytes).hexdigest()
    try:
        jpeg = Jpeg(image_bytes)
    except JpegError as e:
        print(f'[!] {e}')
        return

    jpeg_filename = cache_dir / f'{image_hash}_{amount}_{sequence}.jpg'
    png_filename = cache_dir / f'{image_hash}_{amount}_{sequence}.png'

    print(f'[+] Glitching {image.name} by {amount}')

    # checked for cached files
    if is_cached(png_filename):
        print(f'[+] Found cached frame for {png_filename}')

    else:
        while 1:
            try:
                jpeg.amount      = amount
                jpeg.seed        = random.randint(0,99)
                jpeg.iterations  = max(0, min(115, int(amount*1.15)))

                # create a new image if not cached
                jpeg.save_image(jpeg_filename)
                try:
                    sp.run(['convert', str(jpeg_filename), str(png_filename)], check=True)
                    break
                except Exception:
                    continue

            except JpegError as e:
                print(f'[!] {e}')
                continue

    return png_filename

def main():
    @click.command(context_settings={'show_default': True})
    @click.argument('input', required=True, type=click.Path(exists=True, file_okay=False, dir_okay=True))
    @click.argument('output', required=True, type=click.Path(dir_okay=False))
    @click.option('--amount', default=50, type=click.IntRange(1, 100, clamp=True), help='Glitch amount')
    @click.option('--fps', default=25, type=float, help='Frames per second')
    @click.option('--normal-frames', default=25*25, type=int, help='Number of normal frames')
    @click.option('--transition-frames', default=30, type=int, help='Number of glitchy transition frames')
    @click.option('--dont-shuffle', is_flag=True, help='Don\'t shuffle order of images')
    def go(input, output, amount, fps, normal_frames, transition_frames, dont_shuffle):
        gen_frames(input, output, amount, fps, normal_frames, transition_frames, not dont_shuffle)

    try:
        go()
    except KeyboardInterrupt:
        print("Interrupted")

if __name__ == '__main__':
    main()


VALUE_TYPE_LIST = ["X","Y","Z","Temperature","Humidity","Current"]

SENSOR_TYPE_DICT = {
    0:"Vibration",
    1:"Temperature",
    2:"Humidity",
    3:"TemperatureHumidity",
    4:"Current",
    "Vibration" :0,
    "Temperature":1,
    "Humidity":2,
    "TemperatureHumidity":3,
    "Current": 4
}

SENSOR_VALUE_MAP_DICT = {
    "Vibration" :["X","Y","Z"],
    "Temperature":["Temperature"],
    "Humidity":["Humidity"],
    "TemperatureHumidity":["Temperature","Humidity"],
    "Current":["Current"]
}


#1
x = [ [5,2,3], [10,8,9] ]
students = [
    {'first_name':  'Michael', 'last_name' : 'Jordan'},
    {'first_name' : 'John', 'last_name' : 'Rosales'}
]
sports_directory = {
    'basketball' : ['Kobe', 'Jordan', 'James', 'Curry'],
    'soccer' : ['Messi', 'Ronaldo', 'Rooney']
}
z = [ {'x': 10, 'y': 20} ]

x[1][0] = 15
print(x)

students[0]["first_name"] = "Bryant"
print(students)

sports_directory["soccer"][0] = "Andres"
print(sports_directory)

z[0]["y"] = 30
print(z)

students1 = [
        {'first_name':  'Michael', 'last_name' : 'Jordan'},
        {'first_name' : 'John', 'last_name' : 'Rosales'},
        {'first_name' : 'Mark', 'last_name' : 'Guillen'},
        {'first_name' : 'KB', 'last_name' : 'Tonel'}
    ]
#iterateDictionary(students)
# should output: (it's okay if each key-value pair ends up on 2 separate lines;
# bonus to get them to appear exactly as below!)
#first_name - Michael, last_name - Jordan
#first_name - John, last_name - Rosales
#first_name - Mark, last_name - Guillen
#first_name - KB, last_name - Tonel

#2
def iterate_dictionary(some_list):
    for i in students1:
        print(i["first_name"])
        print(i["last_name"])

print("2")
print(iterate_dictionary(students1))
print("---------------------------------------------------------")

#3
def iterate_dictionary2(key_name, some_list):
    for i in range(len(some_list)):
        for key, val in some_list[i].items():
            if key == key_name:
                print(val)
print("3")
iterate_dictionary2("first_name", students1)
print("------------------------------------------------------")

#4
dojo = {
    'locations': ['San Jose', 'Seattle', 'Dallas', 'Chicago', 'Tulsa', 'DC', 'Burbank'],
    'instructors': ['Michael', 'Amy', 'Eduardo', 'Josh', 'Graham', 'Patrick', 'Minh', 'Devon']
}

"""
printInfo(dojo)
# output:
7 LOCATIONS
San Jose
Seattle
Dallas
Chicago
Tulsa
DC
Burbank

8 INSTRUCTORS
Michael
Amy
Eduardo
Josh
Graham
Patrick
Minh
Devon
"""

#print the name of each key along with the size of its lists,
#and then print the associated values within each key's list

list = ("instructors", "locations")

some_dict = dojo

def printInfo(some_dict):
    for key, val in some_dict.items():
        print(len(val))
    for key, val in some_dict.items():
        print(key)
    for val in some_dict["locations"]:
        print(val)
    for val in some_dict["instructors"]:
        print(val)

printInfo(dojo)

"""

"""


from esphome.components import number
from esphome.const import CONF_MAX_VALUE, CONF_MIN_VALUE, CONF_STEP
import esphome.config_validation as cv
import esphome.codegen as cg

from .. import fourheat_config_validation as fhcv
from .. import (
    fourheat_ns,
    CONF_DATAPOINT,
    CONF_FOURHEAT_ID,
    CONF_PARSER,
    CONF_QUERY_DATAPOINT,
    FourHeat,
    get_parser_expression,
)

DEPENDENCIES = ["fourheat"]

FourHeatNumber = fourheat_ns.class_("FourHeatNumber", number.Number, cg.Component)

def validate_min_max(config):
    if config[CONF_MAX_VALUE] <= config[CONF_MIN_VALUE]:
        raise cv.Invalid("max_value must be greater than min_value")
    return config

CONFIG_SCHEMA = cv.All(
    number.number_schema(FourHeatNumber)
    .extend(
        {
            cv.GenerateID(CONF_FOURHEAT_ID): cv.use_id(FourHeat),
            cv.Required(CONF_DATAPOINT): fhcv.datapoint,
            cv.Optional(CONF_QUERY_DATAPOINT): fhcv.datapoint,

            cv.Optional(CONF_PARSER): cv.returning_lambda,

            cv.Optional(CONF_MAX_VALUE, default=16777215.0): cv.float_,
            cv.Optional(CONF_MIN_VALUE, default=-16777215.0): cv.float_,
            cv.Optional(CONF_STEP, default=1): cv.positive_float,
        }
    )
    .extend(cv.COMPONENT_SCHEMA),
    validate_min_max,
)

async def to_code(config):
    var = await number.new_number(
        config,
        min_value=config[CONF_MIN_VALUE],
        max_value=config[CONF_MAX_VALUE],
        step=config[CONF_STEP]
    )
    await cg.register_component(var, config)

    parent = await cg.get_variable(config[CONF_FOURHEAT_ID])
    cg.add(var.set_fourheat_parent(parent))

    cg.add(var.set_datapoint_id(config[CONF_DATAPOINT]))

    if CONF_QUERY_DATAPOINT in config:
        cg.add(var.set_query_datapoint_id(config[CONF_QUERY_DATAPOINT]))

    if CONF_PARSER in config:
        parser = await get_parser_expression(config[CONF_PARSER], int)
        cg.add(var.set_parser(parser))


#!/usr/bin/python
# -*- coding: utf-8 -*-
from os import listdir,path,curdir
from os.path import isfile, join
import re
import shutil
import pandas as pd
from sklearn.model_selection import train_test_split
import csv
import random

root = "/data01/ML/dataset/FACE_CLASSIFIER"
face_image_path = "/data01/ML/dataset/FACE_CLASSIFIER/CelebA2/CelebA/Img/img_celeba/img_celeba/img_celeba"
objects_image_path = "/data01/ML/dataset/FACE_CLASSIFIER/256_ObjectCategories_modified"
face_image_path2 = "/data01/ML/dataset/FACE_CLASSIFIER/Borderline"

def get_faces(balanced, face_image_path, face_image_path2):
    filenames = []
    classification = []
    files = [f for f in listdir(face_image_path) if
             isfile(join(face_image_path, f))]
    files = random.sample(files, balanced-600)
    for f in files:
        filenames.append(face_image_path + "/" + f)
        classification.append(1)
    files2 = [f for f in listdir(face_image_path2) if
             isfile(join(face_image_path2, f))]
    files2 = random.sample(files2, 600)
    for f in files2:
        filenames.append(face_image_path2 + "/" + f)
        classification.append(1)
    return filenames, classification

def get_balanced_data_from_classes(min,objects_main_folder):
    filenames = []
    classification = []
    dirs = [f for f in listdir(objects_main_folder) if not isfile(join(objects_main_folder, f))]
    for dir_ in dirs:
        files = [f for f in listdir(objects_main_folder + "/" + dir_) if
                 isfile(join(objects_main_folder + "/" + dir_, f))]
        files = random.sample(files, min)
        for f in files:
            filenames.append(objects_main_folder+"/"+dir_+"/"+f)
            classification.append(0)
    return filenames, classification

def get_not_balanced_data_from_classes(objects_main_folder):
    filenames = []
    classification = []
    dirs = [f for f in listdir(objects_main_folder) if not isfile(join(objects_main_folder, f))]
    for dir_ in dirs:
        files = [f for f in listdir(objects_main_folder + "/" + dir_) if
                 isfile(join(objects_main_folder + "/" + dir_, f))]
        for f in files:
            filenames.append(objects_main_folder + "/" + dir_ + "/" + f)
            classification.append(0)
    return filenames, classification

def count_min_in_objects_types(objects_main_folder):
    dirs = [f for f in listdir(objects_main_folder) if not isfile(join(objects_main_folder, f))]
    min_f_numbers = 1000000
    max_f_numbers = 0
    for dir_ in dirs:
        files = [f for f in listdir(objects_main_folder+"/"+dir_) if isfile(join(objects_main_folder+"/"+dir_, f))]
        if len(files)< min_f_numbers:
            min_f_numbers = len(files)
        if len(files) > max_f_numbers:
            max_f_numbers = len(files)
    return min_f_numbers

def get_min_between_celeba_and_objects(objects_main_folder, celeba_images_folder):
    dirs = [f for f in listdir(objects_main_folder) if not isfile(join(objects_main_folder, f))]
    count_objects = 0
    count_faces = 0
    for dir_ in dirs:
        files = [f for f in listdir(objects_main_folder + "/" + dir_) if
                 isfile(join(objects_main_folder + "/" + dir_, f))]
        count_objects = count_objects + len(files)
    files_Faces = [f for f in listdir(celeba_images_folder) if
                   isfile(join(celeba_images_folder, f))]
    count_faces = count_faces + len(files_Faces)
    print(count_faces, count_objects)
    return count_faces if count_faces < count_objects else count_objects

def write_csv(category, non_faces, faces):
    with open("./dataset/" + category+".csv" , mode='w', newline='') as file:
        file = csv.writer(file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        file.writerow(['image_path', 'face'])
        for el in non_faces:
            file.writerow([el[0], el[1]])
        for el in faces:
            file.writerow([el[0], el[1]])

if __name__ == '__main__':
    #min = count_min_in_objects_types(objects_image_path)
    #non_faces = get_balanced_data_from_classes(min,objects_image_path)
    min = get_min_between_celeba_and_objects(objects_image_path, face_image_path)
    non_faces = get_not_balanced_data_from_classes(objects_image_path)
    print(len(non_faces[0]))
    #print(non_faces[1])
    non_faces_x_train, non_faces_x_test, non_faces_y_train, non_faces_y_test = train_test_split(non_faces[0], non_faces[1], test_size = 0.15,shuffle=True)
    non_faces_x_train, non_faces_x_val, non_faces_y_train, non_faces_y_val = train_test_split(non_faces_x_train, non_faces_y_train, test_size= 0.15,shuffle=True)
    print(len(non_faces_x_train),len(non_faces_x_val), len(non_faces_x_test))
    #for el in zip(non_faces_x_train, non_faces_y_train):
    #    print(el)

    faces = get_faces(len(non_faces[0]), face_image_path, face_image_path2)
    print(len(faces[0]))
    #print(faces[1])
    faces_x_train, faces_x_test, faces_y_train, faces_y_test = train_test_split(faces[0],faces[1],test_size=0.15,shuffle=True)
    faces_x_train, faces_x_val, faces_y_train, faces_y_val = train_test_split(faces_x_train,faces_y_train,
                                                                                test_size=0.15,shuffle=True)
    print(len(non_faces_x_train), len(non_faces_x_val), len(non_faces_x_test))
    #Define true dataset
    non_faces_train, faces_train = zip(non_faces_x_train,non_faces_y_train),zip(faces_x_train,faces_y_train)
    non_faces_val, faces_val = zip(non_faces_x_val,non_faces_y_val),zip(faces_x_val,faces_y_val)
    non_faces_test, faces_test = zip(non_faces_x_test,non_faces_y_test),zip(faces_x_test,faces_y_test)

    for tuple in [("train3", non_faces_train,faces_train),("val3",non_faces_val,faces_val),("test3", non_faces_test,faces_test)]:
         print("Writing:", tuple[0])
         write_csv(tuple[0],tuple[1],tuple[2])
    # with open('C:/Users/Lorenzo/Desktop/Università/Machine_learning_project/dataset/FACE_CLASSIFIER/dataset.csv' , mode='w', newline='') as file:
    #     file = csv.writer(file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
    #     file.writerow(['image_path', 'face'])
    #     for el in non_faces:
    #         file.writerow([el[0], el[1]])
    #     for el in faces:
    #         file.writerow([el[0], el[1]])