#!/usr/bin/env python

import re
import os
import pwd
import time
import glob
import json
import shlex
import base64
import socket
import urllib
import urllib2
import hashlib
import tempfile
import subprocess
from functools import wraps
from contextlib import contextmanager

from cloudify import ctx

REST_VERSION = 'v3.1'
PROCESS_POLLING_INTERVAL = 0.1
CLOUDIFY_SOURCES_PATH = '/opt/cloudify/sources'
MANAGER_RESOURCES_HOME = '/opt/manager/resources'
AGENT_ARCHIVES_PATH = '{0}/packages/agents'.format(MANAGER_RESOURCES_HOME)
MANAGER_RESOURCES_SNAPSHOT_PATHS = [
    os.path.join(MANAGER_RESOURCES_HOME, path)
    for path in (
        'blueprints',
        'deployments',
        'uploaded-blueprints',
        'snapshots',
        'plugins',
    )
]

SSL_CERTS_TARGET_DIR = '/etc/cloudify/ssl'

INTERNAL_SSL_CERT_FILENAME = 'cloudify_internal_cert.pem'
INTERNAL_SSL_KEY_FILENAME = 'cloudify_internal_key.pem'
INTERNAL_SSL_CA_CERT_FILENAME = 'cloudify_internal_ca_cert.pem'
INTERNAL_SSL_CA_KEY_FILENAME = 'cloudify_internal_ca_key.pem'
INTERNAL_PKCS12_FILENAME = 'cloudify_internal.p12'
EXTERNAL_SSL_CERT_FILENAME = 'cloudify_external_cert.pem'
EXTERNAL_SSL_KEY_FILENAME = 'cloudify_external_key.pem'

INTERNAL_CERT_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                  INTERNAL_SSL_CERT_FILENAME)
INTERNAL_KEY_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                 INTERNAL_SSL_KEY_FILENAME)
INTERNAL_CA_CERT_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                     INTERNAL_SSL_CA_CERT_FILENAME)
INTERNAL_CA_KEY_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                    INTERNAL_SSL_CA_KEY_FILENAME)
EXTERNAL_CERT_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                  EXTERNAL_SSL_CERT_FILENAME)
EXTERNAL_KEY_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                 EXTERNAL_SSL_KEY_FILENAME)
CERT_METADATA_FILE_PATH = os.path.join(SSL_CERTS_TARGET_DIR,
                                       'certificate_metadata')

BASE_LOG_DIR = '/var/log/cloudify'

NGINX_SERVICE_NAME = 'nginx'
DEFAULT_BUFFER_SIZE = 8192
SINGLE_TAR_PREFIX = 'cloudify-manager-resources'

CLOUDIFY_USER = 'cfyuser'
CLOUDIFY_GROUP = 'cfyuser'
CLOUDIFY_HOME_DIR = '/etc/cloudify'
SUDOERS_INCLUDE_DIR = '/etc/sudoers.d'
CLOUDIFY_SUDOERS_FILE = os.path.join(SUDOERS_INCLUDE_DIR, CLOUDIFY_USER)

CLUSTER_DELETE_SCRIPT = '/opt/cloudify/delete_cluster.py'
CFY_EXEC_TEMPDIR_ENVVAR = 'CFY_EXEC_TEMP'

def retry(exception, tries=4, delay=3, backoff=2):
    """Retry calling the decorated function using an exponential backoff.

    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
    """
    def deco_retry(f):
        @wraps(f)
        def f_retry(*args, **kwargs):
            mtries, mdelay = tries, delay
            while mtries > 1:
                try:
                    return f(*args, **kwargs)
                except exception as ex:
                    msg = "{0}, Retrying in {1} seconds...".format(ex, mdelay)
                    ctx.logger.warn(msg)
                    time.sleep(mdelay)
                    mtries -= 1
                    mdelay *= backoff
            return f(*args, **kwargs)
        return f_retry  # true decorator
    return deco_retry

def escape_for_systemd(the_string):
    if '\n' in the_string:
        the_string = the_string.replace('\n', '\\\\n\\\n')
        the_string = "\"" + the_string + "\""
    return the_string

def run(command, retries=0, stdin=b'', ignore_failures=False,
        globx=False, shell=False, env=None):
    if isinstance(command, str) and not shell:
        command = shlex.split(command)
    stderr = subprocess.PIPE
    stdout = subprocess.PIPE
    if globx:
        glob_command = []
        for arg in command:
            glob_command.append(glob.glob(arg))
        command = glob_command
    ctx.logger.debug('Running: {0}'.format(command))
    proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=stdout,
                            stderr=stderr, shell=shell, env=env)
    proc.aggr_stdout, proc.aggr_stderr = proc.communicate(input=stdin)
    if proc.returncode != 0:
        command_str = ' '.join(command)
        if retries:
            ctx.logger.warn('Failed running command: {0}. Retrying. '
                            '({1} left)'.format(command_str, retries))
            proc = run(command, retries - 1)
        elif not ignore_failures:
            msg = 'Failed running command: {0} ({1}).'.format(
                command_str, proc.aggr_stderr)
            raise RuntimeError(msg)
    return proc

def sudo(command, *args, **kwargs):
    if isinstance(command, str):
        command = shlex.split(command)
    if 'env' in kwargs:
        command = ['sudo', '-E'] + command
    else:
        command.insert(0, 'sudo')
    return run(command=command, *args, **kwargs)

def sudo_write_to_file(contents, destination):
    fd, path = tempfile.mkstemp()
    os.close(fd)
    with open(path, 'w') as f:
        f.write(contents)
    return move(path, destination)

def add_entry_to_sudoers(entry, description):
    # Comment out the description and add a N/L after the entry for visibility
    description = '# {0}'.format(description)
    entry = '{0}\n'.format(entry)

    for line in (description, entry):
        # `visudo` handles sudoers file. Setting EDITOR to `tee -a` means that
        # whatever is piped should be appended to the file passed.
        sudo(['/sbin/visudo', '-f', CLOUDIFY_SUDOERS_FILE],
             stdin=line, env={'EDITOR': '/bin/tee -a'})

    valid = sudo(
        ['visudo', '-cf', CLOUDIFY_SUDOERS_FILE],
        ignore_failures=True,
    )
    if valid.returncode != 0:
        ctx.abort_operation(
            "Generated sudoers entry containing \"{entry}\" was "
            "invalid.".format(entry=entry)
        )

def allow_user_to_sudo_command(full_command, description, allow_as='root'):
    entry = '{user}    ALL=({allow_as}) NOPASSWD:{full_command}'.format(
        user=CLOUDIFY_USER,
        allow_as=allow_as,
        full_command=full_command
    )
    add_entry_to_sudoers(entry, description)

def deploy_sudo_command_script(script, description, component=None,
                               allow_as='root'):
    # If passed a component, then script is a relative path, that needs to
    # be downloaded from the scripts folder. Otherwise, it's an absolute path
    if component:
        config_file_temp_destination = os.path.join(tempfile.gettempdir(),
                                                    script)
        ctx.download_resource_and_render(
            os.path.join('components', component, 'scripts', script),
            config_file_temp_destination)
        script = os.path.join('/opt/cloudify/', component, script)
        move(config_file_temp_destination, script)
        chmod('550', script)
        chown('root', CLOUDIFY_GROUP, script)

    ctx.logger.info('Allowing user `{0}` to run `{1}`'
                    .format(CLOUDIFY_USER, script))
    allow_user_to_sudo_command(full_command=script, description=description,
                               allow_as=allow_as)

def deploy_ssl_certificate(private_or_public, destination, group, cert):
    # Root owner, with permissions set below,
    # allow anyone to read a public cert,
    # and allow the owner to read a private cert, but not change it,
    # mitigating risk in the event of the associated service being vulnerable.
    ownership = 'root.{0}'.format(group)
    if private_or_public == 'private':
        private_cert_ok = 'PRIVATE KEY' in cert.split('\n')[0]
        if private_cert_ok:
            permissions = '440'
        else:
            ctx.abort_operation("Private certificate is expected to begin "
                                "with a line containing 'PRIVATE KEY'.")
    elif private_or_public == 'public':
        public_cert_ok = 'BEGIN CERTIFICATE' in cert.split('\n')[0]
        if public_cert_ok:
            permissions = '444'
        else:
            ctx.abort_operation("Public certificate is expected to begin with "
                                "a line containing 'BEGIN CERTIFICATE'.")
    else:
        ctx.abort_operation("Certificates may only be 'private' or 'public', "
                            "not {0}".format(private_or_public))
    ctx.logger.debug(
        "Deploying {0} SSL certificate in {1} for group {2}".format(
            private_or_public, destination, group))
    sudo_write_to_file(cert, destination)
    ctx.logger.debug('Setting permissions ({0}) and ownership ({1}) of '
                     'SSL certificate at {2}'.format(
                         permissions, ownership, destination))
    chmod(permissions, destination)
    sudo('chown {0} {1}'.format(ownership, destination))

def mkdir(dir, use_sudo=True):
    if os.path.isdir(dir):
        return
    ctx.logger.debug('Creating Directory: {0}'.format(dir))
    cmd = ['mkdir', '-p', dir]
    if use_sudo:
        sudo(cmd)
    else:
        run(cmd)

# idempotent move operation
def move(source, destination, rename_only=False):
    if rename_only:
        sudo(['mv', '-T', source, destination])
    else:
        copy(source, destination)
        remove(source)

def copy(source, destination):
    destination_dir = os.path.dirname(destination)
    if not os.path.exists(destination_dir):
        ctx.logger.debug(
            'Path does not exist: {0}. Creating it...'.format(
                destination_dir))
        sudo(['mkdir', '-p', destination_dir])
    sudo(['cp', '-rp', source, destination])

def remove(path, ignore_failure=False):
    ctx.logger.debug('Removing {0}...'.format(path))
    sudo(['rm', '-rf', path], ignore_failures=ignore_failure)

def generate_ca_cert():
    sudo([
        'openssl', 'req',
        '-x509',
        '-nodes',
        '-newkey', 'rsa:2048',
        '-days', '3650',
        '-batch',
        '-out', INTERNAL_CA_CERT_PATH,
        '-keyout', INTERNAL_CA_KEY_PATH
    ])
    # PKCS12 file required for riemann due to JVM
    # While we don't really want the private key in there, not having it
    # causes failures
    # The password is also a bit pointless here since it's in the same place
    # as a readable copy of the certificate and if this path can be written to
    # maliciously then all is lost already.
    pkcs12_path = os.path.join(SSL_CERTS_TARGET_DIR, INTERNAL_PKCS12_FILENAME)
    sudo([
        'openssl', 'pkcs12', '-export',
        '-out', pkcs12_path,
        '-in', INTERNAL_CA_CERT_PATH,
        '-inkey', INTERNAL_CA_KEY_PATH,
        '-password', 'pass:cloudify',
    ])

def _format_ips(ips):
    altnames = set(ips)

    # Ensure we trust localhost
    altnames.add('127.0.0.1')
    altnames.add('localhost')

    subject_altdns = [
        'DNS:{name}'.format(name=name)
        for name in altnames
    ]
    subject_altips = []
    for name in altnames:
        ip_address = False
        try:
            socket.inet_pton(socket.AF_INET, name)
            ip_address = True
        except socket.error:
            # Not IPv4
            pass
        try:
            socket.inet_pton(socket.AF_INET6, name)
            ip_address = True
        except socket.error:
            # Not IPv6
            pass
        if ip_address:
            subject_altips.append('IP:{name}'.format(name=name))

    cert_metadata = ','.join([
        ','.join(subject_altdns),
        ','.join(subject_altips),
    ])
    return cert_metadata

def store_cert_metadata(internal_rest_host, networks=None,
                        filename=CERT_METADATA_FILE_PATH):
    metadata = load_cert_metadata()
    metadata['internal_rest_host'] = internal_rest_host
    if networks is not None:
        metadata['networks'] = networks
    contents = json.dumps(metadata)
    sudo_write_to_file(contents, filename)
    chown(CLOUDIFY_USER, CLOUDIFY_GROUP, filename)

def load_cert_metadata(filename=CERT_METADATA_FILE_PATH):
    try:
        with open(filename) as f:
            return json.load(f)
    except IOError:
        return {}

CSR_CONFIG_TEMPLATE = """
[req]
distinguished_name = req_distinguished_name
req_extensions = server_req_extensions
[ server_req_extensions ]
subjectAltName={metadata}
[ req_distinguished_name ]
commonName = _common_name # ignored, _default is used instead
commonName_default = {cn}
"""

@contextmanager
def _csr_config(cn, metadata):
    """Prepare a config file for creating a ssl CSR.

    :param cn: the subject commonName
    :param metadata: string to use as the subjectAltName, should be formatted
                     like "IP:1.2.3.4,DNS:www.com"
    """
    with tempfile.NamedTemporaryFile(delete=False) as conf_file:
        conf_file.write(CSR_CONFIG_TEMPLATE.format(cn=cn, metadata=metadata))

    try:
        yield conf_file.name
    finally:
        remove(conf_file.name)

def generate_ssl_certificate(ips, cn, cert_path, key_path,
                             sign_cert=INTERNAL_CA_CERT_PATH,
                             sign_key=INTERNAL_CA_KEY_PATH):
    """Generate a public SSL certificate and a private SSL key

    :param ips: the ips (or names) to be used for subjectAltNames
    :type ips: List[str]
    :param cn: the subject commonName for the new certificate
    :type cn: str
    :param cert_path: path to save the new certificate to
    :type cert_path: str
    :param key_path: path to save the key for the new certificate to
    :type key_path: str
    :param sign_cert: path to the signing cert (internal CA by default)
    :type sign_cert: str
    :param sign_key: path to the signing cert's key (internal CA by default)
    :type sign_key: str
    :return: The path to the cert and key files on the manager
    """
    # Remove duplicates from ips
    subject_altnames = _format_ips(ips)
    ctx.logger.info(
        'Generating SSL certificate {0} and key {1} with subjectAltNames: {2}'
        .format(cert_path, key_path, subject_altnames)
    )

    csr_path = '{0}.csr'.format(cert_path)

    with _csr_config(cn, subject_altnames) as conf_path:
        sudo([
            'openssl', 'req',
            '-newkey', 'rsa:2048',
            '-nodes',
            '-batch',
            '-config', conf_path,
            '-out', csr_path,
            '-keyout', key_path,
        ])
        x509_command = [
            'openssl', 'x509',
            '-days', '3650',
            '-req', '-in', csr_path,
            '-extfile', conf_path,
            '-out', cert_path,
            '-extensions', 'server_req_extensions',
        ]
        if sign_cert and sign_key:
            x509_command += [
                '-CA', sign_cert,
                '-CAkey', sign_key,
                '-CAcreateserial'
            ]
        else:
            x509_command += [
                '-signkey', key_path
            ]
        sudo(x509_command)
        remove(csr_path)

    ctx.logger.info('Generated SSL certificate: {0} and key: {1}'.format(
        cert_path, key_path
    ))
    return cert_path, key_path

def generate_internal_ssl_cert(ips, name):
    return generate_ssl_certificate(
        ips,
        name,
        INTERNAL_CERT_PATH,
        INTERNAL_KEY_PATH
    )

def write_to_tempfile(contents):
    fd, file_path = tempfile.mkstemp()
    os.write(fd, contents)
    os.close(fd)
    return file_path

def install_python_package(source, venv='', constraints_file=None):
    cmdline = []
    if venv:
        cmdline.append('{0}/bin/pip'.format(venv))
    else:
        cmdline.append('pip')

    cmdline.extend(['install', source, '--upgrade'])

    log_message = 'Installing {0}'.format(source)

    if venv:
        log_message += ' in virtualenv {0}'.format(venv)
    if constraints_file:
        cmdline.extend(['-c', constraints_file])
        log_message += ' using constraints file {0}'.format(constraints_file)

    ctx.logger.info(log_message)

    sudo(cmdline)

def get_file_content(file_path):
    """
    Using sudo to copy the file to a temp folder, to allow access to the file
    even if it's in a restricted directory (e.g. '/root').
    Then, chmoding the file so the current user can read it.
    :param file_path: the path to the file
    :return: the content of the file
    """
    try:
        temp_dir = tempfile.mkdtemp()
        copy(file_path, temp_dir)
        filename = os.path.basename(file_path)
        new_file_copy = os.path.join(temp_dir, filename)
        chmod('644', new_file_copy)
        with open(new_file_copy) as the_file:
            return the_file.read()
    finally:
        if os.path.exists(temp_dir):
            remove(temp_dir)

def curl_download_with_retries(source, destination):
    """Download file using the curl command.

    :param source: Source URL for the file to download
    :typ source: str
    :param destination:
        Path to the directory where the file should be downloaded
    :type destination: str

    """
    curl_cmd = [
        'curl',
        '--retry', '10',
        '--fail',
        '--silent',
        '--show-error',
        '--location', source,
        '--create-dir',
        '--output', destination,
    ]
    ctx.logger.debug('curling: {0}'.format(' '.join(curl_cmd)))
    run(curl_cmd)

def download_file(url, destination=''):
    if not destination:
        fd, destination = tempfile.mkstemp()
        os.remove(destination)
        os.close(fd)

    if not os.path.isfile(destination):
        ctx.logger.info('Downloading {0} to {1}...'.format(
            url, destination))
        try:
            final_url = urllib.urlopen(url).geturl()
            if final_url != url:
                ctx.logger.debug('Redirected to {0}'.format(final_url))
            f = urllib.URLopener()
            # TODO: try except with @retry
            f.retrieve(final_url, destination)
        except Exception:
            curl_download_with_retries(url, destination)
    else:
        ctx.logger.debug('File {0} already exists...'.format(destination))
    return destination

def get_file_name_from_url(url):
    try:
        return url.split('/')[-1]
    except Exception:
        # in case of irregular url. precaution.
        # note that urlparse is deprecated in Python 3
        from urlparse import urlparse
        disassembled = urlparse(url)
        return os.path.basename(disassembled.path)

def download_cloudify_resource(url, service_name, destination=None):
    """Downloads a resource and saves it as a cloudify resource.

    The resource will be saved under the appropriate service resource path and
    will be used in case of operation execution failure after the resource has
    already been downloaded.
    """
    if destination:
        source_res_path, _ = resource_factory.create(url,
                                                     destination,
                                                     service_name,
                                                     source_resource=True,
                                                     render=False)
        copy(source_res_path, destination)
    else:
        res_name = os.path.basename(url)
        source_res_path, _ = resource_factory.create(url, res_name,
                                                     service_name,
                                                     source_resource=True,
                                                     render=False)
    return source_res_path

def deploy_blueprint_resource(source, destination, service_name,
                              user_resource=False, render=True, load_ctx=True):
    """
    Downloads a resource from the blueprint to a destination. This expands
    `download-resource` as a `sudo mv` is required after having downloaded
    the resource.
    :param source: Resource source.
    :param destination: Resource destination.
    :param service_name: The service name that requires the resource.
    :param user_resource: Set to true for resources that should potentially
    remain identical upon upgrade such as custom security configuration files.
    :param render: Set to false if resource does not require rendering.
    :param load_ctx: Set to false if node props should not be loaded.
    NOTE: This is normally used when using this function from a preconfigure
    script where node properties are not available.
    """
    ctx.logger.info('Deploying blueprint resource {0} to {1}'.format(
        source, destination))
    resource_file, dest = resource_factory.create(source,
                                                  destination,
                                                  service_name,
                                                  user_resource=user_resource,
                                                  render=render,
                                                  load_ctx=load_ctx)
    copy(resource_file, dest)

def copy_notice(service):
    """Deploys a notice file to /opt/SERVICENAME_NOTICE.txt"""
    destn = os.path.join('/opt', '{0}_NOTICE.txt'.format(service))
    source = 'components/{0}/NOTICE.txt'.format(service)
    resource_file, dest = resource_factory.create(source, destn, service,
                                                  render=False)
    copy(resource_file, dest)

def remove_notice(service):
    """Remove the notice file /opt/SERVICENAME_NOTICE.txt"""
    path = os.path.join('/opt', '{0}_NOTICE.txt'.format(service))
    remove(path)

def is_port_open(port, host='localhost'):
    """Try to connect to (host, port), return if the port was listening."""
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    return sock.connect_ex((host, port)) == 0

def wait_for_port(port, host='localhost'):
    """Helper function to wait for a port to open before continuing"""
    counter = 1

    ctx.logger.info('Waiting for {0}:{1} to become available...'.format(
        host, port))

    for tries in range(24):
        if not is_port_open(port, host=host):
            ctx.logger.info(
                '{0}:{1} is not available yet, retrying... '
                '({2}/24)'.format(host, port, counter))
            time.sleep(2)
            counter += 1
            continue
        ctx.logger.info('{0}:{1} is open!'.format(host, port))
        return
    ctx.abort_operation('Failed to connect to {0}:{1}...'.format(host, port))

def yum_install(source, service_name):
    """Installs a package using yum.

    yum supports installing from URL, path and the default yum repo
    configured within your image.
    you can specify one of the following:
    [yum install -y] mylocalfile.rpm
    [yum install -y] mypackagename

    If the source is a package name, it will check whether it is already
    installed. If it is, it will do nothing. It not, it will install it.

    If the source is a url to an rpm and the file doesn't already exist
    in a predesignated archives file path (${CLOUDIFY_SOURCES_PATH}/),
    it will download it. It will then use that file to check if the
    package is already installed. If it is, it will do nothing. If not,
    it will install it.

    NOTE: This will currently not take into considerations situations
    in which a file was partially downloaded. If a file is partially
    downloaded, a re-download will not take place and rather an
    installation will be attempted, which will obviously fail since
    the rpm file is incomplete.
    ALSO NOTE: you cannot provide `yum_install` with a space
    separated array of packages as you can with `yum install`. You must
    provide one package per invocation.
    """
    # source is a url
    if source.startswith(('http', 'https', 'ftp')):
        filename = get_file_name_from_url(source)
        source_name, ext = os.path.splitext(filename)
    # source is just the name of the file
    elif source.endswith('.rpm'):
        source_name, ext = os.path.splitext(source)
    # source is the name of a yum-repo based package name
    else:
        source_name, ext = source, ''
    source_path = source_name

    if ext.endswith('.rpm'):
        source_path = download_cloudify_resource(source, service_name)

        rpm_handler = RpmPackageHandler(source_path)
        ctx.logger.info(
            'Checking whether {0} is already installed...'.format(
                source_path))
        if rpm_handler.is_rpm_installed():
            ctx.logger.debug('Package {0} is already installed.'.format(
                source))
            return

        # removes any existing versions of the package that do not match
        # the provided package source version
        rpm_handler.remove_existing_rpm_package()
    else:
        installed = run(['yum', '-q', 'list', 'installed', source_path],
                        ignore_failures=True)
        if installed.returncode == 0:
            ctx.logger.debug('Package {0} is already installed.'.format(
                source))
            return
    ctx.logger.info('yum installing {0}...'.format(source_path))
    sudo(['yum', 'install', '-y', '--disablerepo=*', source_path])

def yum_remove(package, ignore_failures=False):
    ctx.logger.info('yum removing {0}...'.format(package))
    try:
        sudo(['yum', 'remove', '-y', package])
    except BaseException:
        msg = 'Package `{0}` may not been removed successfully!'
        if not ignore_failures:
            ctx.logger.error(msg)
            raise
        ctx.logger.warn(msg)

def get_filepath_from_pkg_name(filename, raise_if_not_found=False):
    local_filepath_list = \
        [fn for fn in glob.glob(os.path.join(CLOUDIFY_SOURCES_PATH, filename))
         if not os.path.basename(fn).startswith(SINGLE_TAR_PREFIX)]
    if not local_filepath_list:
        if raise_if_not_found:
            raise IOError('Local resource not found: {0}'.format(filename))
        ctx.abort_operation("File: {0} does not exist in sources path: {1}".
                            format(filename, CLOUDIFY_SOURCES_PATH))
    if len(local_filepath_list) > 1:
        ctx.abort_operation("More than one file: {0} found in sources path:"
                            " {1}".format(filename, CLOUDIFY_SOURCES_PATH))
    local_filepath = ''.join(local_filepath_list[0])
    ctx.logger.debug("File exists in sources path: {0}".format(local_filepath))

    return local_filepath

class RpmPackageHandler(object):

    def __init__(self, source_path):
        self.source_path = source_path

    def remove_existing_rpm_package(self):
        """Removes any version that satisfies the package name of the given
        source path.
        """
        package_name = self.get_rpm_package_name()
        if self.is_package_installed(package_name):
            ctx.logger.debug(
                'Removing existing package sources for package '
                'with name: {0}'.format(package_name))
            sudo(['rpm', '--noscripts', '-e', package_name])

    @staticmethod
    def is_package_installed(name):
        installed = run(['rpm', '-q', name], ignore_failures=True)
        if installed.returncode == 0:
            return True
        return False

    def is_rpm_installed(self):
        """Returns true if provided rpm is already installed.
        """
        src_query = run(['rpm', '-qp', self.source_path])
        source_name = src_query.aggr_stdout.rstrip('\n\r')

        return self.is_package_installed(source_name)

    def get_rpm_package_name(self):
        """Returns the package name according to the info provided in the
        source file.
        """
        split_index = ' : '
        package_details = {}
        package_details_query = run(['rpm', '-qpi', self.source_path])
        rows = package_details_query.aggr_stdout.split('\n')
        # split raw data according to the ' : ' index
        for row in rows:
            if split_index in row:
                first_columb_index = row.index(split_index)
                key = row[:first_columb_index].strip()
                value = row[first_columb_index + len(split_index):].strip()
                package_details[key] = value
        return package_details['Name']

class SystemD(object):

    def systemctl(self, action, service='', retries=0, ignore_failure=False):
        systemctl_cmd = ['systemctl', action]
        if service:
            systemctl_cmd.append(service)
        return sudo(systemctl_cmd, retries=retries,
                    ignore_failures=ignore_failure)

    def configure(self, service_name, render=True, tmpfiles=False):
        """This configures systemd for a specific service.

        It requires that two files are present for each service one containing
        the environment variables and one contains the systemd config.
        All env files will be named "cloudify-SERVICENAME".
        All systemd config files will be named "cloudify-SERVICENAME.service".

        If `tmpfiles` is True, the directory
        `components/{service_name}/tmpfiles.d` must exist and contain the file
        `cloudify-{service_name}.conf`. This will be deployed to
        `/usr/lib/tmpfiles.d`.
        """
        sid = 'cloudify-{0}'.format(service_name)
        env_dst = "/etc/sysconfig/{0}".format(sid)
        srv_dst = "/usr/lib/systemd/system/{0}.service".format(sid)
        env_src = "components/{0}/config/{1}".format(service_name, sid)
        srv_src = "components/{0}/config/{1}.service".format(service_name, sid)

        ctx.logger.debug('Deploying systemd EnvironmentFile...')
        deploy_blueprint_resource(env_src, env_dst, service_name,
                                  render=render)
        ctx.logger.debug('Deploying systemd .service file...')
        deploy_blueprint_resource(srv_src, srv_dst, service_name,
                                  render=render)
        ctx.logger.debug('Enabling systemd .service...')
        self.systemctl('enable', '{0}.service'.format(sid))

        if tmpfiles:
            tmp_dst = "/usr/lib/tmpfiles.d/{0}.conf".format(sid)
            tmp_src = "components/{0}/config/tmpfiles.d/{1}.conf".format(
                service_name, sid)

            ctx.logger.debug('Deploying tmpfiles.d file...')
            deploy_blueprint_resource(
                tmp_src, tmp_dst, service_name, render=render)
            sudo(['systemd-tmpfiles', '--create'])

        self.systemctl('daemon-reload')

    def remove(self, service_name):
        """Stop and disable the service, and then delete its data
        """
        self.stop(service_name, ignore_failure=True)
        self.disable(service_name, ignore_failure=True)
        remove(self.get_service_file_path(service_name))
        remove(self.get_vars_file_path(service_name))

    @staticmethod
    def get_vars_file_path(service_name):
        """Returns the path to a systemd environment variables file
        for a given service_name. (e.g. /etc/sysconfig/cloudify-rabbitmq)
        """
        sid = 'cloudify-{0}'.format(service_name)
        return '/etc/sysconfig/{0}'.format(sid)

    @staticmethod
    def get_service_file_path(service_name):
        """Returns the path to a systemd service file
        for a given service_name.
        (e.g. /usr/lib/systemd/system/cloudify-rabbitmq.service)
        """
        sid = 'cloudify-{0}'.format(service_name)
        return "/usr/lib/systemd/system/{0}.service".format(sid)

    def enable(self, service_name, retries=0, append_prefix=True):
        full_service_name = self._get_full_service_name(service_name,
                                                        append_prefix)
        ctx.logger.debug('Enabling systemd service {0}...'.format(
            full_service_name))
        self.systemctl('enable', full_service_name, retries)

    def disable(self, service_name, retries=0, append_prefix=True,
                ignore_failure=False):
        full_service_name = self._get_full_service_name(service_name,
                                                        append_prefix)
        ctx.logger.debug('Disabling systemd service {0}...'.format(
            full_service_name))
        self.systemctl('disable', full_service_name, retries,
                       ignore_failure=ignore_failure)

    def start(self, service_name, retries=0, append_prefix=True):
        full_service_name = self._get_full_service_name(service_name,
                                                        append_prefix)
        ctx.logger.debug('Starting systemd service {0}...'.format(
            full_service_name))
        self.systemctl('start', full_service_name, retries)

    def stop(self, service_name, retries=0, append_prefix=True,
             ignore_failure=False):
        full_service_name = self._get_full_service_name(service_name,
                                                        append_prefix)
        ctx.logger.debug('Stopping systemd service {0}...'.format(
            full_service_name))
        self.systemctl('stop', full_service_name, retries,
                       ignore_failure=ignore_failure)

    def restart(self,
                service_name,
                retries=0,
                ignore_failure=False,
                append_prefix=True):
        full_service_name = self._get_full_service_name(service_name,
                                                        append_prefix)
        self.systemctl('restart', full_service_name, retries,
                       ignore_failure=ignore_failure)

    def is_alive(self, service_name, append_prefix=True):
        service_name = self._get_full_service_name(service_name, append_prefix)
        result = self.systemctl('status', service_name, ignore_failure=True)
        return result.returncode == 0

    def verify_alive(self, service_name, append_prefix=True):
        if self.is_alive(service_name, append_prefix):
            ctx.logger.debug('{0} is running'.format(service_name))
        else:
            ctx.abort_operation('{0} is not running'.format(service_name))

    @staticmethod
    def _get_full_service_name(service_name, append_prefix):
        if append_prefix:
            return 'cloudify-{0}'.format(service_name)
        return service_name

systemd = SystemD()

def replace_in_file(this, with_this, in_here):
    """Replaces all occurences of the regex in all matches
    from a file with a specific value.
    """
    ctx.logger.debug('Replacing {0} with {1} in {2}...'.format(
        this, with_this, in_here))
    with open(in_here) as f:
        content = f.read()
    new_content = re.sub(this, with_this, content)
    fd, temp_file = tempfile.mkstemp()
    os.close(fd)
    with open(temp_file, 'w') as f:
        f.write(new_content)
    move(temp_file, in_here)

def get_selinux_state():
    return subprocess.check_output('getenforce').rstrip('\n\r')

def set_selinux_permissive():
    """This sets SELinux to permissive mode both for the current session
    and systemwide.
    """
    ctx.logger.info('Checking whether SELinux in enforced...')
    if 'Enforcing' == get_selinux_state():
        ctx.logger.debug('SELinux is enforcing, setting permissive state...')
        sudo(['setenforce', 'permissive'])
        replace_in_file(
            'SELINUX=enforcing',
            'SELINUX=permissive',
            '/etc/selinux/config')
    else:
        ctx.logger.debug('SELinux is not enforced.')

def get_rabbitmq_endpoint_ip(endpoint=None):
    """Gets the rabbitmq endpoint IP, using the manager IP if the node
    property is blank.
    """
    if endpoint:
        return endpoint
    return ctx.instance.host_ip

def create_service_user(user, group, home):
    """Creates a user.

    It will not create the home dir for it and assume that it already exists.
    This user will only be created if it didn't already exist.
    """
    ctx.logger.info('Checking whether user {0} exists...'.format(user))
    try:
        pwd.getpwnam(user)
        ctx.logger.debug('User {0} already exists...'.format(user))
    except KeyError:
        ctx.logger.info('Creating group {group} if it does not exist'.format(
            group=group,
        ))
        # --force in groupadd causes it to return true if the group exists.
        # Other behaviour changes don't affect this basic use of the command.
        sudo(['groupadd', '--force', group])

        ctx.logger.info('Creating user {0}, home: {1}...'.format(
            user, home))
        sudo([
            'useradd',
            '--shell', '/sbin/nologin',
            '--home-dir', home, '--no-create-home',
            '--system',
            '--no-user-group',
            '--gid', group,
            user,
        ])

def logrotate(service):
    """Deploys a logrotate config for a service.

    Note that this is not idempotent in the sense that if a logrotate
    file is already copied to /etc/logrotate.d, it will copy it again
    and override it. This is done as such so that if a service deploys
    its own logrotate configuration, we will override it.
    """
    if not os.path.isfile('/etc/cron.hourly/logrotate'):
        ctx.logger.info('Deploying logrotate hourly cron job...')
        move('/etc/cron.daily/logrotate', '/etc/cron.hourly/logrotate')

    ctx.logger.debug('Deploying logrotate config...')
    config_file_source = 'components/{0}/config/logrotate'.format(service)
    logrotated_path = '/etc/logrotate.d'
    config_file_destination = os.path.join(logrotated_path, service)
    if not os.path.isdir(logrotated_path):
        os.mkdir(logrotated_path)
        chown('root', 'root', logrotated_path)
    deploy_blueprint_resource(config_file_source,
                              config_file_destination,
                              service)
    chmod('644', config_file_destination)
    chown('root', 'root', config_file_destination)

def remove_logrotate(service_name):
    ctx.logger.debug('Removing logrotate config...')
    logrotated_path = '/etc/logrotate.d'
    config_file_destination = os.path.join(logrotated_path, service_name)
    remove(config_file_destination)

def chmod(mode, path, recursive=False):
    ctx.logger.debug('chmoding {0}: {1}'.format(path, mode))
    command = ['chmod']
    if recursive:
        command.append('-R')
    command += [mode, path]
    sudo(command)

def chown(user, group, path):
    ctx.logger.debug('chowning {0} by {1}:{2}...'.format(
        path, user, group))
    sudo(['chown', '-R', '{0}:{1}'.format(user, group), path])

def ln(source, target, params=None):
    ctx.logger.debug('Linking {0} to {1} with params {2}'.format(
        source, target, params))
    command = ['ln']
    if params:
        command.append(params)
    command.append(source)
    command.append(target)
    if '*' in source or '*' in target:
        sudo(command, globx=True)
    else:
        sudo(command)

def clean_var_log_dir(service):
    pass

def untar(source,
          destination=None,
          skip_old_files=False,
          unique_tmp_dir=False):
    # TODO: use tarfile instead
    if not destination:
        destination = tempfile.mkdtemp() if unique_tmp_dir else '/tmp'
    ctx.logger.debug('Extracting {0} to {1}...'.format(
        source, destination))
    tar_command = ['tar', '-xzvf', source, '-C', destination, '--strip=1']
    if skip_old_files:
        tar_command.append('--skip-old-files')
    sudo(tar_command)
    return destination

def validate_md5_checksum(resource_path, md5_checksum_file_path):
    ctx.logger.info('Validating md5 checksum for {0}'.format(
        resource_path))
    with open(md5_checksum_file_path) as checksum_file:
        original_md5 = checksum_file.read().rstrip('\n\r').split()[0]

    with open(resource_path) as file_to_check:
        data = file_to_check.read()
        # pipe contents of the file through
        md5_returned = hashlib.md5(data).hexdigest()

    if original_md5 == md5_returned:
        return True
    else:
        ctx.logger.error(
            'md5 checksum validation failed! Original checksum: {0} '
            'Calculated checksum: {1}.'.format(original_md5, md5_returned))
        return False

def write_to_json_file(content, file_path):
    tmp_file = tempfile.NamedTemporaryFile(delete=False)
    mkdir(os.path.dirname(os.path.abspath(file_path)))
    with open(tmp_file.name, 'w') as f:
        f.write(json.dumps(content))
    move(tmp_file.name, file_path)

def repetitive(condition_func,
               timeout=15,
               interval=3,
               timeout_msg='timed out',
               *args,
               **kwargs):

    deadline = time.time() + timeout
    while True:
        if time.time() > deadline:
            ctx.abort_operation(timeout_msg)
        if condition_func(*args, **kwargs):
            return
        time.sleep(interval)

class BlueprintResourceFactory(object):

    BASE_RESOURCES_PATH = '/opt/cloudify'
    RESOURCES_DIR_NAME = 'resources'

    def create(self, source, destination, service_name, user_resource=False,
               source_resource=False, render=True, load_ctx=True):
        """A Factory used to create a local copy of a resource upon deployment.
        This copy allows to later reuse the resource for upgrade/rollback
        purposes.

        :param source: The resource source url
        :param destination: Resource destination path
        :param service_name: used to retrieve node properties to be rendered
        into a resource template
        :param user_resource: Resources that should potentially remain
        identical upon upgrade such as custom security configuration files.
        These file will be reused provided the ctx node property:
        use_existing_on_upgrade set to True.
        :param source_resource: Source resources are source packages that
        should be downloaded with no rendering and only archived.
        :param render: Set to false if resource does not require rendering.
        :param load_ctx: Set to false if properties are not available in the
        context of the script.
        :return: The local resource file path and destination.
        """
        resource_name = os.path.basename(destination)

        # The local path is decided according to whether we are in upgrade
        local_resource_path = self._get_local_file_path(service_name,
                                                        resource_name)

        if not os.path.isfile(local_resource_path):
            mkdir(os.path.dirname(local_resource_path))
            if user_resource:
                self._download_user_resource(source,
                                             local_resource_path,
                                             render=render,
                                             load_ctx=load_ctx)
            elif source_resource:
                self._download_source_resource(source,
                                               local_resource_path)
            elif render:
                self._download_resource_and_render(source,
                                                   local_resource_path,
                                                   load_ctx)
            else:
                self._download_resource(source, local_resource_path)
        return local_resource_path, destination

    @staticmethod
    def local_resource_exists(filename):
        try:
            get_filepath_from_pkg_name(filename, raise_if_not_found=True)
            return True
        except IOError:
            return False

    def _download_user_resource(self, source, dest,
                                render=True, load_ctx=True):
        if not os.path.isfile(dest):
            if render:
                self._download_resource_and_render(source, dest, load_ctx)
            else:
                self._download_resource(source, dest)

    @staticmethod
    def _download_resource(source, dest):
        resource_name = os.path.basename(dest)
        ctx.logger.info('Downloading resource {0} to {1}'.format(
            resource_name, dest))
        tmp_file = ctx.download_resource(source)
        move(tmp_file, dest)

    def _download_resource_and_render(self, source, dest, load_ctx):
        resource_name = os.path.basename(dest)
        ctx.logger.debug('Downloading resource {0} to {1}'.format(
            resource_name, dest))
        if load_ctx:
            params = self._get_node_props()
            tmp_file = ctx.download_resource_and_render(source, '', params)
        else:
            # rendering will be possible only for runtime properties
            tmp_file = ctx.download_resource_and_render(source, '')
        move(tmp_file, dest)

    @staticmethod
    def _download_source_resource(source, local_resource_path):
        is_url = source.startswith(('http://', 'https://', 'ftp://',
                                    'file://'))
        filename = get_file_name_from_url(source) if is_url else source
        is_manager_package = filename.startswith(SINGLE_TAR_PREFIX)
        if is_manager_package or is_url:
            local_filepath = os.path.join(CLOUDIFY_SOURCES_PATH, filename)
        else:
            local_filepath = get_filepath_from_pkg_name(filename)

        if is_url:
            if not os.path.isfile(local_filepath):
                tmp_path = download_file(source)
            elif os.path.isfile(local_filepath) and not is_manager_package:
                remove(local_filepath)
                tmp_path = download_file(source)
            else:
                tmp_path = local_filepath
        # source is just the name of the file, to be retrieved from
        # the manager resources package
        else:
            tmp_path = local_filepath
        ctx.logger.debug('Saving {0} under {1}'.format(
            tmp_path, local_resource_path))
        move(tmp_path, local_resource_path)

    @staticmethod
    def _get_node_props():
        node_props = ctx.node.properties.get_all()
        return {'node': {'properties': node_props}}

    def _is_cloudify_pkg(self, filename):
        """Cloudify packages start with 'cloudify' or include '-agent_'

        and end with one of the suffix '.rpm', '.tar.gz', '.tgz', '.exe'.
        The function who calls this function wish to gel all cloudify
        packages except of single tar package.
        """

        if (filename.startswith('cloudify') or
            filename.find('-agent_') != -1) \
                and not filename.startswith(SINGLE_TAR_PREFIX) \
                and filename.endswith(('.rpm', '.tar.gz', '.tgz', '.exe')):
            return True
        else:
            return False

    def _get_local_file_path(self, service_name, resource_name):
        base_service_res_dir = self.get_resources_dir(service_name)

        if self._is_cloudify_pkg(resource_name):
            local_filepath = get_filepath_from_pkg_name(resource_name)
            dest_file_path = os.path.join(base_service_res_dir,
                                          os.path.basename(local_filepath))
        else:
            dest_file_path = os.path.join(base_service_res_dir, resource_name)

        return dest_file_path

    def get_resources_dir(self, service_name):
        return os.path.join(self.BASE_RESOURCES_PATH,
                            service_name,
                            self.RESOURCES_DIR_NAME)

resource_factory = BlueprintResourceFactory()

def start_service(service_name, append_prefix=True):
    systemd.start(service_name, append_prefix=append_prefix)

def http_request(url,
                 data=None,
                 method='PUT',
                 headers=None,
                 timeout=None,
                 should_fail=False):
    headers = headers or {}
    request = urllib2.Request(url, data=data, headers=headers)
    request.get_method = lambda: method
    try:
        if timeout:
            return urllib2.urlopen(request, timeout=timeout)
        return urllib2.urlopen(request)
    except urllib2.URLError as e:
        if not should_fail:
            ctx.logger.error('Failed to {0} {1} (reason: {2})'.format(
                method, url, e.reason))

def get_auth_headers(username, password):
    return {
        'Authorization': 'Basic ' + base64.b64encode('{0}:{1}'.format(
            username, password)
        ),
        'tenant': 'default_tenant'
    }

@retry((IOError, ValueError))
def check_http_response(url, predicate=None, **request_kwargs):
    req = urllib2.Request(url, **request_kwargs)
    try:
        response = urllib2.urlopen(req)
    except urllib2.HTTPError as e:
        # HTTPError can also be used as a non-200 response. Pass this
        # through to the predicate function, so it can decide if a
        # non-200 response is fine or not.
        response = e

    if predicate is not None and not predicate(response):
        raise ValueError('Failed opening url {0} ({1})'.format(url, response))
    return response

def verify_service_http(service_name, url, *args, **kwargs):
    try:
        return check_http_response(url, *args, **kwargs)
    except (IOError, ValueError) as e:
        ctx.abort_operation('{0} error: {1}: {2}'.format(service_name, url, e))

def remove_component(runtime_props):
    service_name = runtime_props['service_name']
    ctx.logger.info('Uninstalling {0}'.format(service_name))

    systemd.remove(service_name)
    remove_notice(service_name)
    remove_logrotate(service_name)

    packages_to_remove = runtime_props.get('packages_to_remove', [])
    for package in packages_to_remove:
        yum_remove(package, ignore_failures=True)

    files_to_remove = runtime_props.get('files_to_remove', [])
    for f in files_to_remove:
        remove(f)

    user = runtime_props.get('service_user')
    if user:
        sudo(['userdel', '--force', user], ignore_failures=True)

    group = runtime_props.get('service_group')
    if group:
        sudo(['groupdel', group], ignore_failures=True)

def extend_runtime_properties_list(runtime_props, key_name, new_list):
    """Extend a list in the runtime properties
    list.extend doesn't call __setitem__, so we need to do it explicitly
    """
    list_to_extend = runtime_props.get(key_name, [])
    list_to_extend.extend(new_list)
    runtime_props[key_name] = list_to_extend

def set_service_as_cloudify_service(runtime_props):
    """Set the cloudify user and group as the user/group of a service"""

    runtime_props['service_user'] = CLOUDIFY_USER
    runtime_props['service_group'] = CLOUDIFY_GROUP

def delete_cluster_component(component):
    """If the given cluster component exists, teardown it."""
    if os.path.exists(CLUSTER_DELETE_SCRIPT):
        sudo(['/usr/bin/env', 'python', CLUSTER_DELETE_SCRIPT,
              '--component', component])

def get_exec_tempdir():
    return os.environ.get(CFY_EXEC_TEMPDIR_ENVVAR) or tempfile.gettempdir()


from IPython.display import display, HTML, clear_output
import matplotlib.pyplot as plt
import numpy as np

from utils import generic_dataloader

encoder_classifier_evaluations = None
generator_discriminator_evaluations = None

def display_table(tabldict):
    content = "

"+"".join([f"{key}	" for key in tabldict])+"
"
    for i in range(len(tabldict[next(iter(tabldict))])):
        content += "
"
        for key in tabldict:
            form = f"{tabldict[key][i]:.4f}" if type(tabldict[key][i]) == float else str(tabldict[key][i])
            content = content + ""+form+"	"
        content += "
"
    content += "
"

    display(HTML(content))

def encoder_classifier_notebook_evaluator_evaluate_and_print(self):
    svhn_train_loss, svhn_train_accuracy, *_ = self.evaluate(self.svhn_trainsplit_loader)
    _, svhn_test_accuracy, *_ = self.evaluate(self.svhn_testsplit_loader)

    _, mnist_train_accuracy, _, mnist_train_above_threshold_accuracy = self.evaluate(self.mnist_trainsplit_loader)

    results = {"Epoch": self.epoch,
               "SVHN Train Split Loss": svhn_train_loss,
               "SVHN Train Split Classification Acc. (%)": svhn_train_accuracy,
               "SVHN Test Split Classification Acc. (%)": svhn_test_accuracy,
               "MNIST Train Split Above Threshold (%)": mnist_train_above_threshold_accuracy}

    for key in results:
        encoder_classifier_evaluations[key].append(results[key])

    # clear output before printing same table with more rows
    clear_output(wait=True)
    display_table(encoder_classifier_evaluations)

def encoder_classifier_notebook_evaluator_reset():
    global encoder_classifier_evaluations
    encoder_classifier_evaluations = {"Epoch": [],
                                      "SVHN Train Split Loss": [],
                                      "SVHN Train Split Classification Acc. (%)": [],
                                      "SVHN Test Split Classification Acc. (%)": [],
                                      "MNIST Train Split Above Threshold (%)": []}

def generator_discriminator_notebook_evaluator_reset():
    global generator_discriminator_evaluations
    generator_discriminator_evaluations = {"Epoch": [],
                                           "Discriminator Loss": [],
                                           "Generator Deception Loss": [],
                                           "Generator Reconstruction Loss": [],
                                           "Generator Loss": [],
                                           "MNIST High Conf. Sample Count": []}

def generator_discriminator_notebook_evaluator_evaluate_and_print(self, mnist_hc_loader, pseudolabels):

    svhn_train_result_dict, svhn_train_total = self.evaluate(self.svhn_trainsplit_loader, 0)
    mnist_hc_result_dict, mnist_hc_total = self.evaluate(mnist_hc_loader, 1, pseudolabels=pseudolabels)
    training_result_dict = {key: (svhn_train_result_dict[key] * svhn_train_total +
                                  mnist_hc_result_dict[key] * mnist_hc_total) / (svhn_train_total + mnist_hc_total)
                            for (key, _) in svhn_train_result_dict.items()}
    results = {"Epoch": self.epoch,
               "Discriminator Loss": training_result_dict["discriminator_mnist_loss"].item() +
                                     training_result_dict["discriminator_svhn_loss"].item(),
               "Generator Deception Loss": training_result_dict["generator_deception_loss"].item(),
               "Generator Reconstruction Loss": training_result_dict["generator_reconstruction_loss"].item(),
               "Generator Loss": training_result_dict["generator_loss"].item(),
               "MNIST High Conf. Sample Count": mnist_hc_total}

    for key in results:
        generator_discriminator_evaluations[key].append(results[key])

    clear_output(wait=True)
    img = self.reconstruct_images_from_dataset(self.svhn_trainsplit_loader_notshuffled, 1)
    npimg = img.cpu().numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.axis('off')
    plt.show()

    display_table(generator_discriminator_evaluations)

def generator_discriminator_notebook_evaluator_evaluate_goals_and_print(self):
    mnist_hc_dataset, pseudolabels = self.get_high_confidence_mnist_dataset_with_pseudolabels()
    mnist_hc_loader = generic_dataloader(self.device, mnist_hc_dataset, shuffle=True, batch_size=self.batch_size)
    generator_discriminator_notebook_evaluator_reset()
    generator_discriminator_notebook_evaluator_evaluate_and_print(self, mnist_hc_loader, pseudolabels)
    mnist_test_result_dict, _ = self.evaluate(self.mnist_testsplit_loader, 1)
    results = {"MNIST Test Split Final Classification Accuracy":
                   [mnist_test_result_dict['encoder_classifier_accuracy']]}
    display_table(results)

def generator_discriminator_notebook_load_and_print_params(self, ckpt_file):
    self.load(ckpt_file)
    self.load(ckpt_file)
    print("Discriminator ADAM learning rate:", self.discriminator_mnist_optimizer.param_groups[0]['lr'])
    print("Discriminator ADAM betas:", self.discriminator_mnist_optimizer.param_groups[0]['betas'])
    print("Generator ADAM learning rate:", self.generator_optimizer.param_groups[0]['lr'])
    print("Generator ADAM betas:", self.generator_optimizer.param_groups[0]['betas'])
    print("Encoder-Classifier ADAM learning rate:", self.encoder_classifier_optimizer.param_groups[0]['lr'])
    print("Encoder-Classifier ADAM betas:", self.encoder_classifier_optimizer.param_groups[0]['betas'])
    print("DUPGAN Alpha:", self.dupgan_alpha)
    print("DUPGAN Beta:", self.dupgan_beta)
    print("epoch:", self.epoch)
    print("experiment_name:", self.experiment_name)

def encoder_classifier_notebook_load_and_print_params(self, ckpt_file):
    self.load(ckpt_file)
    print("ADAM learning rate:", self.optimizer.param_groups[0]['lr'])
    print("ADAM betas:", self.optimizer.param_groups[0]['betas'])
    print("epoch:", self.epoch)
    print("experiment_name:", self.experiment_name)


'''
-----------------------------------
Producto Integrador de Aprendizaje
Teoria de Automatas
Equipo III
-----------------------------------
'''

# Librerias
import os
from automata.tm.dtm import DTM

# Imprime los resultados de la Maquina de Turing
def imprimeResultado(cadena, turing_machine):
    tupla = turing_machine.validate_input(cadena, step = True)

    try:
        turing_machine.validate_input(cadena)
    except Exception as error_message:
        print("Transicion no definida en " + str(error_message)[-7:])
        print("[Cadena invalida]")
    else:
        for i in tupla:
            i_str = str(i)
            print(i_str[:-1].replace('TMTape', '') + ')')

        print("\n")
        print("[Cadena valida]")

'''
def imprimeResultado(cadena, turing_machine):
    tupla = turing_machine.validate_input(cadena, step = True)

    try:
        turing_machine.validate_input(cadena)
    except Exception as error_message:
        print("Transicion no definida en " + str(error_message)[-7:])
        print("[Cadena invalida]")
    else:
        for i in tupla:
            i_str = str(i)
            print(i_str[:-1].replace('TMTape', '') + ' \\vdash \\\\')

        print("\n")
        print("[Cadena valida]")
'''

# Programa principal
def main():
    turing_machine = DTM(
        states={'q0',
                'q1',
                'q2',
                'q3',
                'q4',
                'q5',
                'q6',
                'q7',
                'q8',
                'q9',
                'q10',
                'q11',
                'q12',
                'q13',
                'q14',},
        input_symbols={'a', 'b'},
        tape_symbols={'a', 'b', 'x', 'y', '.'},
        transitions={
            'q0': {
                '.': ('q1', '.' ,'R')
            },
            'q1': {
                'a': ('q2', '.', 'R')
            },
            'q2': {
                'a': ('q2', 'a', 'R'),
                'b': ('q3', 'b', 'R')
            },
            'q3': {
                'a': ('q4', 'a','R')
            },
            'q4': {
                'b': ('q5', 'b', 'R')
            },
            'q5': {
                'b': ('q3', 'b', 'R'),
                '.': ('q6', '.', 'L')
            },
            'q6': {
                'b': ('q7', '.', 'L')
            },
            'q7': {
                'a': ('q8', '.', 'L')
            },
            'q8': {
                'b': ('q9', '.', 'L')
            },
            'q9': {
                '.': ('q10', '.', 'R'),
                'b': ('q11', 'b', 'L')
            },
            'q11': {
                'a': ('q12', 'a', 'L')
            },
            'q12': {
                'b': ('q13', 'b', 'L')
            },
            'q13': {
                'b': ('q11', 'b', 'L'),
                'a': ('q14', 'a', 'L')
            },
            'q14': {
                'a': ('q14', 'a', 'L'),
                '.': ('q1', '.', 'R')
            },
        },
        initial_state='q0',
        blank_symbol='.',
        final_states={'q10'}
    )

    centinela = True

    while(centinela):
        os.system('cls||clear')

        print("======       Teoria de Automatas: PIA        ======")
        print("======               Equipo III              ======")
        print("---------------------------------------------------")
        print("Lenguaje: L = {a^n (bab)^n | n > 0}")
        print("---------------------------------------------------")
        print("Ingrese una cadena para ser validada:")

        cadena = '.' + str(input()) + '.'

        print("---------------------------------------------------")

        imprimeResultado(cadena, turing_machine)

        print("---------------------------------------------------")
        print("¿Desea ingresar otra cadena? [S/N]:")

        opcion = input()

        if opcion.upper() == 'S':
            centinela = True
        elif opcion.upper() == 'N':
            centinela = False
        else:
            print("Opcion incorrecta. Ingrese una opcion valida.")
            input("Presione cualquier tecla para continuar...")

if __name__=="__main__":
    main()


import random
import time

numeros = int(input("Ingresa el tamano de lista que deseas --> "))
lista = []

for n in range(numeros):
	lista.append(random.randint(0,999))
print(lista)

start_time = time.time() #Ordenamiento con Burbuja
for n in range(len(lista)-1):
	for j in range(len(lista)-1):
		if lista[j] > lista[j+1]:
			temp = lista[j]
			lista[j]=lista[j+1]
			lista[j+1]=temp

print("La lista ordenada con burbuja es:   ")
print(lista)
print(f"El tiempo logrado fue de {time.time() - start_time}")


import numpy as np
import matplotlib.pyplot as plt

def life():
    # print('Enter seeds location:')
    # M = eval(input())

    # For test
    M = [[1, 0, 1, 1, 1],
         [0, 0, 0, 1, 0],
         [0, 0, 1, 0, 0],
         [0, 1, 1, 1, 0],
         [0, 0, 1, 0, 0]]
    M = np.pad(M, ((1, 1), (1, 1)), 'constant')
    M = np.array(M)
    L, W = np.shape(M)
    X = np.zeros((L, W))
    for i in range(1, L - 1):
        for j in range(1, W - 1):
            sub_matrix = M[i - 1:i + 2, j - 1:j + 2]
            # Any live cell with fewer than two live neighbors dies, as if by underpopulation
            if M[i, j] == 1 and np.count_nonzero(sub_matrix == 1) < 2:
                X[i, j] = 0
            # Any live cell with two or three live neighbors lives on to the next generation
            if M[i, j] == 1 and np.count_nonzero(sub_matrix == 1) in (2, 3):
                X[i, j] = M[i, j]
            # Any live cell with more than three live neighbors dies, as if by overpopulation
            if M[i, j] == 1 and np.count_nonzero(sub_matrix == 1) > 3:
                X[i, j] = 0
            # Any dead cell with exactly three live neighbors becomes a live cell, as if by reproduction
            if M[i, j] == 0 and np.count_nonzero(sub_matrix == 1) == 3:
                X[i, j] = 1
    X = X[1:L - 1, 1:W - 1]
    print("here is the next patten")
    print(X)
    img = plt.imshow(X)
    img.set_cmap('winter')
    plt.show()
    return X

# Run file
life()


'''
@version: 1.0.0
@author: CrossingVoid
@date: 2023/03/05

The default_menu.py is mainly for optional
function in the selective function.

'''
from Tkinter_template.Assets.project_management import new_window, select_cover
from Tkinter_template.Assets.font import font_get
from Tkinter_template.Assets.image import tk_image
from PIL import ImageColor
from tkinter import *
import os

def background_color(canvas: object):
    def color(position):
        str_of_color = f'#{red.get():02x}{green.get():02x}{blue.get():02x}'
        canvas.config(bg=str_of_color)

    def select_color(color):
        canvas.config(bg=color)
        win.destroy()

    win = new_window('Background Color')

    red = Scale(win, font=font_get(20), bg='red', length=255, tickinterval=50, command=color,
                from_=0, to=255, label='Red')
    green = Scale(win, font=font_get(20), bg='green', length=255, tickinterval=50, command=color,
                  from_=0, to=255, label='Green')
    blue = Scale(win, font=font_get(20), bg='blue', length=255, tickinterval=50, command=color,
                 from_=0, to=255, label='Blue')

    color_select = ['chocolate', 'lightblue',
                    'violet', 'silver', 'indigo']
    x, y, z = ImageColor.getrgb(canvas['bg'])
    red.set(x)
    green.set(y)
    blue.set(z)
    red.grid(row=1, column=1, rowspan=len(
        color_select), sticky='sn')
    green.grid(row=1, column=2, rowspan=len(
        color_select), sticky='sn')
    blue.grid(row=1, column=3, rowspan=len(
        color_select), sticky='sn')

    index = 1
    for item in color_select:
        Button(win, font=font_get(20), bg=item, height=15//len(color_select), fg='black', text=item,
               command=lambda x=item: select_color(x)).grid(row=index, column=4, sticky='ew')
        index += 1

def canvas_cover(canvas: object, canvas_side: tuple, side: tuple = None):
    def click(num):
        canvas_.itemconfigure(f'rec{num}', state='normal')
        for i in range(len(cover)):
            if i != num:
                canvas_.itemconfigure(f'rec{i}', state='hidden')

    def double_click(num):
        select_cover(canvas, canvas_side, cover[num])
        win.destroy()
    if side is None:
        side = canvas_side
    win = new_window(
        'Windows Cover', maxsize=side)
    scrollbar = Scrollbar(win)
    scrollbar.grid(row=1, column=2, sticky='ns')
    canvas_ = Canvas(win, width=side[0]-int(scrollbar['width']), height=side[1], bg=canvas['bg'],
                     yscrollcommand=scrollbar.set)
    canvas_.grid(row=1, column=1)
    scrollbar['command'] = canvas.yview

    cover = [img for img in os.listdir('images\\covers') if img.endswith('png')
             or img.endswith('jpg') or img.endswith('jpeg')]
    wi_inter, hi_inter = 100, 120
    wi = (int(canvas_['width']) - 2*wi_inter - 20)//3
    hi = wi * 3 // 4

    for i in range(len(cover)):

        canvas_.create_image(20+i % 3 * (wi + wi_inter), 20+i//3 * (hi + hi_inter), anchor='nw',
                             image=tk_image(
            cover[i], wi, hi),
            tags=(f'img{i}'))

        canvas_.create_text(20+i % 3 * (wi + wi_inter) + wi//2, 20+i//3 * (hi + hi_inter)+20+24//2 + hi, font=font_get(24),
                            text=cover[i], tags=(f'text{i}'))

        x, y, x2, y2 = (20+i % 3 * (wi + wi_inter), 20+i//3 * (hi + hi_inter),
                        20+i % 3 * (wi + wi_inter)+wi, 20+i//3 * (hi + hi_inter)+hi)
        canvas_.create_rectangle(x-4, y-4, x2+4, y2+4, width=4, outline='gold',
                                 state='hidden', tags=(f'rec{i}'))

        canvas_.tag_bind(f'img{i}', '<Button-1>',
                         lambda event, x=i: click(x))
        canvas_.tag_bind(
            f'img{i}', '<Double-Button-1>', lambda event, x=i: double_click(x))

    canvas_['scrollregion'] = (
        0, 0, int(canvas['width']), 20+i//3 *
        (hi + hi_inter)+20+24//2 + hi+60
    )
    canvas_.bind('<MouseWheel>', lambda event: canvas_.yview_scroll(-(event.delta//120), 'units')
                 )


import time
import datetime

now = datetime.datetime.now()
if now.month <= 6:
    before_date = "%s-%s-%s" % (now.year - 1, 6 + now.month, now.day)
else:
    before_date = "%s-%s-%s" % (now.year, now.month - 6)
print(before_date)

# 上面已经可以算出6个月前的今天，是字符串格式。如果想转回时间类型，则加上以下两句
#atime = datetime.datetime.strptime(before_date, "%Y-%m-%d")
#print(atime)


import os
from flask import Flask, render_template, url_for, redirect
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from forms import AddForm, DelForm
app = Flask(__name__)

app.config['SECRET_KEY'] = 'mysecretkey'
###########################
### SQL DATABASE BÖLÜMÜ ###
###########################
basedir = os.path.abspath(os.path.dirname(__file__))
app.config['SQLALCHEMY_DATABASE_URI'] = "sqlite:///" + \
    os.path.join(basedir+"data.sqlite")
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
Migrate(app, db)
###########################
###    MODELS    BÖLÜMÜ ###
###########################

class Ponny(db.Model):
    __tablename__ = 'ponnies'
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.Text)

    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return f"Ponny name is {self.name}"
###########################
###    VİEWS     BÖLÜMÜ ###
###########################
@app.route('/')
def index():
    return render_template('index.html')

@app.route('/add',methods=['GET','POST'])
def add():
    form = AddForm()
    if form.validate_on_submit():
        name = form.name.data
        new_ponny = Ponny(name)
        db.session.add(new_ponny)
        db.session.commit()
        return redirect(url_for('liste'))
    return render_template('add.html', form=form)

@app.route('/list')
def liste():
    ponnies = Ponny.query.all()
    return render_template('list.html', ponnies=ponnies)

@app.route('/delete', methods=['GET', 'POST'])
def delete():
    form = DelForm()
    if form.validate_on_submit():
        id = form.id.data
        ponny = Ponny.query.get(id)
        db.session.delete(ponny)
        db.session.commit()
        return redirect(url_for('liste'))
    return render_template('delete.html', form=form)

if __name__ == "__main__":
    app.run(debug=True)


from unittest import TestCase

import networkx

from gtfspy.routing.connection import Connection
from gtfspy.routing.label import LabelTime
from gtfspy.routing.pseudo_connection_scan_profiler import PseudoConnectionScanProfiler

class TestPseudoPseudoConnectionScanProfiler(TestCase):
    # noinspection PyAttributeOutsideInit

    def setUp(self):
        event_list_raw_data = [
            (2, 4, 40, 50, "trip_6", 1),
            (1, 3, 32, 40, "trip_5", 1),
            (3, 4, 32, 35, "trip_4", 1),
            (2, 3, 25, 30, "trip_3", 1),
            (1, 2, 10, 20, "trip_2", 1),
            (0, 1, 0, 10, "trip_1", 1)
        ]
        self.transit_connections = list(map(lambda el: Connection(*el), event_list_raw_data))
        self.walk_network = networkx.Graph()
        self.walk_network.add_edge(1, 2, d_walk=20)
        self.walk_network.add_edge(3, 4, d_walk=15)
        self.walk_speed = 1
        self.source_stop = 1
        self.target_stop = 4
        self.transfer_margin = 0
        self.start_time = 0
        self.end_time = 50

    def test_basics(self):
        csa_profile = PseudoConnectionScanProfiler(self.transit_connections, self.target_stop,
                                                   self.start_time, self.end_time, self.transfer_margin,
                                                   self.walk_network, self.walk_speed)
        csa_profile.run()

        stop_3_labels = csa_profile.stop_profiles[3].get_final_optimal_labels()
        self.assertEqual(len(stop_3_labels), 1)
        self.assertIn(LabelTime(32, 35), stop_3_labels)

        stop_2_labels = csa_profile.stop_profiles[2].get_final_optimal_labels()
        self.assertEqual(len(stop_2_labels), 2)
        self.assertIn(LabelTime(40, 50), stop_2_labels)
        self.assertIn(LabelTime(25, 35), stop_2_labels)

        source_stop_profile = csa_profile.stop_profiles[self.source_stop]
        source_stop_pareto_optimal_tuples = source_stop_profile.get_final_optimal_labels()

        labels = []
        labels.append(LabelTime(departure_time=10, arrival_time_target=35))
        labels.append(LabelTime(departure_time=20, arrival_time_target=50))
        labels.append(LabelTime(departure_time=32, arrival_time_target=55))

        self._assert_pareto_tuple_sets_equal(
            labels,
            source_stop_pareto_optimal_tuples
        )

    def test_simple(self):
        event_list_raw_data = [
            (2, 4, 40, 50, "trip_5", 1),
        ]
        transit_connections = list(map(lambda el: Connection(*el), event_list_raw_data))
        walk_network = networkx.Graph()
        walk_network.add_edge(1, 2, d_walk=20)
        walk_network.add_edge(3, 4, d_walk=15)
        walk_speed = 1
        source_stop = 1
        target_stop = 4
        transfer_margin = 0
        start_time = 0
        end_time = 50

        labels = []
        labels.append(LabelTime(departure_time=20, arrival_time_target=50))

        csa_profile = PseudoConnectionScanProfiler(transit_connections, target_stop,
                                                   start_time, end_time, transfer_margin,
                                                   walk_network, walk_speed)
        csa_profile.run()
        source_stop_profile = csa_profile.stop_profiles[source_stop]
        source_stop_labels = source_stop_profile.get_final_optimal_labels()

        self._assert_pareto_tuple_sets_equal(
            labels,
            source_stop_labels
        )

    def test_last_leg_is_walk(self):
        event_list_raw_data = [
            (0, 1, 0, 10, "trip_1", 1)
        ]
        transit_connections = list(map(lambda el: Connection(*el), event_list_raw_data))
        walk_network = networkx.Graph()
        walk_network.add_edge(1, 2, d_walk=20)

        walk_speed = 1
        source_stop = 0
        target_stop = 2
        transfer_margin = 0
        start_time = 0
        end_time = 50
        labels = []
        labels.append(LabelTime(departure_time=0, arrival_time_target=30))

        csa_profile = PseudoConnectionScanProfiler(transit_connections, target_stop,
                                                   start_time, end_time, transfer_margin,
                                                   walk_network, walk_speed)
        csa_profile.run()
        found_tuples = csa_profile.stop_profiles[source_stop].get_final_optimal_labels()
        self._assert_pareto_tuple_sets_equal(found_tuples, labels)

    def test_walk_is_faster_than_by_trip(self):
        event_list_raw_data = [
            (0, 1, 0, 10, "trip_1", 1)
        ]
        transit_connections = list(map(lambda el: Connection(*el), event_list_raw_data))
        walk_speed = 2
        source_stop = 0
        target_stop = 1
        transfer_margin = 0
        start_time = 0
        end_time = 50

        walk_network = networkx.Graph()
        walk_network.add_edge(0, 1, d_walk=1)
        csa_profile = PseudoConnectionScanProfiler(transit_connections, target_stop,
                                                   start_time, end_time, transfer_margin,
                                                   walk_network, walk_speed)
        csa_profile.run()
        source_profile = csa_profile.stop_profiles[source_stop]
        self.assertEqual(source_profile.evaluate_earliest_arrival_time_at_target(0, 0), 0.5)
        found_tuples = source_profile.get_final_optimal_labels()
        self.assertEqual(len(found_tuples), 0)

    def test_target_node_not_in_walk_network(self):
        event_list_raw_data = [
            (0, 1, 0, 10, "trip_1", 1)
        ]
        transit_connections = list(map(lambda el: Connection(*el), event_list_raw_data))
        walk_speed = 2
        source_stop = 0
        target_stop = 1
        transfer_margin = 0
        start_time = 0
        end_time = 50

        walk_network = networkx.Graph()
        csa_profile = PseudoConnectionScanProfiler(transit_connections, target_stop,
                                                   start_time, end_time, transfer_margin,
                                                   walk_network, walk_speed)
        csa_profile.run()
        source_profile = csa_profile.stop_profiles[source_stop]
        self.assertEqual(source_profile.evaluate_earliest_arrival_time_at_target(0, 0), 10)
        found_tuples = source_profile.get_final_optimal_labels()
        self.assertEqual(len(found_tuples), 1)

    def _assert_pareto_tuple_sets_equal(self, found_tuples, should_be_tuples):
        for found_tuple in found_tuples:
            self.assertIn(found_tuple, should_be_tuples)
        for should_be_tuple in should_be_tuples:
            self.assertIn(should_be_tuple, found_tuples)


#!/usr/bin/env micropython
from ev3dev2.button import Button
from ev3dev2.sound import Sound
from ev3dev2.motor import LargeMotor,MediumMotor, OUTPUT_D, OUTPUT_A,  OUTPUT_C, OUTPUT_B, follow_for_ms
from time import sleep
from trip1 import runTrip1
from trip2 import runTrip2
from trip3 import runTrip3
from trip4 import runTrip4
from trip5 import runTrip5
from time import sleep
import os
os.system('setfont Uni3-TerminusBold16')
sound = Sound()
LeftAction      = MediumMotor(OUTPUT_A)
RightAction     = MediumMotor(OUTPUT_D)
LeftWheel       = LargeMotor(OUTPUT_B)
RightWheel      = LargeMotor(OUTPUT_C)

btn = Button()
contin = True
print('*** ATHENA ROCKS! ***')
print('1 - Left for Trip 1')
print('2 - Up for Trip 2')
print('3 - Right for Trip 3')
print('4 - Down for Trip 4')
print('5 - Center for Trip 5')
sound.beep()
while contin:
    RightAction.reset()
    LeftAction.reset()
    LeftWheel.reset()
    RightWheel.reset()
    if btn.left == True:
        runTrip1()
        sound.beep()
    elif btn.up == True:
        runTrip2()
        sound.beep()
    elif btn.right == True:
        runTrip3()
        sound.beep()
    elif btn.down == True:
        runTrip4()
        sound.beep()
    elif btn.enter == True:
        runTrip5()
        sound.beep()