"""
requests.utils
~~~~~~~~~~~~~~

This module provides utility functions that are used within Requests
that are also useful for external consumption.
"""

import codecs
import contextlib
import io
import os
import re
import socket
import struct
import sys
import tempfile
import warnings
import zipfile
from collections import OrderedDict

from urllib3.util import make_headers, parse_url

from . import certs
from .__version__ import __version__

# to_native_string is unused here, but imported here for backwards compatibility
from ._internal_utils import (  # noqa: F401
    _HEADER_VALIDATORS_BYTE,
    _HEADER_VALIDATORS_STR,
    HEADER_VALIDATORS,
    to_native_string,
)
from .compat import (
    Mapping,
    basestring,
    bytes,
    getproxies,
    getproxies_environment,
    integer_types,
)
from .compat import parse_http_list as _parse_list_header
from .compat import (
    proxy_bypass,
    proxy_bypass_environment,
    quote,
    str,
    unquote,
    urlparse,
    urlunparse,
)
from .cookies import cookiejar_from_dict
from .exceptions import (
    FileModeWarning,
    InvalidHeader,
    InvalidURL,
    UnrewindableBodyError,
)
from .structures import CaseInsensitiveDict

NETRC_FILES = (".netrc", "_netrc")

DEFAULT_CA_BUNDLE_PATH = certs.where()

DEFAULT_PORTS = {"http": 80, "https": 443}

# Ensure that ', ' is used to preserve previous delimiter behavior.
DEFAULT_ACCEPT_ENCODING = ", ".join(
    re.split(r",\s*", make_headers(accept_encoding=True)["accept-encoding"])
)

if sys.platform == "win32":
    # provide a proxy_bypass version on Windows without DNS lookups

    def proxy_bypass_registry(host):
        try:
            import winreg
        except ImportError:
            return False

        try:
            internetSettings = winreg.OpenKey(
                winreg.HKEY_CURRENT_USER,
                r"Software\Microsoft\Windows\CurrentVersion\Internet Settings",
            )
            # ProxyEnable could be REG_SZ or REG_DWORD, normalizing it
            proxyEnable = int(winreg.QueryValueEx(internetSettings, "ProxyEnable")[0])
            # ProxyOverride is almost always a string
            proxyOverride = winreg.QueryValueEx(internetSettings, "ProxyOverride")[0]
        except (OSError, ValueError):
            return False
        if not proxyEnable or not proxyOverride:
            return False

        # make a check value list from the registry entry: replace the
        # '<local>' string by the localhost entry and the corresponding
        # canonical entry.
        proxyOverride = proxyOverride.split(";")
        # now check if we match one of the registry values.
        for test in proxyOverride:
            if test == "<local>":
                if "." not in host:
                    return True
            test = test.replace(".", r"\.")  # mask dots
            test = test.replace("*", r".*")  # change glob sequence
            test = test.replace("?", r".")  # change glob char
            if re.match(test, host, re.I):
                return True
        return False

    def proxy_bypass(host):  # noqa
        """Return True, if the host should be bypassed.

        Checks proxy settings gathered from the environment, if specified,
        or the registry.
        """
        if getproxies_environment():
            return proxy_bypass_environment(host)
        else:
            return proxy_bypass_registry(host)

def dict_to_sequence(d):
    """Returns an internal sequence dictionary update."""

    if hasattr(d, "items"):
        d = d.items()

    return d

def super_len(o):
    total_length = None
    current_position = 0

    if hasattr(o, "__len__"):
        total_length = len(o)

    elif hasattr(o, "len"):
        total_length = o.len

    elif hasattr(o, "fileno"):
        try:
            fileno = o.fileno()
        except (io.UnsupportedOperation, AttributeError):
            # AttributeError is a surprising exception, seeing as how we've just checked
            # that `hasattr(o, 'fileno')`.  It happens for objects obtained via
            # `Tarfile.extractfile()`, per issue 5229.
            pass
        else:
            total_length = os.fstat(fileno).st_size

            # Having used fstat to determine the file length, we need to
            # confirm that this file was opened up in binary mode.
            if "b" not in o.mode:
                warnings.warn(
                    (
                        "Requests has determined the content-length for this "
                        "request using the binary size of the file: however, the "
                        "file has been opened in text mode (i.e. without the 'b' "
                        "flag in the mode). This may lead to an incorrect "
                        "content-length. In Requests 3.0, support will be removed "
                        "for files in text mode."
                    ),
                    FileModeWarning,
                )

    if hasattr(o, "tell"):
        try:
            current_position = o.tell()
        except OSError:
            # This can happen in some weird situations, such as when the file
            # is actually a special file descriptor like stdin. In this
            # instance, we don't know what the length is, so set it to zero and
            # let requests chunk it instead.
            if total_length is not None:
                current_position = total_length
        else:
            if hasattr(o, "seek") and total_length is None:
                # StringIO and BytesIO have seek but no usable fileno
                try:
                    # seek to end of file
                    o.seek(0, 2)
                    total_length = o.tell()

                    # seek back to current position to support
                    # partially read file-like objects
                    o.seek(current_position or 0)
                except OSError:
                    total_length = 0

    if total_length is None:
        total_length = 0

    return max(0, total_length - current_position)

def get_netrc_auth(url, raise_errors=False):
    """Returns the Requests tuple auth for a given url from netrc."""

    netrc_file = os.environ.get("NETRC")
    if netrc_file is not None:
        netrc_locations = (netrc_file,)
    else:
        netrc_locations = (f"~/{f}" for f in NETRC_FILES)

    try:
        from netrc import NetrcParseError, netrc

        netrc_path = None

        for f in netrc_locations:
            try:
                loc = os.path.expanduser(f)
            except KeyError:
                # os.path.expanduser can fail when $HOME is undefined and
                # getpwuid fails. See https://bugs.python.org/issue20164 &
                # https://github.com/psf/requests/issues/1846
                return

            if os.path.exists(loc):
                netrc_path = loc
                break

        # Abort early if there isn't one.
        if netrc_path is None:
            return

        ri = urlparse(url)

        # Strip port numbers from netloc. This weird `if...encode`` dance is
        # used for Python 3.2, which doesn't support unicode literals.
        splitstr = b":"
        if isinstance(url, str):
            splitstr = splitstr.decode("ascii")
        host = ri.netloc.split(splitstr)[0]

        try:
            _netrc = netrc(netrc_path).authenticators(host)
            if _netrc:
                # Return with login / password
                login_i = 0 if _netrc[0] else 1
                return (_netrc[login_i], _netrc[2])
        except (NetrcParseError, OSError):
            # If there was a parsing error or a permissions issue reading the file,
            # we'll just skip netrc auth unless explicitly asked to raise errors.
            if raise_errors:
                raise

    # App Engine hackiness.
    except (ImportError, AttributeError):
        pass

def guess_filename(obj):
    """Tries to guess the filename of the given object."""
    name = getattr(obj, "name", None)
    if name and isinstance(name, basestring) and name[0] != "<" and name[-1] != ">":
        return os.path.basename(name)

def extract_zipped_paths(path):
    """Replace nonexistent paths that look like they refer to a member of a zip
    archive with the location of an extracted copy of the target, or else
    just return the provided path unchanged.
    """
    if os.path.exists(path):
        # this is already a valid path, no need to do anything further
        return path

    # find the first valid part of the provided path and treat that as a zip archive
    # assume the rest of the path is the name of a member in the archive
    archive, member = os.path.split(path)
    while archive and not os.path.exists(archive):
        archive, prefix = os.path.split(archive)
        if not prefix:
            # If we don't check for an empty prefix after the split (in other words, archive remains unchanged after the split),
            # we _can_ end up in an infinite loop on a rare corner case affecting a small number of users
            break
        member = "/".join([prefix, member])

    if not zipfile.is_zipfile(archive):
        return path

    zip_file = zipfile.ZipFile(archive)
    if member not in zip_file.namelist():
        return path

    # we have a valid zip archive and a valid member of that archive
    tmp = tempfile.gettempdir()
    extracted_path = os.path.join(tmp, member.split("/")[-1])
    if not os.path.exists(extracted_path):
        # use read + write to avoid the creating nested folders, we only want the file, avoids mkdir racing condition
        with atomic_open(extracted_path) as file_handler:
            file_handler.write(zip_file.read(member))
    return extracted_path

@contextlib.contextmanager
def atomic_open(filename):
    """Write a file to the disk in an atomic fashion"""
    tmp_descriptor, tmp_name = tempfile.mkstemp(dir=os.path.dirname(filename))
    try:
        with os.fdopen(tmp_descriptor, "wb") as tmp_handler:
            yield tmp_handler
        os.replace(tmp_name, filename)
    except BaseException:
        os.remove(tmp_name)
        raise

def from_key_val_list(value):
    """Take an object and test to see if it can be represented as a
    dictionary. Unless it can not be represented as such, return an
    OrderedDict, e.g.,

    ::

        >>> from_key_val_list([('key', 'val')])
        OrderedDict([('key', 'val')])
        >>> from_key_val_list('string')
        Traceback (most recent call last):
        ...
        ValueError: cannot encode objects that are not 2-tuples
        >>> from_key_val_list({'key': 'val'})
        OrderedDict([('key', 'val')])

    :rtype: OrderedDict
    """
    if value is None:
        return None

    if isinstance(value, (str, bytes, bool, int)):
        raise ValueError("cannot encode objects that are not 2-tuples")

    return OrderedDict(value)

def to_key_val_list(value):
    """Take an object and test to see if it can be represented as a
    dictionary. If it can be, return a list of tuples, e.g.,

    ::

        >>> to_key_val_list([('key', 'val')])
        [('key', 'val')]
        >>> to_key_val_list({'key': 'val'})
        [('key', 'val')]
        >>> to_key_val_list('string')
        Traceback (most recent call last):
        ...
        ValueError: cannot encode objects that are not 2-tuples

    :rtype: list
    """
    if value is None:
        return None

    if isinstance(value, (str, bytes, bool, int)):
        raise ValueError("cannot encode objects that are not 2-tuples")

    if isinstance(value, Mapping):
        value = value.items()

    return list(value)

# From mitsuhiko/werkzeug (used with permission).
def parse_list_header(value):
    """Parse lists as described by RFC 2068 Section 2.

    In particular, parse comma-separated lists where the elements of
    the list may include quoted-strings.  A quoted-string could
    contain a comma.  A non-quoted string could have quotes in the
    middle.  Quotes are removed automatically after parsing.

    It basically works like :func:`parse_set_header` just that items
    may appear multiple times and case sensitivity is preserved.

    The return value is a standard :class:`list`:

    >>> parse_list_header('token, "quoted value"')
    ['token', 'quoted value']

    To create a header from the :class:`list` again, use the
    :func:`dump_header` function.

    :param value: a string with a list header.
    :return: :class:`list`
    :rtype: list
    """
    result = []
    for item in _parse_list_header(value):
        if item[:1] == item[-1:] == '"':
            item = unquote_header_value(item[1:-1])
        result.append(item)
    return result

# From mitsuhiko/werkzeug (used with permission).
def parse_dict_header(value):
    """Parse lists of key, value pairs as described by RFC 2068 Section 2 and
    convert them into a python dict:

    >>> d = parse_dict_header('foo="is a fish", bar="as well"')
    >>> type(d) is dict
    True
    >>> sorted(d.items())
    [('bar', 'as well'), ('foo', 'is a fish')]

    If there is no value for a key it will be `None`:

    >>> parse_dict_header('key_without_value')
    {'key_without_value': None}

    To create a header from the :class:`dict` again, use the
    :func:`dump_header` function.

    :param value: a string with a dict header.
    :return: :class:`dict`
    :rtype: dict
    """
    result = {}
    for item in _parse_list_header(value):
        if "=" not in item:
            result[item] = None
            continue
        name, value = item.split("=", 1)
        if value[:1] == value[-1:] == '"':
            value = unquote_header_value(value[1:-1])
        result[name] = value
    return result

# From mitsuhiko/werkzeug (used with permission).
def unquote_header_value(value, is_filename=False):
    r"""Unquotes a header value.  (Reversal of :func:`quote_header_value`).
    This does not use the real unquoting but what browsers are actually
    using for quoting.

    :param value: the header value to unquote.
    :rtype: str
    """
    if value and value[0] == value[-1] == '"':
        # this is not the real unquoting, but fixing this so that the
        # RFC is met will result in bugs with internet explorer and
        # probably some other browsers as well.  IE for example is
        # uploading files with "C:\foo\bar.txt" as filename
        value = value[1:-1]

        # if this is a filename and the starting characters look like
        # a UNC path, then just return the value without quotes.  Using the
        # replace sequence below on a UNC path has the effect of turning
        # the leading double slash into a single slash and then
        # _fix_ie_filename() doesn't work correctly.  See #458.
        if not is_filename or value[:2] != "\\\\":
            return value.replace("\\\\", "\\").replace('\\"', '"')
    return value

def dict_from_cookiejar(cj):
    """Returns a key/value dictionary from a CookieJar.

    :param cj: CookieJar object to extract cookies from.
    :rtype: dict
    """

    cookie_dict = {cookie.name: cookie.value for cookie in cj}
    return cookie_dict

def add_dict_to_cookiejar(cj, cookie_dict):
    """Returns a CookieJar from a key/value dictionary.

    :param cj: CookieJar to insert cookies into.
    :param cookie_dict: Dict of key/values to insert into CookieJar.
    :rtype: CookieJar
    """

    return cookiejar_from_dict(cookie_dict, cj)

def get_encodings_from_content(content):
    """Returns encodings from given content string.

    :param content: bytestring to extract encodings from.
    """
    warnings.warn(
        (
            "In requests 3.0, get_encodings_from_content will be removed. For "
            "more information, please see the discussion on issue #2266. (This"
            " warning should only appear once.)"
        ),
        DeprecationWarning,
    )

    charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
    pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)
    xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]')

    return (
        charset_re.findall(content)
        + pragma_re.findall(content)
        + xml_re.findall(content)
    )

def _parse_content_type_header(header):
    """Returns content type and parameters from given header

    :param header: string
    :return: tuple containing content type and dictionary of
         parameters
    """

    tokens = header.split(";")
    content_type, params = tokens[0].strip(), tokens[1:]
    params_dict = {}
    items_to_strip = "\"' "

    for param in params:
        param = param.strip()
        if param:
            key, value = param, True
            index_of_equals = param.find("=")
            if index_of_equals != -1:
                key = param[:index_of_equals].strip(items_to_strip)
                value = param[index_of_equals + 1 :].strip(items_to_strip)
            params_dict[key.lower()] = value
    return content_type, params_dict

def get_encoding_from_headers(headers):
    """Returns encodings from given HTTP Header Dict.

    :param headers: dictionary to extract encoding from.
    :rtype: str
    """

    content_type = headers.get("content-type")

    if not content_type:
        return None

    content_type, params = _parse_content_type_header(content_type)

    if "charset" in params:
        return params["charset"].strip("'\"")

    if "text" in content_type:
        return "ISO-8859-1"

    if "application/json" in content_type:
        # Assume UTF-8 based on RFC 4627: https://www.ietf.org/rfc/rfc4627.txt since the charset was unset
        return "utf-8"

def stream_decode_response_unicode(iterator, r):
    """Stream decodes an iterator."""

    if r.encoding is None:
        yield from iterator
        return

    decoder = codecs.getincrementaldecoder(r.encoding)(errors="replace")
    for chunk in iterator:
        rv = decoder.decode(chunk)
        if rv:
            yield rv
    rv = decoder.decode(b"", final=True)
    if rv:
        yield rv

def iter_slices(string, slice_length):
    """Iterate over slices of a string."""
    pos = 0
    if slice_length is None or slice_length <= 0:
        slice_length = len(string)
    while pos < len(string):
        yield string[pos : pos + slice_length]
        pos += slice_length

def get_unicode_from_response(r):
    """Returns the requested content back in unicode.

    :param r: Response object to get unicode content from.

    Tried:

    1. charset from content-type
    2. fall back and replace all unicode characters

    :rtype: str
    """
    warnings.warn(
        (
            "In requests 3.0, get_unicode_from_response will be removed. For "
            "more information, please see the discussion on issue #2266. (This"
            " warning should only appear once.)"
        ),
        DeprecationWarning,
    )

    tried_encodings = []

    # Try charset from content-type
    encoding = get_encoding_from_headers(r.headers)

    if encoding:
        try:
            return str(r.content, encoding)
        except UnicodeError:
            tried_encodings.append(encoding)

    # Fall back:
    try:
        return str(r.content, encoding, errors="replace")
    except TypeError:
        return r.content

# The unreserved URI characters (RFC 3986)
UNRESERVED_SET = frozenset(
    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~"
)

def unquote_unreserved(uri):
    """Un-escape any percent-escape sequences in a URI that are unreserved
    characters. This leaves all reserved, illegal and non-ASCII bytes encoded.

    :rtype: str
    """
    parts = uri.split("%")
    for i in range(1, len(parts)):
        h = parts[i][0:2]
        if len(h) == 2 and h.isalnum():
            try:
                c = chr(int(h, 16))
            except ValueError:
                raise InvalidURL(f"Invalid percent-escape sequence: '{h}'")

            if c in UNRESERVED_SET:
                parts[i] = c + parts[i][2:]
            else:
                parts[i] = f"%{parts[i]}"
        else:
            parts[i] = f"%{parts[i]}"
    return "".join(parts)

def requote_uri(uri):
    """Re-quote the given URI.

    This function passes the given URI through an unquote/quote cycle to
    ensure that it is fully and consistently quoted.

    :rtype: str
    """
    safe_with_percent = "!#$%&'()*+,/:;=?@[]~"
    safe_without_percent = "!#$&'()*+,/:;=?@[]~"
    try:
        # Unquote only the unreserved characters
        # Then quote only illegal characters (do not quote reserved,
        # unreserved, or '%')
        return quote(unquote_unreserved(uri), safe=safe_with_percent)
    except InvalidURL:
        # We couldn't unquote the given URI, so let's try quoting it, but
        # there may be unquoted '%'s in the URI. We need to make sure they're
        # properly quoted so they do not cause issues elsewhere.
        return quote(uri, safe=safe_without_percent)

def address_in_network(ip, net):
    """This function allows you to check if an IP belongs to a network subnet

    Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24
             returns False if ip = 192.168.1.1 and net = 192.168.100.0/24

    :rtype: bool
    """
    ipaddr = struct.unpack("=L", socket.inet_aton(ip))[0]
    netaddr, bits = net.split("/")
    netmask = struct.unpack("=L", socket.inet_aton(dotted_netmask(int(bits))))[0]
    network = struct.unpack("=L", socket.inet_aton(netaddr))[0] & netmask
    return (ipaddr & netmask) == (network & netmask)

def dotted_netmask(mask):
    """Converts mask from /xx format to xxx.xxx.xxx.xxx

    Example: if mask is 24 function returns 255.255.255.0

    :rtype: str
    """
    bits = 0xFFFFFFFF ^ (1 << 32 - mask) - 1
    return socket.inet_ntoa(struct.pack(">I", bits))

def is_ipv4_address(string_ip):
    """
    :rtype: bool
    """
    try:
        socket.inet_aton(string_ip)
    except OSError:
        return False
    return True

def is_valid_cidr(string_network):
    """
    Very simple check of the cidr format in no_proxy variable.

    :rtype: bool
    """
    if string_network.count("/") == 1:
        try:
            mask = int(string_network.split("/")[1])
        except ValueError:
            return False

        if mask < 1 or mask > 32:
            return False

        try:
            socket.inet_aton(string_network.split("/")[0])
        except OSError:
            return False
    else:
        return False
    return True

@contextlib.contextmanager
def set_environ(env_name, value):
    """Set the environment variable 'env_name' to 'value'

    Save previous value, yield, and then restore the previous value stored in
    the environment variable 'env_name'.

    If 'value' is None, do nothing"""
    value_changed = value is not None
    if value_changed:
        old_value = os.environ.get(env_name)
        os.environ[env_name] = value
    try:
        yield
    finally:
        if value_changed:
            if old_value is None:
                del os.environ[env_name]
            else:
                os.environ[env_name] = old_value

def should_bypass_proxies(url, no_proxy):
    """
    Returns whether we should bypass proxies or not.

    :rtype: bool
    """

    # Prioritize lowercase environment variables over uppercase
    # to keep a consistent behaviour with other http projects (curl, wget).
    def get_proxy(key):
        return os.environ.get(key) or os.environ.get(key.upper())

    # First check whether no_proxy is defined. If it is, check that the URL
    # we're getting isn't in the no_proxy list.
    no_proxy_arg = no_proxy
    if no_proxy is None:
        no_proxy = get_proxy("no_proxy")
    parsed = urlparse(url)

    if parsed.hostname is None:
        # URLs don't always have hostnames, e.g. file:/// urls.
        return True

    if no_proxy:
        # We need to check whether we match here. We need to see if we match
        # the end of the hostname, both with and without the port.
        no_proxy = (host for host in no_proxy.replace(" ", "").split(",") if host)

        if is_ipv4_address(parsed.hostname):
            for proxy_ip in no_proxy:
                if is_valid_cidr(proxy_ip):
                    if address_in_network(parsed.hostname, proxy_ip):
                        return True
                elif parsed.hostname == proxy_ip:
                    # If no_proxy ip was defined in plain IP notation instead of cidr notation &
                    # matches the IP of the index
                    return True
        else:
            host_with_port = parsed.hostname
            if parsed.port:
                host_with_port += f":{parsed.port}"

            for host in no_proxy:
                if parsed.hostname.endswith(host) or host_with_port.endswith(host):
                    # The URL does match something in no_proxy, so we don't want
                    # to apply the proxies on this URL.
                    return True

    with set_environ("no_proxy", no_proxy_arg):
        # parsed.hostname can be `None` in cases such as a file URI.
        try:
            bypass = proxy_bypass(parsed.hostname)
        except (TypeError, socket.gaierror):
            bypass = False

    if bypass:
        return True

    return False

def get_environ_proxies(url, no_proxy=None):
    """
    Return a dict of environment proxies.

    :rtype: dict
    """
    if should_bypass_proxies(url, no_proxy=no_proxy):
        return {}
    else:
        return getproxies()

def select_proxy(url, proxies):
    """Select a proxy for the url, if applicable.

    :param url: The url being for the request
    :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs
    """
    proxies = proxies or {}
    urlparts = urlparse(url)
    if urlparts.hostname is None:
        return proxies.get(urlparts.scheme, proxies.get("all"))

    proxy_keys = [
        urlparts.scheme + "://" + urlparts.hostname,
        urlparts.scheme,
        "all://" + urlparts.hostname,
        "all",
    ]
    proxy = None
    for proxy_key in proxy_keys:
        if proxy_key in proxies:
            proxy = proxies[proxy_key]
            break

    return proxy

def resolve_proxies(request, proxies, trust_env=True):
    """This method takes proxy information from a request and configuration
    input to resolve a mapping of target proxies. This will consider settings
    such a NO_PROXY to strip proxy configurations.

    :param request: Request or PreparedRequest
    :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs
    :param trust_env: Boolean declaring whether to trust environment configs

    :rtype: dict
    """
    proxies = proxies if proxies is not None else {}
    url = request.url
    scheme = urlparse(url).scheme
    no_proxy = proxies.get("no_proxy")
    new_proxies = proxies.copy()

    if trust_env and not should_bypass_proxies(url, no_proxy=no_proxy):
        environ_proxies = get_environ_proxies(url, no_proxy=no_proxy)

        proxy = environ_proxies.get(scheme, environ_proxies.get("all"))

        if proxy:
            new_proxies.setdefault(scheme, proxy)
    return new_proxies

def default_user_agent(name="python-requests"):
    """
    Return a string representing the default user agent.

    :rtype: str
    """
    return f"{name}/{__version__}"

def default_headers():
    """
    :rtype: requests.structures.CaseInsensitiveDict
    """
    return CaseInsensitiveDict(
        {
            "User-Agent": default_user_agent(),
            "Accept-Encoding": DEFAULT_ACCEPT_ENCODING,
            "Accept": "*/*",
            "Connection": "keep-alive",
        }
    )

def parse_header_links(value):
    """Return a list of parsed link headers proxies.

    i.e. Link: <http:/.../front.jpeg>; rel=front; type="image/jpeg",<http://.../back.jpeg>; rel=back;type="image/jpeg"

    :rtype: list
    """

    links = []

    replace_chars = " '\""

    value = value.strip(replace_chars)
    if not value:
        return links

    for val in re.split(", *<", value):
        try:
            url, params = val.split(";", 1)
        except ValueError:
            url, params = val, ""

        link = {"url": url.strip("<> '\"")}

        for param in params.split(";"):
            try:
                key, value = param.split("=")
            except ValueError:
                break

            link[key.strip(replace_chars)] = value.strip(replace_chars)

        links.append(link)

    return links

# Null bytes; no need to recreate these on each call to guess_json_utf
_null = "\x00".encode("ascii")  # encoding to ASCII for Python 3
_null2 = _null * 2
_null3 = _null * 3

def guess_json_utf(data):
    """
    :rtype: str
    """
    # JSON always starts with two ASCII characters, so detection is as
    # easy as counting the nulls and from their location and count
    # determine the encoding. Also detect a BOM, if present.
    sample = data[:4]
    if sample in (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE):
        return "utf-32"  # BOM included
    if sample[:3] == codecs.BOM_UTF8:
        return "utf-8-sig"  # BOM included, MS style (discouraged)
    if sample[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):
        return "utf-16"  # BOM included
    nullcount = sample.count(_null)
    if nullcount == 0:
        return "utf-8"
    if nullcount == 2:
        if sample[::2] == _null2:  # 1st and 3rd are null
            return "utf-16-be"
        if sample[1::2] == _null2:  # 2nd and 4th are null
            return "utf-16-le"
        # Did not detect 2 valid UTF-16 ascii-range characters
    if nullcount == 3:
        if sample[:3] == _null3:
            return "utf-32-be"
        if sample[1:] == _null3:
            return "utf-32-le"
        # Did not detect a valid UTF-32 ascii-range character
    return None

def prepend_scheme_if_needed(url, new_scheme):
    """Given a URL that may or may not have a scheme, prepend the given scheme.
    Does not replace a present scheme with the one provided as an argument.

    :rtype: str
    """
    parsed = parse_url(url)
    scheme, auth, host, port, path, query, fragment = parsed

    # A defect in urlparse determines that there isn't a netloc present in some
    # urls. We previously assumed parsing was overly cautious, and swapped the
    # netloc and path. Due to a lack of tests on the original defect, this is
    # maintained with parse_url for backwards compatibility.
    netloc = parsed.netloc
    if not netloc:
        netloc, path = path, netloc

    if auth:
        # parse_url doesn't provide the netloc with auth
        # so we'll add it ourselves.
        netloc = "@".join([auth, netloc])
    if scheme is None:
        scheme = new_scheme
    if path is None:
        path = ""

    return urlunparse((scheme, netloc, path, "", query, fragment))

def get_auth_from_url(url):
    """Given a url with authentication components, extract them into a tuple of
    username,password.

    :rtype: (str,str)
    """
    parsed = urlparse(url)

    try:
        auth = (unquote(parsed.username), unquote(parsed.password))
    except (AttributeError, TypeError):
        auth = ("", "")

    return auth

def check_header_validity(header):
    """Verifies that header parts don't contain leading whitespace
    reserved characters, or return characters.

    :param header: tuple, in the format (name, value).
    """
    name, value = header
    _validate_header_part(header, name, 0)
    _validate_header_part(header, value, 1)

def _validate_header_part(header, header_part, header_validator_index):
    if isinstance(header_part, str):
        validator = _HEADER_VALIDATORS_STR[header_validator_index]
    elif isinstance(header_part, bytes):
        validator = _HEADER_VALIDATORS_BYTE[header_validator_index]
    else:
        raise InvalidHeader(
            f"Header part ({header_part!r}) from {header} "
            f"must be of type str or bytes, not {type(header_part)}"
        )

    if not validator.match(header_part):
        header_kind = "name" if header_validator_index == 0 else "value"
        raise InvalidHeader(
            f"Invalid leading whitespace, reserved character(s), or return"
            f"character(s) in header {header_kind}: {header_part!r}"
        )

def urldefragauth(url):
    """
    Given a url remove the fragment and the authentication part.

    :rtype: str
    """
    scheme, netloc, path, params, query, fragment = urlparse(url)

    # see func:`prepend_scheme_if_needed`
    if not netloc:
        netloc, path = path, netloc

    netloc = netloc.rsplit("@", 1)[-1]

    return urlunparse((scheme, netloc, path, params, query, ""))

def rewind_body(prepared_request):
    """Move file pointer back to its recorded starting position
    so it can be read again on redirect.
    """
    body_seek = getattr(prepared_request.body, "seek", None)
    if body_seek is not None and isinstance(
        prepared_request._body_position, integer_types
    ):
        try:
            body_seek(prepared_request._body_position)
        except OSError:
            raise UnrewindableBodyError(
                "An error occurred when rewinding request body for redirect."
            )
    else:
        raise UnrewindableBodyError("Unable to rewind request body for redirect.")


from selenium import webdriver
from selenium.webdriver.common.by import By

driver = webdriver.Chrome()
driver.get('https://m.place.naver.com/restaurant/1239997161/home')

images_container = driver.find_element(By.CLASS_NAME, "CB8aP")
images = images_container.find_elements(By.CLASS_NAME, "K0PDV")

for image in images:
    print(image.value_of_css_property("background-image").split("\"")[1])
driver.quit()


"""
Author: Manish Saroya
Contact: saroyam@oregonstate.edu
"""
import networkx as nx
import numpy as np
from persistence.utils import load_hilbert_map, convert_map_dict_to_array, collision_check
import matplotlib.pyplot as pl
from sklearn.neighbors import NearestNeighbors
from bresenham import bresenham
import os
import argparse
import datetime
import pickle
from math import sqrt, log
import time

def save_img(data, graph, dir, save_data=True, save_graph=True, num_sample=0):
	"""
	:param data: map data dictionary
	:param graph: prm nx graph
	:param dir: output directory path
	:return: save prm graph and visualization image to output directory
	"""
	fig = pl.figure(figsize=(40/4, 35/4))
	ax = fig.add_subplot(111)
	pl.axis("equal")
	pl.scatter(data['Xq'][:, 0], data['Xq'][:, 1], c=data['yq'], cmap='viridis', s=70, vmin=0, vmax=1, edgecolors='')
	pl.colorbar()
	position = nx.get_node_attributes(graph, 'pos')
	for node_1, node_2 in graph.edges:
		weights = np.concatenate([[position[node_1]], [position[node_2]]])
		line, = pl.plot(*weights.T, color='lightsteelblue')
		pl.setp(line, linewidth=2, color='lightsteelblue')
	#pl.title('Test image')
	if save_data:
		pl.savefig(dir + str(num_sample) + "prm.png")
	if save_graph:
		with open(dir + str(num_sample) + 'prm.pickle', 'wb') as handle:
			pickle.dump(graph, handle)

def hilbert_samples(map_data, exp_factor, obs_threshold, num_samples=600):
	"""
	:param map_data: map info
	:param exp_factor: exponential factor to sample more from free space
	:param num_samples: number of samples required
	:return: samples
	"""
	map_data['yq'] = np.ones(len(map_data['yq'])) - map_data['yq']
	#map_data['yq'] = np.exp(exp_factor * map_data['yq'])
	map_data["yq"][map_data["yq"] < (1-obs_threshold)] = 0
	# normalize the probabilities
	map_data['yq'] /= np.linalg.norm(map_data['yq'], ord=1)

	samples_list = map_data['Xq'][np.random.choice(len(map_data['Xq']), size=num_samples, p=map_data['yq'])]
	return samples_list

if __name__ == "__main__":
	parser = argparse.ArgumentParser()
	parser.add_argument('--number_of_samples', type=int, default=15000)
	parser.add_argument('--exp_factor', type=int, default=20)
	parser.add_argument('--obstacle_threshold', type=float, default=0.25)
	parser.add_argument('--max_nodes', type=int, default=4000)
	parser.add_argument('--k_nearest', type=int, default=3900)
	parser.add_argument('--log_dir', type=str, default='./output')
	parser.add_argument('--connection_radius', type=float, default=5.0)
	parser.add_argument('--map_type', type=str, default="intel")
	parser.add_argument('--gamma_prm', type=float, default=30.0)
	#EDIT PRM STAR PARAM
	args = parser.parse_args()
	args.log_dir = './output/max_nodes-' + args.map_type + str(args.max_nodes) + "-obs-thres" + str(args.obstacle_threshold) +\
				   "-k_nearest-" + \
				   str(args.k_nearest) + "-connection_radius-" + str(args.connection_radius) + "-date-" + \
				   datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + '/'

	if not os.path.exists(args.log_dir):
		os.makedirs(args.log_dir)
	tic = time.time()
	# load map
	map_data, resolution = load_hilbert_map(map_type=args.map_type)
	#resolution = 0.2
	#with open("freiburg_ground_map_q_resolution_final.pickle", 'rb') as tf:
	#	map_data = pickle.load(tf)
	map_array = convert_map_dict_to_array(map_data, resolution)
	map_data["yq"] = 1.0 * (map_data["yq"] > args.obstacle_threshold)
	# get samples from hilbert maps
	sample_list = hilbert_samples(map_data.copy(), args.exp_factor, args.obstacle_threshold, num_samples=args.number_of_samples)
	# take unique samples
	sample_list = [list(t) for t in set(tuple(element) for element in sample_list)]
	# truncated based on max nodes
	sample_list = sample_list[:args.max_nodes]
	# find k nearest neighbor
	nbrs = NearestNeighbors(n_neighbors=args.k_nearest, algorithm='ball_tree').fit(sample_list)
	distances, indices = nbrs.kneighbors(sample_list)
	# create gragh
	prm_graph = nx.Graph()
	# add graph nodes
	for indx, s in enumerate(sample_list):
		prm_graph.add_node(indx, pos=(s[0], s[1]))
	# add graph edges
	for row, node_adjacency_list in enumerate(indices):
		args.connection_radius = args.gamma_prm * sqrt(log(row + 1) / (row+1))
		#print(row, args.connection_radius)
		for column, other_node in enumerate(node_adjacency_list):
			distance_metric = distances[row][column] < args.connection_radius
			if distance_metric:
				validation_metric = (node_adjacency_list[0] != other_node) and (node_adjacency_list[0] > other_node)
				if validation_metric:
					collision_metric = collision_check(map_array, sample_list[node_adjacency_list[0]],
													   sample_list[other_node], args.obstacle_threshold, resolution)
					if collision_metric:
						prm_graph.add_edge(node_adjacency_list[0], other_node, distance=distances[row][column])
			#if distance_metric and collision_metric and validation_metric:
				#prm_graph.add_edge(node_adjacency_list[0], other_node, distance=distances[row][column])
		if row == 500 or row == 1000 or row == 2000:
			print("rendering", str(row))
			print("generation time ", row , args.map_type, " ", time.time()-tic)
			save_img(map_data, prm_graph, args.log_dir, num_sample=row)
	print("rendering")
	print("generation time ", str(4000), args.map_type, " ", time.time() - tic)
	#save_img(map_data, prm_graph, args.log_dir, num_sample=4000)


"""
Tests for pandabase.

Note that the working directory is must be project_root_dir; when running tests from (for example) PyCharm,
the default may be project_root_dir/tests. In Pycharm this default can be set from Run/Debug Configurations:Templates

see, e.g.:
https://pythonspot.com/python-database-postgresql/
for postgres configuration commands to run postgres tests (use command 'pytest --run-postgres')
"""
import pytest

import pandabase as pb
from pandabase.helpers import *

import pytz

UTC = pytz.utc
LA_TZ = pytz.timezone('America/Los_Angeles')  # test timezone

bad_df = pd.DataFrame(index=[2, 2], data=['x', 'y'])
bad_df.index.name = 'bad_index'

bad_df2 = pd.DataFrame(index=[1, None], columns=['a'], data=['x', 'y'])
bad_df2.index.name = 'bad_index'

@pytest.mark.parametrize('df, how', [
    (bad_df, 'create_only'),
    (bad_df2, 'create_only'),
    (pd.Series([1, 2, 3]), 'create_only'),
    (pd.DataFrame(index=[1, 2, 3], columns=['a'], data=['x', 'y', 'a']), 'fake_mode'),
])
def test_invalid_inputs(df, how, empty_db):
    """all these inputs are invalid"""
    with pytest.raises(ValueError):
        pb.to_sql(df, table_name='blank', con=empty_db, how=how)

def test_get_sql_dtype_from_db(simple_df, empty_db):
    """test that datatype extraction functions work in pandas and SQL"""
    df = simple_df
    table = pb.to_sql(df,
                      table_name='sample',
                      con=empty_db,
                      how='create_only')

    for col in table.columns:
        if col.primary_key:
            # different syntax for index
            assert_sqla_types_equivalent(get_column_dtype(col, 'sqla'), get_column_dtype(df.index, 'sqla'))
            continue
        assert_sqla_types_equivalent(get_column_dtype(col, 'sqla'), get_column_dtype(df[col.name], 'sqla'))

def assert_sqla_types_equivalent(type1, type2):
    """weak equality test"""
    assert str(type1) == str(type2)

def test_get_sql_dtype_from_db_nans(simple_df_with_nans, empty_db):
    """test that datatype extraction functions work as expected"""
    df = simple_df_with_nans
    table = pb.to_sql(df,
                      table_name='sample',
                      con=empty_db,
                      how='create_only')

    for col in table.columns:
        if col.primary_key:
            # different syntax for index
            assert_sqla_types_equivalent(get_column_dtype(col, 'sqla'), get_column_dtype(df.index, 'sqla'))
            continue
        assert_sqla_types_equivalent(get_column_dtype(col, 'sqla'), get_column_dtype(df[col.name], 'sqla'))

@pytest.mark.parametrize('actually_do', [True, False])
def test_add_column_to_database(pandabase_loaded_db, actually_do, constants):
    """possibly add new column to db"""
    name = 'a_new_column'
    col = sqa.Column(name, primary_key=False, type_=Integer, nullable=True)
    if actually_do:
        pb.sql._add_columns_to_db(col, table_name=constants.TABLE_NAME, con=pandabase_loaded_db)
    df = pb.read_sql(table_name=constants.TABLE_NAME, con=pandabase_loaded_db)

    if actually_do:
        assert name in df.columns
        assert is_integer_dtype(df[name])
    else:
        assert name not in df.columns

def test_drop_table(pandabase_loaded_db):
    names = pb.util.get_db_table_names(pandabase_loaded_db)
    for name in names:
        assert pb.has_table(pandabase_loaded_db, table_name=name)
        pb.util.drop_db_table(con=pandabase_loaded_db, table_name=name)
        assert not pb.has_table(pandabase_loaded_db, table_name=name)

def test_get_tables(pandabase_loaded_db, constants):
    names = pb.util.get_db_table_names(pandabase_loaded_db)
    assert len(names) == 1
    assert names[0] == constants.TABLE_NAME

def test_get_columns(pandabase_loaded_db, simple_df, constants):
    cols = pb.util.get_table_column_names(pandabase_loaded_db, constants.TABLE_NAME)
    assert len(cols) == len(simple_df.columns) + 1
    assert constants.SAMPLE_INDEX_NAME in [col.name for col in cols]

def test_describe_db(pandabase_loaded_db, constants):
    desc = pb.util.describe_database(pandabase_loaded_db)
    assert len(desc) == 1  # 1 table in sample db
    assert desc[constants.TABLE_NAME]['min'] == 0  # min
    assert desc[constants.TABLE_NAME]['max'] == 5  # max
    assert desc[constants.TABLE_NAME]['count'] == 6  # count


#Echo server program
import socket
import pickle

HOST = '192.168.1.100'                 # Symbolic name meaning all available interfaces
PORT = 50007              # Arbitrary non-privileged port
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
    s.bind((HOST, PORT))
    s.listen(1)
    conn, addr = s.accept()
    with conn:
        print('Connected by', addr)
        while True:
            data = pickle.loads(conn.recv(1024))
            for i in range(len(data)):
                data[i] += 1
            print(data)
            if not data: break
            conn.sendall(pickle.dumps(data))


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
from torch.utils.tensorboard import SummaryWriter
from model import SkipGramLanguageModeler
import pickle
import numpy as np
import matplotlib.pyplot as plt
from helper import reduce_to_k_dim, plot_embeddings
import warnings
warnings.filterwarnings('ignore')

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
torch.cuda.get_device_name(device)

# In[2]:

with open('vocabulary.pickle', 'rb') as handle:
    vocab = pickle.load(handle)

with open('bigrams_dataset.pickle', 'rb') as handle:
    bigrams_dataset = pickle.load(handle)

word_to_ix = {word: i for i, word in enumerate(vocab)}
ix_to_word = { i: word for i, word in enumerate(vocab)}

x_data = []
y_data = []
for bigram in bigrams_dataset:
    x_data.append(word_to_ix[bigram[0]])
    y_data.append(word_to_ix[bigram[1]])

# In[3]:

class Word_dataset(Dataset):

    def __init__(self, x_data, y_data):
        self.x_data = torch.tensor(x_data).to(device)
        self.y_data = torch.tensor(y_data).to(device)
        self.length = len(x_data)

    def __getitem__(self,index):
        return self.x_data[index], self.y_data[index]

    def __len__(self):
        return self.length

train_dataset = Word_dataset(x_data, y_data)
train_loader = DataLoader(dataset=train_dataset, batch_size = 2**14+10000, shuffle=True)

iterator = iter(train_loader)
x, y = iterator.next()
(x.shape, y.shape)

# In[4]:

EMBEDDING_DIM = 300
net = SkipGramLanguageModeler(len(vocab), EMBEDDING_DIM)
net.to(device)
criterion = nn.NLLLoss()
optimizer = torch.optim.Adam(net.parameters())

# In[5]:

losses = []

iteration = 10001
k = 0
vocab_ind_np = torch.tensor(list(ix_to_word.keys())).cuda()
words = ['barrels', 'ecuador', 'energy', 'industry', 'kuwait', 'oil', 'output', 'petroleum', 'venezuela','free',
         'freedom','jury','favour','capable','capacity','capital', 'county','india','america', 'americans']

for epoch in range(iteration):
    total_loss = 0

    for data in train_loader:
        x, y = data
        log_probs = net.forward(x)
        loss = criterion(log_probs, y).to(device)
        loss.backward()
        optimizer.step()
        total_loss += loss.item()
        net.zero_grad()

    losses.append(total_loss)

    if epoch %3 == 0:
        print('Epoch: {} Loss: {}'.format(epoch, total_loss))

    if epoch %50 == 0:

        PATH = './saved_model/saved_model_{}.pth'.format(k)
        torch.save(net.state_dict(), PATH)
        print("\n Model Saved! at epoch no.: {}".format(epoch))
        k+=1

        embedding_matrix = net.embedding_outputs(vocab_ind_np).cpu().detach().numpy()
        embedding_matrix_reduced = reduce_to_k_dim(embedding_matrix, k=2)
        plot_embeddings(embedding_matrix_reduced, word_to_ix, words, epoch)

print('\n Done Training')

# In[ ]:


import argparse
import logging
import os
import yaml

from twisted.internet.defer import (
    inlineCallbacks, returnValue, fail, Deferred)

from juju.agents.unit import UnitAgent, CharmUpgradeOperation
from juju.agents.base import TwistedOptionNamespace
from juju.charm import get_charm_from_path
from juju.charm.url import CharmURL
from juju.errors import JujuError
from juju.state.environment import GlobalSettingsStateManager
from juju.state.errors import ServiceStateNotFound
from juju.state.service import NO_HOOKS, RETRY_HOOKS

from juju.agents.tests.common import AgentTestBase
from juju.control.tests.test_upgrade_charm import CharmUpgradeTestBase
from juju.hooks.tests.test_invoker import get_cli_environ_path
from juju.tests.common import get_test_zookeeper_address
from juju.unit.tests.test_charm import CharmPublisherTestBase
from juju.unit.tests.test_workflow import WorkflowTestBase

class UnitAgentTestBase(AgentTestBase, WorkflowTestBase):

    agent_class = UnitAgent

    @inlineCallbacks
    def setUp(self):
        yield super(UnitAgentTestBase, self).setUp()
        settings = GlobalSettingsStateManager(self.client)
        yield settings.set_provider_type("dummy")
        self.change_environment(
            PATH=get_cli_environ_path(),
            JUJU_UNIT_NAME="mysql/0")

    @inlineCallbacks
    def tearDown(self):
        if self.agent.api_socket:
            yield self.agent.api_socket.stopListening()
        yield super(UnitAgentTestBase, self).tearDown()

    @inlineCallbacks
    def get_agent_config(self):
        yield self.setup_default_test_relation()
        options = yield super(UnitAgentTestBase, self).get_agent_config()
        options["unit_name"] = str(self.states["unit"].unit_name)
        returnValue(options)

    def write_empty_hooks(self, start=True, stop=True, install=True, **kw):
        # NB Tests that use this helper method must properly wait on
        # the agent being stopped (yield self.agent.stopService()) to
        # avoid the environment being restored while asynchronously
        # the stop hook continues to execute. Otherwise
        # JUJU_UNIT_NAME, which hook invocation depends on, will
        # not be available and the stop hook will fail (somewhat
        # mysteriously!). The alternative is to set stop=False so that
        # the stop hook will not be created when writing the empty
        # hooks.
        output_file = self.makeFile()

        if install:
            self.write_hook(
                "install", "#!/bin/bash\necho install >> %s" % output_file)
        if start:
            self.write_hook(
                "start", "#!/bin/bash\necho start >> %s" % output_file)
        if stop:
            self.write_hook(
                "stop", "#!/bin/bash\necho stop >> %s" % output_file)

        for k in kw.keys():
            self.write_hook(k.replace("_", "-"),
                            "#!/bin/bash\necho $0 >> %s" % output_file)

        return output_file

    def parse_output(self, output_file):
        return filter(None, open(output_file).read().split("\n"))

class UnitAgentTest(UnitAgentTestBase):

    @inlineCallbacks
    def test_agent_start_stop_service(self):
        """Verify workflow state when starting and stopping the unit agent."""
        self.write_empty_hooks()
        yield self.agent.startService()
        current_state = yield self.agent.workflow.get_state()
        self.assertEqual(current_state, "started")
        yield self.agent.stopService()
        current_state = yield self.agent.workflow.get_state()
        self.assertEqual(current_state, "stopped")

    def test_agent_unit_name_environment_extraction(self):
        """Verify extraction of unit name from the environment."""
        self.change_args("unit-agent")
        self.change_environment(JUJU_UNIT_NAME="rabbit/1")
        parser = argparse.ArgumentParser()
        self.agent.setup_options(parser)
        options = parser.parse_args(namespace=TwistedOptionNamespace())
        self.assertEqual(options["unit_name"], "rabbit/1")

    def test_agent_unit_name_cli_extraction_error(self):
        """Failure to extract the unit name, results in a nice error message.
        """
        # We don't want JUJU_UNIT_NAME set, so that the expected
        # JujuError will be raised
        self.change_environment(
            PATH=get_cli_environ_path())
        self.change_args(
            "unit-agent",
            "--juju-directory", self.makeDir(),
            "--zookeeper-servers", get_test_zookeeper_address())

        parser = argparse.ArgumentParser()
        self.agent.setup_options(parser)
        options = parser.parse_args(namespace=TwistedOptionNamespace())

        e = self.assertRaises(JujuError,
                              self.agent.configure,
                              options)
        self.assertEquals(
            str(e),
            "--unit-name must be provided in the command line, or "
            "$JUJU_UNIT_NAME in the environment")

    def test_agent_unit_name_cli_extraction(self):
        """The unit agent can parse its unit-name from the cli.
        """
        self.change_args("unit-agent", "--unit-name", "rabbit/1")
        parser = argparse.ArgumentParser()
        self.agent.setup_options(parser)
        options = parser.parse_args(namespace=TwistedOptionNamespace())
        self.assertEqual(options["unit_name"], "rabbit/1")

    def test_get_agent_name(self):
        self.assertEqual(self.agent.get_agent_name(), "unit:mysql/0")

    def test_agent_invalid_unit_name(self):
        """If the unit agent is given an invalid unit name, an error
        message is raised."""
        options = {}
        options["juju_directory"] = self.juju_directory
        options["zookeeper_servers"] = get_test_zookeeper_address()
        options["unit_name"] = "rabbit-1"
        agent = self.agent_class()
        agent.configure(options)
        return self.assertFailure(agent.startService(), ServiceStateNotFound)

    @inlineCallbacks
    def test_agent_records_address_on_startup(self):
        """On startup the agent will record the unit's addresses.
        """
        yield self.agent.startService()
        self.assertEqual(
            (yield self.agent.unit_state.get_public_address()),
            "localhost")
        self.assertEqual(
            (yield self.agent.unit_state.get_private_address()),
            "localhost")

    @inlineCallbacks
    def test_agent_agent_executes_install_and_start_hooks_on_startup(self):
        """On initial startup, the unit agent executes install and start hooks.
        """
        output_file = self.write_empty_hooks()
        hooks_complete = self.wait_on_hook(
            sequence=["install", "config-changed", "start"],
            executor=self.agent.executor)
        yield self.agent.startService()
        # Verify the hook has executed.
        yield hooks_complete
        # config-changed is not mentioned in the output below as the
        # hook is optional and not written by default
        self.assertEqual(self.parse_output(output_file),
                         ["install", "start"])
        yield self.assertState(self.agent.workflow, "started")
        yield self.agent.stopService()

    @inlineCallbacks
    def test_agent_install_error_transitions_install_error(self):
        self.write_hook("install", "!/bin/bash\nexit 1\n")

        hooks_complete = self.wait_on_hook(
            "install",
            executor=self.agent.executor)

        yield self.agent.startService()

        # Verify the hook has executed.
        yield hooks_complete
        yield self.assertState(self.agent.workflow, "install_error")

    @inlineCallbacks
    def test_agent_executes_stop_hook_on_shutdown(self):
        """On shutdown, the agent stops the hook."""
        output_file = self.write_empty_hooks()
        yield self.agent.startService()
        hooks_complete = self.wait_on_hook(
            "stop", executor=self.agent.executor)
        yield self.agent.stopService()
        # Verify the hook has executed.
        yield hooks_complete
        self.assertEqual(self.parse_output(output_file),
                         ["install", "start", "stop"])
        yield self.assertState(self.agent.workflow, "stopped")

        # verify workflow state.
        f_state, history, zk_state = yield self.read_persistent_state(
            workflow=self.agent.workflow)
        self.assertEqual(f_state, zk_state)
        self.assertEqual(f_state, {"state": "stopped", "state_variables": {}})

    @inlineCallbacks
    def test_agent_executes_relation_changed_hook(self):
        """If a relation changes after the unit is started, a relation change
        hook is executed."""
        self.write_empty_hooks()
        file_path = self.makeFile()
        self.write_hook("app-relation-changed",
                ("#!/bin/sh\n"
                 "echo $JUJU_REMOTE_UNIT >> %s\n" % file_path))
        yield self.agent.startService()

        hook_complete = self.wait_on_hook(
            "app-relation-changed", executor=self.agent.executor)
        wordpress_states = yield self.add_opposite_service_unit(
            self.states)

        # Verify the hook has executed.
        yield hook_complete
        self.assertEqual(open(file_path).read().strip(),
                         wordpress_states["unit"].unit_name)

    @inlineCallbacks
    def test_agent_executes_config_changed_hook(self):
        """Service config changes fire a config-changed hook."""
        self.agent.set_watch_enabled(True)
        self.write_empty_hooks()
        file_path = self.makeFile()
        self.write_hook("config-changed",
                ("#!/bin/sh\n"
                 "config-get foo >> %s\n" % file_path))
        yield self.agent.startService()

        transition_complete = self.wait_on_state(
            self.agent.workflow, "started")

        service = self.states["service"]
        config = yield service.get_config()
        config["foo"] = "bar"
        yield config.write()

        # Verify the hook has executed, and transition has completed.
        yield transition_complete
        self.assertEqual(open(file_path).read().strip(), "bar")

    @inlineCallbacks
    def test_agent_can_execute_config_changed_in_relation_hook(self):
        """Service config changes fire a config-changed hook."""
        self.agent.set_watch_enabled(True)

        self.write_empty_hooks()
        file_path = self.makeFile()

        self.write_hook("app-relation-changed",
                ("#!/bin/sh\n"
                 "config-get foo >> %s\n" % file_path))

        # set service config
        service = self.states["service"]
        config = yield service.get_config()
        config["foo"] = "bar"
        yield config.write()

        yield self.agent.startService()
        hook_complete = self.wait_on_hook(
            "app-relation-changed", executor=self.agent.executor)

        # trigger the hook that will read service options
        yield self.add_opposite_service_unit(self.states)

        # Verify the hook has executed.
        yield hook_complete
        self.assertEqual(open(file_path).read().strip(), "bar")

    @inlineCallbacks
    def test_agent_hook_api_usage(self):
        """If a relation changes after the unit is started, a relation change
        hook is executed."""
        self.write_empty_hooks()
        file_path = self.makeFile()
        self.write_hook("app-relation-changed",
                        "\n".join(
                            ["#!/bin/sh",
                             "echo `relation-list` >> %s" % file_path,
                             "echo `relation-set greeting=hello`",
                             "echo `relation-set planet=earth`",
                             "echo `relation-get planet %s` >> %s" % (
                                 self.states["unit"].unit_name, file_path)]))

        yield self.agent.startService()

        hook_complete = self.wait_on_hook(
            "app-relation-changed", executor=self.agent.executor)
        yield self.add_opposite_service_unit(self.states)

        # Verify the hook has executed.
        yield hook_complete

        # Verify hook output
        output = open(file_path).read().strip().split("\n")
        self.assertEqual(output, ["wordpress/0", "earth"])

        # Verify zookeeper state
        contents = yield self.states["unit_relation"].get_data()
        self.assertEqual(
            {"greeting": "hello", "planet": "earth",
             "private-address": "mysql-0.example.com"},
            yaml.load(contents))

        self.failUnlessIn("wordpress/0", output)

    @inlineCallbacks
    def test_agent_executes_depart_hook(self):
        """If a relation changes after the unit is started, a relation change
        hook is executed."""
        self.write_empty_hooks(app_relation_changed=True)
        file_path = self.makeFile()
        self.write_hook("app-relation-broken",
                ("#!/bin/sh\n"
                 "echo broken hook >> %s\n" % file_path))
        yield self.agent.startService()

        hook_complete = self.wait_on_hook(
            "app-relation-changed", executor=self.agent.executor)

        yield self.add_opposite_service_unit(self.states)
        yield hook_complete

        # Watch the unit relation workflow complete
        workflow_complete = self.wait_on_state(
            self.agent.lifecycle.get_relation_workflow(
                self.states["relation"].internal_id),
            "departed")

        yield self.relation_manager.remove_relation_state(
            self.states["relation"])

        hook_complete = self.wait_on_hook(
            "app-relation-broken", executor=self.agent.executor)

        # Verify the hook has executed.
        yield hook_complete
        self.assertEqual(open(file_path).read().strip(), "broken hook")

        # Wait for the workflow transition to complete.
        yield workflow_complete

    @inlineCallbacks
    def test_agent_debug_watch(self):
        """The unit agent subscribes to changes to the hook debug settings.
        """
        self.agent.set_watch_enabled(True)
        yield self.agent.startService()
        yield self.states["unit"].enable_hook_debug(["*"])
        # Wait for watch to fire invoke callback and reset
        yield self.sleep(0.1)
        # Check the propogation to the executor
        self.assertNotEquals(
            self.agent.executor.get_hook_path("x"), "x")

class UnitAgentResolvedTest(UnitAgentTestBase):

    @inlineCallbacks
    def test_resolved_unit_already_running(self):
        """If the unit already running the setting is cleared,
        and no transition is performed.
        """
        self.write_empty_hooks()
        start_deferred = self.wait_on_hook(
            "start", executor=self.agent.executor)

        self.agent.set_watch_enabled(True)
        yield self.agent.startService()
        yield start_deferred
        self.assertEqual(
            "started", (yield self.agent.workflow.get_state()))

        yield self.agent.unit_state.set_resolved(RETRY_HOOKS)
        # Wait for watch to fire and reset
        yield self.sleep(0.1)

        self.assertEqual(
            "started", (yield self.agent.workflow.get_state()))
        self.assertEqual(
            None, (yield self.agent.unit_state.get_resolved()))

    @inlineCallbacks
    def test_resolved_install_error(self):
        """If the unit has an install error it will automatically
        be transitioned to the installed state after the recovery.
        """
        self.write_empty_hooks()
        install_deferred = self.wait_on_hook(
            "install", executor=self.agent.executor)
        self.write_hook("install", "#!/bin/sh\nexit 1")

        self.agent.set_watch_enabled(True)
        yield self.agent.startService()

        yield install_deferred
        self.assertEqual(
            "install_error", (yield self.agent.workflow.get_state()))

        install_deferred = self.wait_on_state(self.agent.workflow, "started")
        self.write_hook("install", "#!/bin/sh\nexit 0")
        yield self.agent.unit_state.set_resolved(RETRY_HOOKS)
        yield install_deferred
        self.assertEqual("started", (yield self.agent.workflow.get_state()))
        # Ensure we clear out background activity from the watch firing
        yield self.poke_zk()

    @inlineCallbacks
    def test_resolved_start_error(self):
        """If the unit has a start error it will automatically
        be transitioned to started after the recovery.
        """
        self.write_empty_hooks()
        hook_deferred = self.wait_on_hook(
            "start", executor=self.agent.executor)
        self.write_hook("start", "#!/bin/sh\nexit 1")

        self.agent.set_watch_enabled(True)
        yield self.agent.startService()

        yield hook_deferred
        self.assertEqual(
            "start_error", (yield self.agent.workflow.get_state()))

        state_deferred = self.wait_on_state(self.agent.workflow, "started")
        yield self.agent.unit_state.set_resolved(NO_HOOKS)
        yield state_deferred
        self.assertEqual("started", (yield self.agent.workflow.get_state()))
        # Resolving to the started state from the resolved watch will cause the
        # lifecycle start to execute in the background context, wait
        # for it to finish.
        yield self.sleep(0.1)

    @inlineCallbacks
    def test_resolved_stopped(self):
        """If the unit has a stop error it will automatically
        be transitioned to stopped after the recovery.
        """
        self.write_empty_hooks()
        self.write_hook("stop", "#!/bin/sh\nexit 1")

        hook_deferred = self.wait_on_hook(
            "start", executor=self.agent.executor)
        self.agent.set_watch_enabled(True)
        yield self.agent.startService()
        yield hook_deferred

        hook_deferred = self.wait_on_hook("stop", executor=self.agent.executor)
        yield self.agent.workflow.fire_transition("stop")
        yield hook_deferred

        self.assertEqual("stop_error", (yield self.agent.workflow.get_state()))

        state_deferred = self.wait_on_state(self.agent.workflow, "stopped")
        self.write_hook("stop", "#!/bin/sh\nexit 0")
        yield self.agent.unit_state.set_resolved(RETRY_HOOKS)
        yield state_deferred
        self.assertEqual("stopped", (yield self.agent.workflow.get_state()))
        # Ensure we clear out background activity from the watch firing
        yield self.poke_zk()

    @inlineCallbacks
    def test_hook_error_on_resolved_retry_remains_in_error_state(self):
        """If the unit has an install error it will automatically
        be transitioned to started after the recovery.
        """
        self.write_empty_hooks()
        self.write_hook("stop", "#!/bin/sh\nexit 1")

        hook_deferred = self.wait_on_hook(
            "start", executor=self.agent.executor)
        self.agent.set_watch_enabled(True)
        yield self.agent.startService()
        yield hook_deferred

        hook_deferred = self.wait_on_hook("stop", executor=self.agent.executor)
        yield self.agent.workflow.fire_transition("stop")
        yield hook_deferred

        self.assertEqual("stop_error", (yield self.agent.workflow.get_state()))

        hook_deferred = self.wait_on_hook("stop", executor=self.agent.executor)
        yield self.agent.unit_state.set_resolved(RETRY_HOOKS)
        yield hook_deferred
        # Ensure we clear out background activity from the watch firing
        yield self.poke_zk()
        self.assertEqual("stop_error", (yield self.agent.workflow.get_state()))

class UnitAgentUpgradeTest(
    UnitAgentTestBase, CharmPublisherTestBase, CharmUpgradeTestBase):

    @inlineCallbacks
    def setUp(self):
        yield super(UnitAgentTestBase, self).setUp()
        settings = GlobalSettingsStateManager(self.client)
        yield settings.set_provider_type("dummy")
        self.makeDir(path=os.path.join(self.juju_directory, "charms"))

    @inlineCallbacks
    def mark_charm_upgrade(self):
        # Create a new version of the charm
        repository = self.increment_charm(self.charm)

        # Upload the new charm version
        charm = yield repository.find(CharmURL.parse("local:series/mysql"))
        charm, charm_state = yield self.publish_charm(charm.path)

        # Mark the unit for upgrade
        yield self.states["service"].set_charm_id(charm_state.id)
        yield self.states["unit"].set_upgrade_flag()

    @inlineCallbacks
    def test_agent_upgrade_watch(self):
        """The agent watches for unit upgrades."""
        yield self.mark_charm_upgrade()
        self.agent.set_watch_enabled(True)
        hook_done = self.wait_on_hook(
            "upgrade-charm", executor=self.agent.executor)
        yield self.agent.startService()
        yield hook_done
        yield self.assertState(self.agent.workflow, "started")

    @inlineCallbacks
    def test_agent_upgrade_watch_continues_on_unexpected_error(self):
        """The agent watches for unit upgrades and continues if there is an
        unexpected error."""
        yield self.mark_charm_upgrade()
        self.agent.set_watch_enabled(True)

        output = self.capture_logging(
            "juju.agents.unit", level=logging.DEBUG)

        upgrade_done = Deferred()

        def operation_has_run():
            upgrade_done.callback(True)

        operation = self.mocker.patch(CharmUpgradeOperation)
        operation.run()

        self.mocker.call(operation_has_run)
        self.mocker.result(fail(ValueError("magic mouse")))
        self.mocker.replay()

        yield self.agent.startService()

        yield upgrade_done
        self.assertIn("Error while upgrading", output.getvalue())
        self.assertIn("magic mouse", output.getvalue())

        yield self.agent.workflow.fire_transition("stop")

    @inlineCallbacks
    def test_agent_upgrade(self):
        """The agent can succesfully upgrade its charm."""
        self.agent.set_watch_enabled(False)
        yield self.agent.startService()

        yield self.mark_charm_upgrade()

        hook_done = self.wait_on_hook(
            "upgrade-charm", executor=self.agent.executor)
        self.write_hook("upgrade-charm", "#!/bin/bash\nexit 0")
        output = self.capture_logging("unit.upgrade", level=logging.DEBUG)

        # Do the upgrade
        upgrade = CharmUpgradeOperation(self.agent)
        value = yield upgrade.run()

        # Verify the upgrade.
        self.assertIdentical(value, True)
        self.assertIn("Unit upgraded", output.getvalue())
        yield hook_done

        new_charm = get_charm_from_path(
            os.path.join(self.agent.unit_directory, "charm"))

        self.assertEqual(
            self.charm.get_revision() + 1, new_charm.get_revision())

    @inlineCallbacks
    def test_agent_upgrade_bad_unit_state(self):
        """The an upgrade fails if the unit is in a bad state."""
        self.agent.set_watch_enabled(False)
        yield self.agent.startService()

        # Upload a new version of the unit's charm
        repository = self.increment_charm(self.charm)
        charm = yield repository.find(CharmURL.parse("local:series/mysql"))
        charm, charm_state = yield self.publish_charm(charm.path)

        # Mark the unit for upgrade, with an invalid state.
        yield self.states["service"].set_charm_id(charm_state.id)
        yield self.states["unit"].set_upgrade_flag()
        yield self.agent.workflow.set_state("start_error")

        output = self.capture_logging("unit.upgrade", level=logging.DEBUG)

        # Do the upgrade
        upgrade = CharmUpgradeOperation(self.agent)
        value = yield upgrade.run()

        # Verify the upgrade.
        self.assertIdentical(value, False)
        self.assertIn("Unit not in an upgradeable state: start_error",
                      output.getvalue())
        self.assertIdentical(
            (yield self.states["unit"].get_upgrade_flag()),
            False)

    @inlineCallbacks
    def test_agent_upgrade_no_flag(self):
        """An upgrade fails if there is no upgrade flag set."""
        self.agent.set_watch_enabled(False)
        yield self.agent.startService()
        output = self.capture_logging("unit.upgrade", level=logging.DEBUG)
        upgrade = CharmUpgradeOperation(self.agent)
        value = yield upgrade.run()
        self.assertIdentical(value, False)
        self.assertIn("No upgrade flag set", output.getvalue())
        yield self.agent.workflow.fire_transition("stop")

    @inlineCallbacks
    def test_agent_upgrade_version_current(self):
        """An upgrade fails if the unit is running the latest charm."""
        self.agent.set_watch_enabled(False)
        yield self.agent.startService()
        yield self.states["unit"].set_upgrade_flag()
        output = self.capture_logging("unit.upgrade", level=logging.DEBUG)
        upgrade = CharmUpgradeOperation(self.agent)
        value = yield upgrade.run()
        self.assertIdentical(value, True)
        self.assertIn("Unit already running latest charm", output.getvalue())
        self.assertFalse((yield self.states["unit"].get_upgrade_flag()))

    @inlineCallbacks
    def test_agent_upgrade_hook_failure(self):
        """An upgrade fails if the upgrade hook errors."""
        self.agent.set_watch_enabled(False)
        yield self.agent.startService()

        # Upload a new version of the unit's charm
        repository = self.increment_charm(self.charm)
        charm = yield repository.find(CharmURL.parse("local:series/mysql"))
        charm, charm_state = yield self.publish_charm(charm.path)

        # Mark the unit for upgrade
        yield self.states["service"].set_charm_id(charm_state.id)
        yield self.states["unit"].set_upgrade_flag()

        hook_done = self.wait_on_hook(
            "upgrade-charm", executor=self.agent.executor)
        self.write_hook("upgrade-charm", "#!/bin/bash\nexit 1")
        output = self.capture_logging("unit.upgrade", level=logging.DEBUG)

        # Do the upgrade
        upgrade = CharmUpgradeOperation(self.agent)
        value = yield upgrade.run()

        # Verify the failed upgrade.
        self.assertIdentical(value, False)
        self.assertIn("Invoking upgrade transition", output.getvalue())
        self.assertIn("Upgrade failed.", output.getvalue())
        yield hook_done

        # Verify state
        workflow_state = yield self.agent.workflow.get_state()
        self.assertEqual("charm_upgrade_error", workflow_state)

        # Verify new charm is in place
        new_charm = get_charm_from_path(
            os.path.join(self.agent.unit_directory, "charm"))

        self.assertEqual(
            self.charm.get_revision() + 1, new_charm.get_revision())

        # Verify upgrade flag is cleared.
        self.assertFalse((yield self.states["unit"].get_upgrade_flag()))


# Loading all necessary libraries

import matplotlib.pyplot as plt
import pandas as pd
import re
import csv
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from collections import Counter
set(stopwords.words('english'))

# We are getting our data for categories and polarity, for the further sentiment analysis
category = pd.read_csv(r"C:/Users/Diana/Desktop/AUA/hpc/category.csv", header=None)
polarity = pd.read_csv(r"C:/Users/Diana/Desktop/AUA/hpc/polarity.csv", header=None)
df = pd.concat([category, polarity], axis=1)
df.columns = ['Category', 'Polarity']

# Here we are plotting the percentage of tweets for each category

fig1 = plt.figure()
df.Category.value_counts().plot(kind='pie', autopct='%1.0f%%')
plt.title('The percentage of tweets for each category')
fig1.savefig(r"C:/Users/Diana/Desktop/AUA/hpc/img1.png")
plt.show()
plt.close(fig1)

# This snippet of code generates the plot for the distribution of sentiments across all the tweets

fig2 = plt.figure()
df.Polarity.value_counts().plot(kind='pie', autopct='%1.0f%%', colors=["red", "yellow", "green"])
plt.title('The distribution of sentiments across all the tweets')
plt.ylabel(' ')
fig2.savefig(r"C:/Users/Diana/Desktop/AUA/hpc/img2.png")
plt.show()

# Finally, we draw the distribution of sentiment for each category
df.groupby(['Category', 'Polarity']).Polarity.count().unstack().plot(kind='bar')
plt.title('The distribution of sentiment for each category')
plt.grid(False)
plt.savefig(r"C:/Users/Diana/Desktop/AUA/hpc/img3.png")
plt.show()

# Cleaning our data from tweets

def clean_tweet(tweet):
    tweet = re.sub('http\S+\s*', '', tweet)  # remove URLs
    tweet = re.sub('RT|cc', '', tweet)  # remove RT and cc
    tweet = re.sub('#\S+', '', tweet)  # remove hash tags
    tweet = re.sub('@\S+', '', tweet)  # remove mentions
    tweet = re.sub(r"\s+[a-zA-Z]\s+", " ", tweet)
    tweet = re.sub('[%s]' % re.escape("""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""), '', tweet)  # remove punctuations
    tweet = re.sub('\s+', ' ', tweet)  # remove extra whitespace
    tweet = re.sub('(?<=[a-z])\'(?=[a-z])', '', tweet)
    tweet = tweet.replace('’', '')  # remove apostrophes
    return tweet

def remove_emoji(string):
    emoji_pattern = re.compile("["
                               u"\U0001F600-\U0001F64F"  # emoticons
                               u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                               u"\U0001F680-\U0001F6FF"  # transport & map symbols
                               u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                               u"\U00002702-\U000027B0"
                               u"\U000024C2-\U0001F251"

                               "]+", flags=re.UNICODE)

    return emoji_pattern.sub(r'', string)

# We get our data from the csv file then
# We are removing all emoticons and other redundant characters from our data
array = []
with open(r"C:/Users/Diana/Desktop/AUA/hpc/tweets.csv", encoding='utf-8-sig') as csv_file:
    df = csv.reader(csv_file)
    for row in df:
        row[0] = clean_tweet(row[0])
        row[0] = remove_emoji(row[0])
        row[0] = row[0].lower()
        array.append(row[0])

# Removing stopwords from the data set
data_set = ''.join(array)
stop_words = set(stopwords.words('english'))
word_tokens = word_tokenize(data_set)
filtered_data = [w for w in word_tokens if not w in stop_words]
filtered_data = []

for w in word_tokens:
    if w not in stop_words:
        filtered_data.append(w)

final_data = ' '.join(filtered_data)

# Now we are getting 10 most frequent words from our cleaned data
# split() returns list of all the words in the string
split_it = final_data.split()

# We pass the split_it list to instance of Counter class, this counts the occurrence of each word in the string.
Counter = Counter(split_it)

# most_common(10) produces 10 frequently encountered words and their respective counts.
most_occur = Counter.most_common(10)

# We specify labels and frequencies for plotting the graph of 10 frequently encountered words.
label = []
frequency = []
for word in most_occur:
    label.append(word[0].capitalize())
    frequency.append(word[1])

# Here we are plotting the graph
labels = label[0:10]
fig4 = plt.figure(figsize=(10, 5))
plt.bar(labels, frequency[0:10], width=0.8, alpha=0.5, capsize=8, linewidth=1)
plt.xlabel("10 more frequent words", size=14)
plt.ylabel("Frequency", size=14)
plt.title('Word Frequencies', size=18)
plt.grid(False)
plt.gca().spines["top"].set_visible(False)
plt.gca().spines["right"].set_visible(False)
fig4.savefig(r"C:/Users/Diana/Desktop/AUA/hpc/img4.png")
plt.show()
plt.close(fig4)

# Now we create and generate a word cloud image.
plt.figure(figsize=(10, 10))
wordcloud = WordCloud(width=800, height=500, max_font_size=80, max_words=150, background_color="white",
                      contour_color="steelblue", colormap="nipy_spectral").generate(final_data)
plt.imshow(wordcloud, interpolation="hermite")
plt.axis("off")
plt.show()

# Now we save the image
wordcloud.to_file(r"C:/Users/Diana/Desktop/AUA/hpc/wordcloud_img.png")


# -*- coding: utf-8 -*-
"""
Created on Mon May 14 16:24:53 2018

@author: avarfolomeev
"""

for a in np.arange(000,35000,100):
    a2.plot([0, 10*np.cos(yaws[a])],[0, 10*np.sin(yaws[a])],color=colors20[a])


import lru_cache.Node as Node

class LRUCache:
    """
    Least Recently Used (LRU) Cache Implementation in Python
    The LRU Cache is initialized with a maximum size.  During use, if the
    max size is exceeded when adding new keys, the least recently used
    item is discarded from the cache to make room for the new item.

    Assumptions:
    - Minimum size = 1
    - Maximum size = any integer greater than 1
    - Get() operation return -1 if key not present in cache
    - Get() and Put() operation on an item that exists in the cache causes
      the item to become the most recently used item (moves to tail of list)

    This LRU Cache is implemented as a doubly linked list where the least
    recently used item is at the head of the list, and the most recently used
    item is at the tail of the list.
    """
    def __init__(self, maxsize):
        self.maxsize = maxsize # number of max items
        self.currentsize = 0 # holds current items in cache

        self.dictionary = {} # stores nodes by key

        self.head = None # least recent request (oldest)
        self.tail = None # most recent request (newest)

    def put(self, key, value):
        """
        Adds a value to the lru cache.
        If there is space available - the item is appended to the end
        If the cache is full - the least recently used item is removed to make room for the new item
        """

        if key in self.dictionary:
            # key exists - move to end of list only if not already at the end
            # i.e. make most recent
            self._makeRecentlyUsed(key)
        else:
            if self.currentsize != self.maxsize:

                # this is most recent - append to end
                self._append(key, value)

            else:
                # cache is full
                # remove the least recently used
                leastRecentlyUsed = self.head
                self.delete(leastRecentlyUsed.key)

                # append to end
                self._append(key, value)

    def get(self, key):
        """
        Gets the value for a key in the cache
        """
        value = -1
        if key in self.dictionary:
            node = self.dictionary[key]
            value = node.value

            if node.nextNode != None: # not the tail
                self._makeRecentlyUsed(key)

        return value

    def delete(self, key):
        """
        Delete a key
        """
        if key in self.dictionary:
            node = self.dictionary.pop(key)
            self.currentsize -= 1

            if node.nextNode == None and node.prevNode != None: # last node
                prevNode = node.prevNode
                prevNode.nextNode = None
                self.tail = prevNode

            elif node.prevNode == None and node.nextNode != None: # first node
                nextNode = node.nextNode
                nextNode.prevNode = None
                self.head = nextNode

            elif node.prevNode != None and node.nextNode != None:
                node.prevNode.nextNode = node.nextNode
                node.nextNode.prevNode = node.prevNode

            elif node.prevNode == None and node.nextNode == None: # only one node
                self.head = None
                self.tail = None

    def reset(self):
        """Clears the cache"""
        if self.currentsize > 0:
            currentNode = self.head
            while currentNode != None:
                nextNode = currentNode.nextNode
                currentNode.nextNode = None
                currentNode.prevNode = None
                currentNode.value = None
                currentNode = nextNode

            self.head = None
            self.tail = None
            self.dictionary.clear()
            self.currentsize = 0

    def _append(self, key, value):
        """
        Appends a node to the end of the lru cache making it the
        most recently used item.
        """
        node = Node.ListNode(key, value)

        #there are no values in the linked list
        #make this node the head node and return
        if self.head is None:
            self.head = node
            self.tail = node
            self.dictionary[key] = node
            self.currentsize += 1
            return

        lastNode = self.tail
        lastNode.nextNode = node
        node.prevNode = lastNode
        self.tail = node

        self.dictionary[node.key] = node
        self.currentsize += 1

    def _makeRecentlyUsed(self, key):
        """
        Makes a node recently used by moving it to the end of the cache
        """
        node = self.dictionary[key]

        if node.nextNode != None: # not the tail
            if node.prevNode == None and node.nextNode != None: # head node before another node
                nextNode = node.nextNode
                nextNode.prevNode = None
                self.head = nextNode
            elif node.prevNode != None and node.nextNode != None: # in between nodes
                node.prevNode.nextNode = node.nextNode
                node.nextNode.prevNode = node.prevNode

            # place after last node - make it the tail
            lastNode = self.tail
            lastNode.nextNode = node
            node.prevNode = lastNode
            node.nextNode = None
            self.tail = node

    def _printForward(self):
        """
        Helper function - return contents of cache, forward direction
        """
        contents = []
        node = self.head

        while (node != None):
            contents.append(node.value)
            node = node.nextNode

        return contents

    def _printReverse(self):
        """
        Helper function - return contents of cache, reverse direction
        """
        node = self.tail
        contents = []

        while (node != None):
            contents.append(node.value)
            node = node.prevNode

        return contents


# -*- coding: utf-8 -*-
"""
Created on Fri Nov 24 12:52:26 2017

@author: eh94
"""

from bokeh.layouts import layout
from bokeh.models import PanTool, ResetTool, WheelZoomTool, SaveTool, TapTool, ColumnDataSource, Label
from bokeh.models.widgets import DataTable, TableColumn, Button, TextInput
from bokeh.plotting import figure
from bokeh.io import curdoc
from bokeh.models.callbacks import CustomJS
import numpy as np
from numpy import exp, log10, log, sin, sinh, sinc, cos, cosh, tan, tanh
from scipy import optimize
import pandas as pd

#initialize source data
filepath='data.xlsx'
filetype=filepath.split('.')[1]
if filetype=='csv':
    data=pd.read_csv(filepath, sep=';')
elif filetype in ['xlsx', 'xslm', 'xls', 'xlsb', 'xlt']:
    data=pd.read_excel(filepath, sep=';')
else:
    print('*.{} is invalid filetype!'.format(filetype))
columns = data.columns
source = ColumnDataSource(data=dict(x=data[columns[0]], y=data[columns[1]]))
table_source=ColumnDataSource(data=data)

#create add_button callback
def add_row():
    new_data=dict(x=[0.0], y=[0.0])
    source.stream(new_data)

#create del_button callback
def del_row():
    new_data=source.data[:-1]
    source.data=new_data

#create fit function
def fit_data():
    #define the fit function
    func=lambda x,a,b,c,d,e: eval(model_text_input.value)

    #get indices of selected data points
    indices=source.selected['1d']['indices']
    indices.sort()

    if not indices:
        x=np.array(source.data['x'])
        y=np.array(source.data['y'])
    else:
        #get x and y values of selected data points
        x=np.array([source.data['x'][index] for index in indices])
        y=np.array([source.data['y'][index] for index in indices])

    #fit the selected data points to predefined function
    parameter, variance = optimize.curve_fit(func, x, y)

    #plot the fit
    fig.line(x=source.data['x'], y=func(source.data['x'],*parameter), line_width=3, line_color='firebrick', line_alpha=1)

    parameter_text_input.value=str(parameter)
    #illustrate the parameter in textboxes
#    amplitude=Label(x=20, y=300, x_units='screen', y_units='screen', text='Amp.: {:.2f}'.format(parameter[0]), text_font_style='bold')
#    mean=Label(x=20, y=280, x_units='screen', y_units='screen', text='Mean: {:.2f}'.format(parameter[1]), text_font_style='bold')
#    std=Label(x=20, y=260, x_units='screen', y_units='screen', text='Std. Dev.: {:.2f}'.format(parameter[2]), text_font_style='bold')
#    fig.add_layout(amplitude)
#    fig.add_layout(mean)
#    fig.add_layout(std)

#create update function
def update():
    return

#create table widget
table_columns = [TableColumn(field=column, title=column) for column in columns]
table = DataTable(source=table_source, columns=table_columns, editable=True, width=400, height=600)

#create figure
fig = figure(tools=[PanTool(), WheelZoomTool(), ResetTool(), SaveTool(), TapTool()],output_backend='webgl')
fig.plot_width = 900
fig.plot_height = 600
fig.toolbar.logo = None

#style axis
fig.axis.minor_tick_line_color='black'
fig.axis.minor_tick_in=-6
fig.xaxis.axis_label=columns[0]
fig.yaxis.axis_label=columns[1]
fig.axis.axis_label_text_color=(0.7,0.7,0.7)
fig.axis.major_label_text_color=(0.7,0.7,0.7)
fig.axis.axis_label_text_font = 'helvetica'
fig.xaxis.axis_label_text_font_size = '16pt'
fig.yaxis.axis_label_text_font_size = '16pt'
fig.axis.axis_label_text_font_style = 'normal'
fig.axis.major_label_text_font = 'helvetica'
fig.axis.major_label_text_font_size = '10pt'
fig.axis.major_label_text_font_style = 'normal'

#plot data
fig.circle(x='x', y='y', size=10, line_color='firebrick', fill_color='firebrick', line_alpha=0.8, fill_alpha=0.3, source=source)
fig.line(x='x', y='y', line_width=2, line_color='gray', line_alpha=0.8, source=source)

#add widgets
add_button = Button(
        label='Add row')
add_button.on_click(add_row)

del_button = Button(
        label='Delete row')
del_button.on_click(del_row)

fit_button = Button(label='Fit data', button_type='danger')
fit_button.on_click(fit_data)

#create text input to enter fit model
model_text_input=TextInput(title='Fit model: ', value='a*exp(-((x-b)/(2*c))**2)')

#create text input to display parameter
parameter_text_input=TextInput(title='Parameter: ', value='')

#create the layout with figure and widgets
layout=layout([[[fig,[model_text_input, parameter_text_input], fit_button],[table,del_button,add_button]]])

#stream data
curdoc().add_root(layout)
curdoc().add_periodic_callback(update,500) #updates each 500ms