# ----------------------------------------------------------------------
# |
# |  Utilities.py
# |
# |  David Brownell <db@DavidBrownell.com>
# |      2018-02-11 13:16:51
# |
# ----------------------------------------------------------------------
# |
# |  Copyright David Brownell 2018.
# |  Distributed under the Boost Software License, Version 1.0.
# |  (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
# |
# ----------------------------------------------------------------------
import hashlib
import os
import re
import sys
import textwrap

from collections import OrderedDict

import six
import six.moves.cPickle as pickle

import SourceRepositoryTools
from SourceRepositoryTools.Impl import CommonEnvironmentImports
from SourceRepositoryTools.Impl import Constants

# ----------------------------------------------------------------------
_script_fullpath = os.path.abspath(__file__) if "python" in sys.executable.lower() else sys.executable
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------

# ----------------------------------------------------------------------
def GenerateCommands( functor,              # def Func() -> []
                      environment,
                      is_debug,
                    ):
    """Returns (result, generated_commands)"""

    assert functor
    assert environment

    commands = []

    try:
        result = functor()

        if isinstance(result, tuple):
            result, commands = result
        else:
            commands = result
            result = 0

    except Exception as ex:
        if is_debug:
            import traceback

            error = traceback.format_exc()
        else:
            error = str(ex)

            if not error.endswith('.'):
                error += '.'

        commands = [ CommonEnvironmentImports.Shell.Message("ERROR: {}".format(CommonEnvironmentImports.StreamDecorator.LeftJustify(error, len("ERROR: ")))),
                     CommonEnvironmentImports.Shell.Exit(pause_on_error=True, return_code=-1),
                   ]

        result = -1

    if is_debug:
        sink = six.moves.StringIO()
        CommonEnvironmentImports.StreamDecorator(sink, line_prefix="Debug: ").write(environment.GenerateCommands(commands))
        sink = sink.getvalue()

        commands = [ CommonEnvironmentImports.Shell.Message("{}\n".format(sink)), ] + commands

    return result, commands

# ----------------------------------------------------------------------
def DelayExecute(method, *args, **kwargs):
    """\
    The vast majority of activation activities can be created at script generation time.
    But, there are some actions that must be run at execution time, as they rely on information
    or state changes output by the statements that run immediately before them.

    For example, an action may rely on an environment variable being created; this environment
    variable will not exist at generation time (as the code to create the variable will
    have not been executed yet).

    In situations like this, this method will create a series of Shell commands that
    run the provided method at execution time rather than code generation time.
    """

    assert method

    environment = CommonEnvironmentImports.Shell.GetEnvironment()

    script_tempfile = environment.CreateTempFilename(environment.ScriptExtension)
    python_tempfile = environment.CreateTempFilename(".py")
    pickle_tempfile = environment.CreateTempFilename(".pickle")

    # Write the arguments
    with open(pickle_tempfile, 'wb') as f:
        pickle.dump((args, kwargs), f)

    # Write the python code
    with open(python_tempfile, 'w') as f:
        file_path, file_name = os.path.split(six.get_function_code(method).co_filename)
        file_name = [ os.path.splitext(file_name)[0], ]

        # Import by the fully qualified name
        while os.path.isfile(os.path.join(file_path, "__init__.py")):
            potential_file_path, name_part = os.path.split(file_path)

            assert potential_file_path != file_path, file_path
            assert name_part

            file_path = potential_file_path
            file_name.insert(0, name_part)

        file_name = '.'.join(file_name)

        f.write(textwrap.dedent(
            """\
            import os
            import sys

            import six

            import six.moves.cPickle as pickle

            sys.path.insert(0, r"{common_environment_path}")
            from CommonEnvironment import Shell
            from CommonEnvironment.StreamDecorator import StreamDecorator
            del sys.path[0]

            sys.path.insert(0, r"{file_path}")
            from {file_name} import {method_name}
            del sys.path[0]

            # Read the arguments
            with open(r"{pickle_tempfile}", 'rb') as f:
                args, kwargs = pickle.load(f)

            try:
                result = {method_name}(*args, **kwargs)

                if isinstance(result, tuple):
                    result, commands = result
                elif result is None:
                    result = 0
                    commands = []
                else:
                    commands = result
                    result = 0

            except:
                result = -1
                commands = []

                import traceback
                sys.stderr.write("ERROR: {{}}".format(StreamDecorator.LeftJustify(traceback.format_exc(), len("ERROR: "))))

            if result != 0:
                messages = [ command for command in commands if isinstance(command, Shell.Message) ]
                for mesage in messages:
                    sys.stderr.write(message.value)

                sys.stderr.write("\\n")
                sys.exit(result)

            # Prep for execution
            environment = Shell.GetEnvironment()

            with open(r"{script_tempfile}", 'w') as f:
                f.write(environment.GenerateCommands(commands))

            environment.MakeFileExecutable(r"{script_tempfile}")

            """).format( common_environment_path=os.path.join(SourceRepositoryTools.GetFundamentalRepository(), Constants.LIBRARIES_SUBDIR, "Python", "CommonEnvironment", "v1.0"),
                         file_path=file_path,
                         file_name=file_name,
                         method_name=method.__name__,
                         script_tempfile=script_tempfile,
                         pickle_tempfile=pickle_tempfile,
                       ))

    return [ environment.Comment("-- Delay executing '{}' in '{}'".format(method.__name__, six.get_function_code(method).co_filename)),

             environment.Execute('{} "{}"'.format(os.getenv("PYTHON_BINARY"), python_tempfile)),
             environment.ExitOnError(),
             environment.Call(script_tempfile),
             environment.ExitOnError(),

             environment.RemoveFile(python_tempfile),
             environment.RemoveFile(pickle_tempfile),
             environment.RemoveFile(script_tempfile),

             environment.Comment("-- End delay execution"),
           ]

# ----------------------------------------------------------------------
_GetRepositoryUniqueId_regex                = None

def GetRepositoryUniqueId( repo_root,
                           scm=None,
                           throw_on_error=True,
                           find_by_scm=True,
                         ):
    global _GetRepositoryUniqueId_regex

    if _GetRepositoryUniqueId_regex is None:
        _GetRepositoryUniqueId_regex = CommonEnvironmentImports.RegularExpression.TemplateStringToRegex(Constants.REPOSITORY_ID_CONTENT_TEMPLATE)

    filename = os.path.join(repo_root, Constants.REPOSITORY_ID_FILENAME)
    if os.path.isfile(filename):
        match = _GetRepositoryUniqueId_regex.match(open(filename).read())
        if not match:
            raise Exception("The content in '{}' appears to be corrupt".format(filename))

        name = match.group("name")
        unique_id = match.group("guid").upper()

    elif find_by_scm:
        if scm is None:
            scm = CommonEnvironmentImports.SourceControlManagement.GetSCM(repo_root, throw_on_error=False)
            if scm is None:
                if throw_on_error:
                    raise Exception("Id information was not found nor could SCM information be extracted from '{}'".format(repo_root))

                return None

        unique_id = scm.GetUniqueName(repo_root)
        name = unique_id

    else:
        if throw_on_error:
            raise Exception("Unable to find repository information for '{}'".format(repo_root))

        return None

    return name, unique_id

# ----------------------------------------------------------------------
def CalculateFingerprint(repo_dirs, relative_root=None):
    """\
    Returns a value that can be used to determine if any configuration info has
    changed for a repo and its dependencies.
    """

    results = {}

    for repo_dir in repo_dirs:
        md5 = hashlib.md5()

        filename = os.path.join(repo_dir, Constants.SETUP_ENVIRONMENT_CUSTOMIZATION_FILENAME)
        if os.path.isfile(filename):
            with open(filename, 'rb') as f:
                # Skip the file header, as it has no impact on the file's actual contents
                in_file_header = True

                for line in f:
                    if in_file_header and line.lstrip().startswith(b'#'):
                        continue

                    in_file_header = False
                    md5.update(line)

            if relative_root:
                repo_dir = CommonEnvironmentImports.FileSystem.GetRelativePath(relative_root, repo_dir)

            results[repo_dir] = md5.hexdigest()

    return results

# ----------------------------------------------------------------------
def GetLatestVersion(version_strings):
    if not version_strings:
        return

    # Convert the version strings into semantic version values
    import semantic_version

    lookup = {}
    versions = []

    for version_string in version_strings:
        original_version_string = version_string

        for potential_prefix in [ "v", "r", ]:
            if version_string.startswith(potential_prefix):
                version_string = version_string[len(potential_prefix):]
                break

        try:
            # Remove leading zeros from version string values
            parts = []

            for part in version_string.split('.'):
                index = 0

                if len(part) > 1:
                    while index < len(part) and part[index] == '0':
                        index += 1

                parts.append(part[index:])

            version = semantic_version.Version.coerce('.'.join(parts))

            lookup[version] = original_version_string
            versions.append(version)

        except ValueError:
            continue

    if not versions:
        return None

    versions.sort(reverse=True)
    return lookup[versions[0]]

# ----------------------------------------------------------------------
def GetVersionedDirectory(version_info, *path_components_or_fullpath):
    """\
    Returns the name of the latest versioned directory.
    """
    return GetVersionedDirectoryEx(version_info, *path_components_or_fullpath)[0]

# ----------------------------------------------------------------------
def GetVersionedDirectoryEx(version_info, *path_components_or_fullpath):
    """\
    Returns the name of the latest versioned directory and its version.
    """

    # This method can be called in these ways:
    #   1) With a fullpath
    #   2) With path components

    if len(path_components_or_fullpath) == 1:
        fullpath = path_components_or_fullpath[0]
        path_components = fullpath.split(os.path.sep)
    else:
        fullpath = os.path.join(*path_components_or_fullpath)
        path_components = path_components_or_fullpath

    # The last part of the path is the name of the library/tool.
    # The second to last part is the type (e.g. Tools, Library, etc.)
    assert len(path_components) >= 1, fullpath

    name = path_components[-1]

    explict_version = CommonEnvironmentImports.CommonEnvironment.Get( version_info,
                                                                      lambda vi: vi.Name == name,
                                                                      lambda vi: vi.Version,
                                                                    )
    if explict_version is None:
        explict_version = GetLatestVersion([ item for item in os.listdir(fullpath) if os.path.isdir(os.path.join(fullpath, item)) ])

    if explict_version:
        fullpath = os.path.join(fullpath, explict_version)

    assert os.path.isdir(fullpath), fullpath

    fullpath = GetCustomizedPath(fullpath)
    assert os.path.isdir(fullpath), fullpath

    return fullpath, explict_version

# ----------------------------------------------------------------------
def GetCustomizedPath(path, environment=None):
    """\
    Drill into environment-specific folders if they exist.
    """

    assert os.path.isdir(path), path
    environment = environment or CommonEnvironmentImports.Shell.GetEnvironment()

    # ----------------------------------------------------------------------
    def OnCustomizedPathError(path, error):
        raise Exception(error)

    # ----------------------------------------------------------------------

    return GetCustomizedPathEx(path, OnCustomizedPathError, environment=environment)

# ----------------------------------------------------------------------
def GetCustomizedPathEx( path,
                         onError,           # def Func(path, error_string) -> GetCustomizedPathEx result value
                         environment=None,
                       ):
    assert os.path.isdir(path), path
    assert onError
    environment = environment or CommonEnvironmentImports.Shell.GetEnvironment()

    while True:
        subdirs = os.listdir(path)

        if IsOSNameDirectory(path):
            if environment.Name in subdirs:
                path = os.path.join(path, environment.Name)
            elif environment.CategoryName in subdirs:
                path = os.path.join(path, environment.CategoryName)
            elif Constants.AGNOSTIC_OS_NAME in subdirs:
                path = os.path.join(path, Constants.AGNOSTIC_OS_NAME)
            else:
                return onError(path, "OS names were found in '{}', but no customization was found for '{}' (is one of {} missing?).".format( path,
                                                                                environment.Name,
                                                                                ', '.join([ "'{}'".format(name) for name in [ environment.Name, environment.CategoryName, Constants.AGNOSTIC_OS_NAME, ] ]),
                                                                                ))

        elif IsOSVersionDirectory(path, environment):
            if environment.OSVersion in subdirs:
                path = os.path.join(path, environment.OSVersion)
            else:
                return onError(path, "OS versions were found in '{}', but no customization was found for '{}'.".format(path, environment.OSVersion))

        elif IsOSArchitectureDirectory(path, environment):
            if environment.OSArchitecture in subdirs:
                path = os.path.join(path, environment.OSArchitecture)
            else:
                return onError(path, "OS architectures were found in '{}', but no customizations were found for '{}'.".format(path, environment.OSArchitecture))

        else:
            break

    return path

# ----------------------------------------------------------------------
def IsOSNameDirectory(path, subdirs=None):
    potential_os_names = set([ Constants.AGNOSTIC_OS_NAME, "src", ])

    for env in CommonEnvironmentImports.Shell.GetPotentialEnvironments():
        potential_os_names.add(env.Name)
        potential_os_names.add(env.CategoryName)

    return _IsImpl(path, potential_os_names, subdirs=subdirs)

# ----------------------------------------------------------------------
def IsOSVersionDirectory(path, environment, subdirs=None):
    return _IsImpl(path, environment.PotentialOSVersionDirectoryNames, subdirs=subdirs)

# ----------------------------------------------------------------------
def IsOSArchitectureDirectory(path, environment, subdirs=None):
    return _IsImpl(path, environment.PotentialOSArchitectureDirectoryNames, subdirs=subdirs)

# ----------------------------------------------------------------------
def GetActivationDir(environment, repo_root, configuration):
    return os.path.join( repo_root,
                         Constants.GENERATED_DIRECTORY_NAME,
                         environment.CategoryName,
                         configuration or "Default",
                       )

# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
def _IsImpl(path, valid_items, subdirs=None):
    subdirs = subdirs or [ item for item in os.listdir(path) if os.path.isdir(os.path.join(path, item)) ]

    for subdir in subdirs:
        if subdir not in valid_items:
            return False

    return bool(subdirs)


from __future__ import absolute_import, division, print_function

from setuptools import find_packages, setup

requirements = ['numpy>=1.16']

setup(packages=find_packages(exclude=['docs']),
    name="ConvCNP-Climate", # Replace with your own username
    version="0.0.1",
    author="Anna Vaughan",
    author_email="av555@cam.ac.uk",
    python_requires='>=3.6',
    install_requires=requirements,
    include_package_data=True)


import cv2 as cv
import numpy as np

img = cv.imread("Photos/piter.jpg")
img = cv.resize(img, (int(img.shape[0]*.3), int(img.shape[1]*.2)), interpolation=cv.INTER_LINEAR)

gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)

# Laplatian
lap = cv.Laplacian(gray, cv.CV_64F)
lap = np.uint8(np.absolute(lap))
cv.imshow("Laplacian", lap)

# Sobel
sobelx = cv.Sobel(gray, cv.CV_64F, 1, 0)
sobely = cv.Sobel(gray, cv.CV_64F, 0, 1)

cv.imshow("Sobel X", sobelx)
cv.imshow("Sobel Y", sobely)

combined = cv.bitwise_or(sobelx, sobely)

cv.imshow("Combined", combined)

canny = cv.Canny(gray, 150, 175)
cv.imshow("Canny", canny)

cv.imshow("Original", img)
cv.waitKey(0)


import  sys
import _pickle as pickle
import json

import websocket

class Network:
    """
    class to connect, send and recieve information from the server

    need to hardcode the host attirbute to be the server's ip
    """

    def __init__(self):
        # self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.client = websocket.WebSocket()
        # self.client.settimeout(10.0)
        # self.host = "127.0.0.1"
        self.host = "13.71.64.234"
        self.port = 5555
        # self.addr = (self.host, self.port)

        self.addr = f"ws://{self.host}:{self.port}"

    def connect(self, name):
        """
        connects to server and returns the id of the client that connected
        :param name: str
        :return: int reprsenting id
        """

        self.client.connect(self.addr)
        self.client.send(name)
        c_id = self.client.recv()

        if type(c_id) == bytes:
            c_id = c_id.decode("utf-8")

        print(f"Val recieved connecting: {c_id}")
        return c_id

        # self.client.connect(self.addr)
        # self.client.send(str.encode(name))
        # val = self.client.recv(128).decode("utf-8")
        # print(f"Val recieved connecting: {val}")
        # return val

    def disconnect(self):
        """
        disconnects from the server
        :return: None
        """
        self.client.close()

    def send(self, data, j=True):
        """
        sends information to the server

        :param data: str
        :param pick: boolean if should pickle or not
        :return: str
        """
        try:
            if j:
                self.client.send(json.dumps(data))
            else:
                self.client.send(data)

            # reply = self.client.recv(int(size.decode()))
            reply = self.client.recv()

            try:
                reply = json.loads(reply)
            except Exception as e:
                print("Error loading json", "-"*20)
                print(f"reply: {reply}")
                print("Error loading json", "-"*20)
                print(e)

            return reply
        except Exception as e:
            print(e)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Validate and correct gradients from eddy outputs
With full AP-PA eddy outputs a full bvec bval (2x nb of dirs and bval)
that doesnt fit with the output dwi (1x nb of dir)
"""

import argparse

import numpy as np

from scilpy.io.utils import (add_overwrite_arg, assert_inputs_exist,
                             assert_outputs_exist)

def _build_arg_parser():
    p = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawTextHelpFormatter)
    p.add_argument('in_bvec',
                   help='In bvec file.')
    p.add_argument('in_bval',
                   help='In bval file.')
    p.add_argument('nb_dirs', type=int,
                   help='Number of directions per DWI.')
    p.add_argument('out_bvec',
                   help='Out bvec file.')
    p.add_argument('out_bval',
                   help='Out bval file.')
    add_overwrite_arg(p)
    return p

def main():
    parser = _build_arg_parser()
    args = parser.parse_args()

    assert_inputs_exist(parser, [args.in_bvec, args.in_bval])
    assert_outputs_exist(parser, args, [args.out_bval, args.out_bvec])

    """
    IN BVEC
    """
    in_bvec = np.genfromtxt(args.in_bvec)
    split_dirs = in_bvec.shape[1] / args.nb_dirs
    if int(split_dirs) != split_dirs:
        parser.error('Number of directions in bvec ({}) can\'t be splited in '
                     'even parts using nb_dirs ({}).'.format(in_bvec.shape[1],
                                                             args.nb_dirs))
    in_bvec_split = np.hsplit(in_bvec, int(split_dirs))
    if len(in_bvec_split) == 2:
        out_bvec = np.mean(np.array([in_bvec_split[0],
                                     in_bvec_split[1]]), axis=0)
    else:
        out_bvec = in_bvec_split[0]
    np.savetxt(args.out_bvec, out_bvec, '%.8f')

    """
    IN BVAL
    """
    in_bval = np.genfromtxt(args.in_bval)
    np.savetxt(args.out_bval, in_bval[0:out_bvec.shape[1]], '%.0f')

if __name__ == '__main__':
    main()


"""
This module implements the Response class which is used to represent HTTP
responses in Scrapy.

See documentation in docs/topics/request-response.rst
"""
from __future__ import annotations

from ipaddress import IPv4Address, IPv6Address
from typing import (
    TYPE_CHECKING,
    Any,
    AnyStr,
    Callable,
    Dict,
    Generator,
    Iterable,
    List,
    Mapping,
    Optional,
    Tuple,
    Union,
    cast,
)
from urllib.parse import urljoin

from twisted.internet.ssl import Certificate

from scrapy.exceptions import NotSupported
from scrapy.http.headers import Headers
from scrapy.http.request import Request
from scrapy.link import Link
from scrapy.utils.trackref import object_ref

if TYPE_CHECKING:
    from scrapy.selector import SelectorList

class Response(object_ref):
    """An object that represents an HTTP response, which is usually
    downloaded (by the Downloader) and fed to the Spiders for processing.
    """

    attributes: Tuple[str, ...] = (
        "url",
        "status",
        "headers",
        "body",
        "flags",
        "request",
        "certificate",
        "ip_address",
        "protocol",
    )
    """A tuple of :class:`str` objects containing the name of all public
    attributes of the class that are also keyword parameters of the
    ``__init__`` method.

    Currently used by :meth:`Response.replace`.
    """

    def __init__(
        self,
        url: str,
        status: int = 200,
        headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
        body: bytes = b"",
        flags: Optional[List[str]] = None,
        request: Optional[Request] = None,
        certificate: Optional[Certificate] = None,
        ip_address: Union[IPv4Address, IPv6Address, None] = None,
        protocol: Optional[str] = None,
    ):
        self.headers: Headers = Headers(headers or {})
        self.status: int = int(status)
        self._set_body(body)
        self._set_url(url)
        self.request: Optional[Request] = request
        self.flags: List[str] = [] if flags is None else list(flags)
        self.certificate: Optional[Certificate] = certificate
        self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address
        self.protocol: Optional[str] = protocol

    @property
    def cb_kwargs(self) -> Dict[str, Any]:
        try:
            return self.request.cb_kwargs  # type: ignore[union-attr]
        except AttributeError:
            raise AttributeError(
                "Response.cb_kwargs not available, this response "
                "is not tied to any request"
            )

    @property
    def meta(self) -> Dict[str, Any]:
        try:
            return self.request.meta  # type: ignore[union-attr]
        except AttributeError:
            raise AttributeError(
                "Response.meta not available, this response "
                "is not tied to any request"
            )

    @property
    def url(self) -> str:
        return self._url

    def _set_url(self, url: str) -> None:
        if isinstance(url, str):
            self._url: str = url
        else:
            raise TypeError(
                f"{type(self).__name__} url must be str, " f"got {type(url).__name__}"
            )

    @property
    def body(self) -> bytes:
        return self._body

    def _set_body(self, body: Optional[bytes]) -> None:
        if body is None:
            self._body = b""
        elif not isinstance(body, bytes):
            raise TypeError(
                "Response body must be bytes. "
                "If you want to pass unicode body use TextResponse "
                "or HtmlResponse."
            )
        else:
            self._body = body

    def __repr__(self) -> str:
        return f"<{self.status} {self.url}>"

    def copy(self) -> Response:
        """Return a copy of this Response"""
        return self.replace()

    def replace(self, *args: Any, **kwargs: Any) -> Response:
        """Create a new Response with the same attributes except for those given new values"""
        for x in self.attributes:
            kwargs.setdefault(x, getattr(self, x))
        cls = kwargs.pop("cls", self.__class__)
        return cast(Response, cls(*args, **kwargs))

    def urljoin(self, url: str) -> str:
        """Join this Response's url with a possible relative url to form an
        absolute interpretation of the latter."""
        return urljoin(self.url, url)

    @property
    def text(self) -> str:
        """For subclasses of TextResponse, this will return the body
        as str
        """
        raise AttributeError("Response content isn't text")

    def css(self, *a: Any, **kw: Any) -> SelectorList:
        """Shortcut method implemented only by responses whose content
        is text (subclasses of TextResponse).
        """
        raise NotSupported("Response content isn't text")

    def jmespath(self, *a: Any, **kw: Any) -> SelectorList:
        """Shortcut method implemented only by responses whose content
        is text (subclasses of TextResponse).
        """
        raise NotSupported("Response content isn't text")

    def xpath(self, *a: Any, **kw: Any) -> SelectorList:
        """Shortcut method implemented only by responses whose content
        is text (subclasses of TextResponse).
        """
        raise NotSupported("Response content isn't text")

    def follow(
        self,
        url: Union[str, Link],
        callback: Optional[Callable] = None,
        method: str = "GET",
        headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
        body: Optional[Union[bytes, str]] = None,
        cookies: Optional[Union[dict, List[dict]]] = None,
        meta: Optional[Dict[str, Any]] = None,
        encoding: Optional[str] = "utf-8",
        priority: int = 0,
        dont_filter: bool = False,
        errback: Optional[Callable] = None,
        cb_kwargs: Optional[Dict[str, Any]] = None,
        flags: Optional[List[str]] = None,
    ) -> Request:
        """
        Return a :class:`~.Request` instance to follow a link ``url``.
        It accepts the same arguments as ``Request.__init__`` method,
        but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
        not only an absolute URL.

        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
        method which supports selectors in addition to absolute/relative URLs
        and Link objects.

        .. versionadded:: 2.0
           The *flags* parameter.
        """
        if encoding is None:
            raise ValueError("encoding can't be None")
        if isinstance(url, Link):
            url = url.url
        elif url is None:
            raise ValueError("url can't be None")
        url = self.urljoin(url)

        return Request(
            url=url,
            callback=callback,
            method=method,
            headers=headers,
            body=body,
            cookies=cookies,
            meta=meta,
            encoding=encoding,
            priority=priority,
            dont_filter=dont_filter,
            errback=errback,
            cb_kwargs=cb_kwargs,
            flags=flags,
        )

    def follow_all(
        self,
        urls: Iterable[Union[str, Link]],
        callback: Optional[Callable] = None,
        method: str = "GET",
        headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
        body: Optional[Union[bytes, str]] = None,
        cookies: Optional[Union[dict, List[dict]]] = None,
        meta: Optional[Dict[str, Any]] = None,
        encoding: Optional[str] = "utf-8",
        priority: int = 0,
        dont_filter: bool = False,
        errback: Optional[Callable] = None,
        cb_kwargs: Optional[Dict[str, Any]] = None,
        flags: Optional[List[str]] = None,
    ) -> Generator[Request, None, None]:
        """
        .. versionadded:: 2.0

        Return an iterable of :class:`~.Request` instances to follow all links
        in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
        but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects,
        not only absolute URLs.

        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`
        method which supports selectors in addition to absolute/relative URLs
        and Link objects.
        """
        if not hasattr(urls, "__iter__"):
            raise TypeError("'urls' argument must be an iterable")
        return (
            self.follow(
                url=url,
                callback=callback,
                method=method,
                headers=headers,
                body=body,
                cookies=cookies,
                meta=meta,
                encoding=encoding,
                priority=priority,
                dont_filter=dont_filter,
                errback=errback,
                cb_kwargs=cb_kwargs,
                flags=flags,
            )
            for url in urls
        )


from rq import Queue
from rq.job import Job
from rq import Retry
import yfinance
import json
# from worker import conn
from db_helpers import *
from scraper import get_sp_companies, get_high_short_interest, _parse_tickers, get_small_cap_tickers, get_dow_companies, get_nasdaq, read_russell_csv

q = get_redis_queue()

def queue_collection_tickers(tickers, collection):

    for tick in tickers:
        q.enqueue_call(
            func=write_ticker_to_collections,
            args=(tick, collection),
            retry=Retry(3),
            result_ttl=5000
        )

def large_cap():

    collection = "S&P 500"
    url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
    tickers = _parse_tickers(*get_sp_companies(url))

    jobs = queue_collection_tickers(
        tickers,
        collection
    )

def medium_cap():

    collection = "S&P 400"
    url = "https://en.wikipedia.org/wiki/List_of_S%26P_400_companies"
    tickers = _parse_tickers(*get_sp_companies(url))

    jobs = queue_collection_tickers(
        tickers,
        collection
    )

def small_cap():
    collection = "S&P 600"
    tickers = get_small_cap_tickers()

    jobs = queue_collection_tickers(
        tickers,
        collection
    )

def dow_jones():
    collection = "Dow Jones"
    tickers = get_dow_companies()

    jobs = queue_collection_tickers(
        tickers,
        collection
    )

def nasdaq():
    collection = "Nasdaq 100"
    tickers = get_nasdaq()

    jobs = queue_collection_tickers(
        tickers,
        collection
    )

def high_short():

    collection = "High Short Interest"
    tickers = get_high_short_interest()

    jobs = queue_collection_tickers(
        tickers,
        collection
    )

def russell_3000():

    url = "https://www.ishares.com/us/products/239714/ishares-russell-3000-etf/1467271812596.ajax?fileType=csv&fileName=IWV_holdings&dataType=fund"

    collection = "Russell 3000"
    headers, data = read_russell_csv(url)

    tickers = [row[0] for row in data]

    jobs = queue_collection_tickers(
        tickers,
        collection
    )

def russell_microcap():

    url = "https://www.ishares.com/us/products/239716/ishares-microcap-etf/1467271812596.ajax?fileType=csv&fileName=IWC_holdings&dataType=fund"

    collection = "Russell Microcap"
    headers, data = read_russell_csv(url)

    tickers = [row[0] for row in data]

    jobs = queue_collection_tickers(
        tickers,
        collection
    )

if __name__ == "__main__":

    print("Large Cap")
    large_cap()
    print("Medium Cap")
    medium_cap()
    print("Small Cap")
    small_cap()
    print("Dow Jones")
    dow_jones()
    print("Nasdaq")
    nasdaq()
    print("High Shorts")
    high_short()
    print("Russell 3000")
    russell_3000()
    print("Russell Microcap")
    russell_microcap()
    print("Finding Removed")
    find_removed()


# -*- coding: utf-8 -*-
"""
Created on Fri Oct 22 13:02:18 2021

@author: leand
"""

import LimitCase as lC
import LimitCaseFileIO as lCFIO

import time

# EXECUTES ONE RUN
def do_run(file_controller, size, N_steps, pop_coeffs, spread_coeffs, sim, n):
    output = sim.simulate()
    file_controller.start_section(output[0].shape, [('run_number', n), ('size', size), ('N_steps', N_steps)])
    for o in output:
        file_controller.append(o)
    file_controller.end_section()

#%% I DO NOT RECOMMEND RUNNING THIS PROGRAMME. VERY RESOURCE INTENSIVE

def main():

    # Set custom filename. WARNING: FILE CAN BE VERY LARGE
    filename = 'run.txt'

    #%% LOADOUT 0: SERIOUS RUNS
    # # List of coefficients to simulate
    # coefficients_c = [1, 0.1, 0.01, 0.001]

    # # Parameters for each coefficient
    # SIZE = [50, 50, 50, 50]
    # N_STEPS = [1000, 1000, 1000, 1000]

    # # Number of runs per coefficient
    # N_runs_per_coefficient = 50

    #%% LOADOUT 1: TESTING PURPOSES
    coefficients_c = [1, 0.1, 0.01, 0.001]
    SIZE = [20, 20, 20, 20]
    N_STEPS = [100, 100, 100, 100]
    N_runs_per_coefficient = 2

    print("REMEMBER TO ADJUST THE DOMAINS FOR EACH C IN THE ANALYSIS SCRIPTS")

    #%% Execution

    file_controller = lCFIO.FileIO()
    file_controller.create_fresh_file(filename)

    for i, c in enumerate(coefficients_c):

        size = SIZE[i]
        N_steps = N_STEPS[i]

        pop_coeffs = [[[0.5, 0.5] for j in range(size)] for i in range(size)]
        spread_coeffs = [[[c, c] for j in range(size)] for i in range(size)]

        sim = lC.Simulator(size, N_steps, pop_coeffs, spread_coeffs)

        for j, n in enumerate(range(N_runs_per_coefficient)):
            t_init = time.time()
            print('c:', c, 'n:', n)

            do_run(file_controller, size, N_steps, pop_coeffs, spread_coeffs, sim, n)

            t_final = time.time()
            print('duration:', t_final - t_init, 'seconds')
            if i == 0 and j == 0:
                print("ESTIMATED DURATION:", round(((t_final - t_init) / 60 / 60) * len(coefficients_c) * N_runs_per_coefficient, 3), 'hours')

if __name__ == '__main__':
    main()

#%% JUST COPY BLOCKS OF CODE IF YOU WANT TO START DIFFERENT RUNS

# coefficients_c = [0.1, 0.01]

# N_runs_per_coefficient = 100

# file_controller = lCFIO.FileIO()
# file_controller.create_fresh_file('run4_long_time.txt')

# for i, c in enumerate(coefficients_c):

#     size = 20
#     N_steps = 5000

#     pop_coeffs = [[[0.5, 0.5] for j in range(size)] for i in range(size)]
#     spread_coeffs = [[[c, c] for j in range(size)] for i in range(size)]

#     sim = lC.Simulator(size, N_steps, pop_coeffs, spread_coeffs)

#     for j, n in enumerate(range(N_runs_per_coefficient)):

#         t_init = time.time()

#         print('c:', c, 'n:', n)

#         output = sim.simulate()

#         file_controller.start_section(output[0].shape, [('run_number', n), ('size', size), ('N_steps', N_steps)])

#         for o in output:
#             file_controller.append(o)

#         file_controller.end_section()

#         t_final = time.time()
#         print('duration:', t_final - t_init, 'seconds')

#         if i == 0 and j == 0:
#             print("ESTIMATED DURATION:", round(((t_final - t_init) / 60 / 60) * len(coefficients_c) * N_runs_per_coefficient, 3), 'hours')


SYMBOLS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"

def get_char_index(char, alphabet):
    char_index = alphabet.find(char)
    return char_index

def is_int(str):
    try:
        int(str)
        return True
    except ValueError:
        return False

def encryption_lineal():
    phrase = str(input("Enter your phrase:")).upper()
    result = []
    A = int(input("Enter A:"))
    B = int(input("Enter B:"))

    p = 0
    for char in phrase:
        x = get_char_index(char, SYMBOLS)
        K = A*p + B
        y = (x + K)%26 if x >= 0 else char
        result.append(SYMBOLS[y]) if is_int(y) else result.append(" ")
        p += 1
    return ''.join(result)

def decryption_lineal():
    e_phrase = str(input("Enter your encryption phrase:")).upper()
    d_phrase = str(input("Enter your decryption phrase:")).upper()
    B = get_char_index(d_phrase[0], SYMBOLS) - get_char_index(e_phrase[0],SYMBOLS)
    A = get_char_index(d_phrase[1], SYMBOLS) - get_char_index(e_phrase[1],
SYMBOLS) - B
    print("A - ", A)
    print("B - ", B)

def encryption_sq():
    phrase = str(input("Enter your phrase:")).upper()
    result = []
    A = int(input("Enter A:"))
    B = int(input("Enter B:"))
    C = int(input("Enter C:"))

    p = 0
    for char in phrase:
        x = get_char_index(char, SYMBOLS)
        K = A * p * p + B * p + C
        y = (x + K) % 26 if x >= 0 else char
        if is_int(y):
            result.append(SYMBOLS[y])
        else:
            result.append(char)
            p = p - 1
        p += 1
    return ''.join(result)

def decryption_sq():
    phrase = str(input("Enter your phrase:")).upper()
    result = []

    p = 0
    for A in range(1, 6):
        for B in range(1, 6):
            for C in range(1, 6):
                for char in phrase:
                    y = get_char_index(char, SYMBOLS)
                    K = (A * p * p + B * p + C) % 26
                    x = (y - K)%26 if y >= 0 else char

                    if is_int(x):
                        result.append(SYMBOLS[x])
                    else:
                        result.append(char)
                        p = p - 1
                    p += 1

                print(A, B, C, ''.join(result))
                result = []
                p = 0


# 611<BFS>[S2]4963_py
from collections import deque
n=1
m=1
while n>0 or m>0 :
    n,m =map(int, input().split())
    board = [ list(map(int, input().split())) for _ in range(m)]
    vis = [[0]*n for _ in range(m)]
    dX = [0, 0,-1, 1, 1, 1,-1,-1]
    dY = [1,-1, 0, 0, 1,-1, 1,-1]
    cnt = 0
    for i in range(m):
        for j in range(n):
            if vis[i][j]==1 or board[i][j]==0 : continue
            q = deque()
            q.append([i,j])
            vis[i][j]=1
            cnt+=1
            while q :
                now = q.popleft()
                for d in range(len(dX)):
                    nX = now[0]+dX[d]
                    nY = now[1]+dY[d]
                    if 0<=nX<m and 0<=nY<n and vis[nX][nY]==0 and board[nX][nY]==1:
                        q.append([nX,nY])
                        vis[nX][nY]=1
    if n!=0 and m!=0 :print(cnt)
    # 0 0 입력시 출력되게 했더니 틀렸다고 나옴..;;;


from flask import request, jsonify
from .auth import isAuth
from models import Kategori
from . import db

def get_kategori():
    a = isAuth()
    if a == None:
        return {
            "message": "Unauthorized"
        }, 400
    return jsonify([
        {
            'id_kategori': i.id_kategori,
            'nama': i.nama,
            'deskripsi' : i.deskripsi,
        } for i in Kategori.query.all()
    ])

def get_kategori_id(id):
    a = isAuth()
    if a == None:
        return {
            "message": "Unauthorized"
        }, 400
    kategori = Kategori.query.filter_by(id_kategori=id).first_or_404()
    return {
        'id': kategori.id_kategori,
        'nama': kategori.nama,
        'deskripsi': kategori.deskripsi
    }

def create_kategori():
    # from db import db
    a = isAuth()
    if a == None:
        return {
            "message": "Unauthorized"
        }, 400
    if a.is_admin == False:
        return {
            "YOU ARE NOT ADMIN"
        }
    data = request.get_json()
    if not 'nama' in data or not 'deskripsi' in data :
        return jsonify({
            'error': 'Bad Request',
            'message': 'Nama ata Deskripisi Tidak ada'
        }), 400
    k = Kategori (
                    nama = data['nama'],
                    deskripsi = data['deskripsi']
                )
    db.session.add(k)
    db.session.commit()
    return {
        'nama': k.nama,
        'deskripsi': k.deskripsi
    }, 201

def update_kategori(id):
    # from db import db
    a = isAuth()
    if a == None:
        return {
            "message": "Unauthorized"
        }, 400
    if a.is_admin == False:
        return {
            "YOU ARE NOT ADMIN"
        }
    data=request.get_json()
    if 'nama' not in data:
        return {
            'error': 'Bad Request',
            'message': 'Name field needs to be present'
        }, 400
    k = Kategori.query.filter_by(id_kategori=id).first_or_404()
    k.nama=data['nama']
    if 'deskripsi' in data:
        k.deskripsi= data['deskripsi']
    db.session.commit()
    return jsonify({
        'nama': k.nama,
        'deskripsi': k.deskripsi
    })

def delete_kategori(id):
    # from db import db
    a = isAuth()
    if a == None:
        return {
            "message": "Unauthorized"
        }, 400
    if a.is_admin == False:
        return {
            "YOU ARE NOT ADMIN"
        }
    k = Kategori.query.filter_by(id_kategori=id).first_or_404()
    db.session.delete(k)
    db.session.commit()
    return {
        'success': 'Data Berhasil Dihapus'
    }


r"""
Basic training script for PyTorch
"""

# Set up custom environment before nearly anything else is imported
# NOTE: this should be the first import (no not reorder)
from maskrcnn_benchmark.utils.env import setup_environment  # noqa F401 isort:skip

import argparse
import os
import json
import torch
from collections import defaultdict
from maskrcnn_benchmark.config import cfg
from maskrcnn_benchmark.utils.collect_env import collect_env_info
from maskrcnn_benchmark.utils.comm import synchronize, get_rank
from maskrcnn_benchmark.utils.imports import import_file
from maskrcnn_benchmark.utils.logger import setup_logger
from maskrcnn_benchmark.utils.miscellaneous import mkdir
from maskrcnn_benchmark.data.transforms import build_transforms
from maskrcnn_benchmark.data.build import build_dataset
from maskrcnn_benchmark.data.collate_batch import BatchCollator
from maskrcnn_benchmark.structures.segmentation_mask import SegmentationMask
from pycocotools import mask as maskUtils
import numpy as np
import pdb
import cv2
import math

def get_median(data):
    data = sorted(data)
    size = len(data)
    if size % 2 == 0:
        median = (data[size // 2] + data[size // 2 - 1]) / 2
        data[0] = median
    if size % 2 == 1:
        median = data[(size - 1) // 2]
        data[0] = median
    return data[0]
def chunks(arr, m):
    n = int(math.ceil(len(arr) / float(m)))
    return [arr[i:i + n] for i in range(0, len(arr), n)]
def annToMask(boxlist):
    """
    Convert annotation which can be polygons, uncompressed RLE to RLE.
    :return: binary mask (numpy 2D array)
    """
    h,w = boxlist.size
    masks = boxlist.get_field('masks')
    labels = boxlist.get_field('labels')
    # pdb.set_trace()
    index = list(np.nonzero(labels==1)[:,0])
    polygons = []
    for i, seg in enumerate(masks.polygons):
        if i in index:
            polygons.append(seg)
    RLES = []
    for segm in polygons:
        rles =  maskUtils.frPyObjects([p.numpy() for p in segm.polygons], h, w )
        rle = maskUtils.merge(rles)
        RLES.append(rle)

    return RLES

def split(cfg, num_fold):
    # when calculate the overlap ratio, set the sliding window overlap = 0
    paths_catalog = import_file(
        "maskrcnn_benchmark.config.paths_catalog", cfg.PATHS_CATALOG, True
    )
    DatasetCatalog = paths_catalog.DatasetCatalog
    dataset_list = cfg.DATASETS.TEST

    transforms = build_transforms(cfg,False)
    datasets = build_dataset(dataset_list, transforms, DatasetCatalog, False)
    collator = BatchCollator(cfg.DATALOADER.SIZE_DIVISIBILITY)
    data_loader = torch.utils.data.DataLoader(
        datasets[0],
        num_workers=1,
        collate_fn=collator,

    )

    name_dic = defaultdict(list)
    for i,  data in enumerate(data_loader,0):
        boxlists = data[1][0]

        masks = annToMask(boxlists)
        isCrowed = len(masks) * [0]
        iou = maskUtils.iou(masks, masks, isCrowed) - np.eye(len(masks))
        maxiou = np.max(iou,0)
        avgiou = np.mean(maxiou)
        name = datasets[0].get_img_info(i)['file_name']
        name_dic[name].append(avgiou)
    names = []
    overlapping = []
    for k, v in name_dic.items():
        names.append(k)
        overlapping.append(sum(v)/len(v))
    median = get_median(overlapping)
    # pdb.set_trace()
    hard = []
    easy = []
    for name, miou in zip(names, overlapping):
        if miou>median:
            hard.append(name)
        else:
            easy.append(name)

    # pdb.set_trace()
    easy_n  = chunks(easy, num_fold)
    hard_n = chunks(hard, num_fold)

    i = 1
    splitfile = {}
    for e,h in zip(easy_n,hard_n):
        splitfile[i] = {'easy': e, 'hard':h}
        i+=1

    with open('split.json', 'w') as f:
        json.dump(splitfile, f)

def main():

    parser = argparse.ArgumentParser(description="PyTorch Object Detection Training")
    parser.add_argument(
        "--config-file",
        default="",
        metavar="FILE",
        help="path to config file",
        type=str,
    )
    parser.add_argument("--local_rank", type=int, default=0)
    parser.add_argument(
        "--skip-test",
        dest="skip_test",
        help="Do not test the final model",
        action="store_true",
    )
    parser.add_argument(
        "opts",
        help="Modify config options using the command-line",
        default=None,
        nargs=argparse.REMAINDER,
    )
    cv2.setNumThreads(0)
    args = parser.parse_args()

    num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
    args.distributed = num_gpus > 1

    if args.distributed:
        torch.cuda.set_device(args.local_rank)
        torch.distributed.init_process_group(
            backend="nccl", init_method="env://"
        )

    cfg.merge_from_file(args.config_file)
    cfg.merge_from_list(args.opts)
    cfg.freeze()

    output_dir = cfg.OUTPUT_DIR
    if output_dir:
        mkdir(output_dir)

    logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
    logger.info("Using {} GPUs".format(num_gpus))
    logger.info(args)

    logger.info("Collecting env info (might take some time)")
    logger.info("\n" + collect_env_info())

    logger.info("Loaded configuration file {}".format(args.config_file))
    with open(args.config_file, "r") as cf:
        config_str = "\n" + cf.read()
        logger.info(config_str)
    logger.info("Running with config:\n{}".format(cfg))

    split(cfg, 3)

if __name__ == "__main__":
    main()


import multiprocessing
from time import sleep
import time

#work function
def calculate(process_name, tasks, results):
    print("{} evaluation routine starts".format(process_name))
    while True:
        new_value = tasks.get()
        if new_value < 0:
            print("{} evaluation routine quits".format(process_name))
            results.put(-1)
            break
        else:
            compute = new_value * new_value
            sleep(0.02 * new_value)

            print("{} received value: {}".format(process_name, new_value))
            print("{} calculated value: {}".format(process_name, compute))
            results.put(compute)
    return

if __name__ == "__main__":
    manager = multiprocessing.Manager()

    tasks = manager.Queue()
    results = manager.Queue()

    num_processes = 1
    pool = multiprocessing.Pool(processes=num_processes)
    processes = []
    start = time.clock()
    for i in range(num_processes):
        process_name = "P{}".format(i)
        new_process = multiprocessing.Process(target=calculate, args=(process_name, tasks, results))
        processes.append(new_process)
        new_process.start()
    task_list = [43, 1, 780, 256, 142, 68, 183, 334, 325, 3]
    for single_task in task_list:
        tasks.put(single_task)
    sleep(5)
    for i in range(num_processes):
        tasks.put(-1)

    num_finished_processes = 0
    while True:
        new_result = results.get()
        if new_result == -1:
            num_finished_processes += 1
            if num_finished_processes == num_processes:
                print("ALL DONE")
                break
        else:
            print("Result:" + str(new_result))

    print(time.clock() - start)

###############################################################
# "C:\Users\jsun\My Documents\venv\Scripts\python.exe" C:/Users/jsun/Documents/Py_projects/mul_process/mul_process_queue_log16.py
# P0 evaluation routine starts
# P0 received value: 43
# P0 calculated value: 1849
# P0 received value: 1
# P0 calculated value: 1
# Result:1849
# Result:1
# P0 received value: 780
# P0 calculated value: 608400
# Result:608400
# P0 received value: 256
# P0 calculated value: 65536
# Result:65536
# P0 received value: 142
# P0 calculated value: 20164
# Result:20164
# P0 received value: 68
# P0 calculated value: 4624
# Result:4624
# P0 received value: 183
# P0 calculated value: 33489
# Result:33489
# P0 received value: 334
# P0 calculated value: 111556
# Result:111556
# P0 received value: 325
# P0 calculated value: 105625
# Result:105625
# P0 received value: 3
# P0 calculated value: 9
# Result:9
# P0 evaluation routine quits
# ALL DONE
# 42.790012


# ! /usr/bin/env python
# - * - coding:utf-8 - * -
# __author__ : KingWolf
# createtime : 2019/8/11 4:41

"""

    一个大于1的自然数，除了1和它本身外，不能被其他自然数（质数）整除（2, 3, 5, 7等），换句话说就是该数除了1和它本身以外不再有其他的因数。
    (else的另外知识点：我们常见的是if…else…或者if…elif…else诸如此类，但其实for也可以和else搭配出现)

"""
from math import sqrt

#第一种传统方法
def prime_tradition():
    num = int(input('请输入正整数：'))
    if num >1:
        for i in range(2,int(sqrt(num) + 1)):
            if num % i == 0:
                print('输入的数字: %d 不是素数！' %num)
                break
        else:
            print('输入的数字: %d 是素数！' %num)
    else:
        print('输入有误，请重新输入！')
if __name__ == '__main__':
    prime_tradition()


startted = False
call = input('> ')
if call.upper() == 'HELP' or call.lower() == 'help' :
    print('start -> for start car')
    print('stop -> for stop car')
    print('quit -> for exit')
    while True:
        choice = input('> ')
        if choice.lower() == 'start' or choice.upper() == 'START':
            if startted :
                print('car already started')
            else:
                print('car started')
                startted = True
        elif choice.lower() == 'stop' or choice.upper() == 'STOP':
            if not startted:
                print('car already stopped')
            else:
                print('car stopped')
                startted = False
        elif choice.upper() == 'QUIT' or choice.lower() == 'quit':
            break
        else:
            print("I don't understand")
else:
    print("I don't understand")


import torch
from torch import nn
from .gaussian_smoothing import GaussianSmoothing

class SoftFrangiFilter2D(nn.Module):

    def __init__(self, channels, kernel_size, sigmas, beta, c, device):
        """
        Apply Soft Frangi filter on a 3d tensor.
        Arguments:
            channels (int, sequence): Number of channels of the input tensors. Output will
                have this number of channels as well.
            kernel_size (int, sequence): Size of the gaussian kernel.
            sigmas (list, sequence): List of standard deviations of the gaussian kernels.
            beta (float, sequence): Beta parameter of Frangi filter.
            beta (с, sequence): С parameter of Frangi filter.
                Default value is 2 (spatial).
        """
        super(SoftFrangiFilter2D, self).__init__()
        self.sigmas = sigmas
        self.gaus_filters = []
        self.beta = beta
        self.c = c
        for sigma in sigmas:
            self.gaus_filters.append([])
            for order in ['xx', 'yy', 'xy']:
                self.gaus_filters[-1].append(GaussianSmoothing(channels, kernel_size, sigma, 2, order, device))

    def _calc_frangi_response(self, xx, yy, xy):
        """
        Calculate Frangi filter response give the second order derivatives.
        Arguments:
            xx (torch.Tensor, sequence): (bs, channels, h, w), second order derivative on x-axis
            yy (torch.Tensor, sequence): (bs, channels, h, w), second order derivative on y-axis
            xy (torch.Tensor, sequence): (bs, channels, h, w), second order derivative on xy-axes
        """
        lambda_t1 = ((xx + yy) + torch.sqrt((xx - yy) ** 2 + 4 * xy ** 2 + 1e-6)) / 2
        lambda_t2 = ((xx + yy) - torch.sqrt((xx - yy) ** 2 + 4 * xy ** 2 + 1e-6)) / 2
        lambdas = torch.stack((lambda_t1, lambda_t2), dim=0)
        lambdas_abs, sorted_ind = torch.sort(torch.abs(lambdas), dim=0)
        lambda2_sign = (lambdas * sorted_ind).sum(dim=0)
        lambda1 = lambdas_abs[0]
        lambda2 = lambdas_abs[1]
        blobness = torch.zeros(lambda1.shape, device=lambda1.device)
        blobness[lambda2 != 0] = blobness[lambda2 != 0] + torch.exp(
            -(lambda1[lambda2 != 0] / lambda2[lambda2 != 0]) ** 2 / (2 * self.beta ** 2))
        hess_struc = (1 - torch.exp(-torch.sqrt(lambda1 ** 2 + lambda2 ** 2) ** 2 / (2 * self.c ** 2)))
        vness = blobness * hess_struc
        vness[lambda2_sign > 0] = vness[lambda2_sign > 0] * 0
        return vness, blobness, hess_struc, lambda1, lambda2

    def forward(self, img):
        """
        Apply Soft Frangi filter on a batch of images.
        Arguments:
            img (torch.Tensor, sequence): Tensor of shape (bs, channels, h, w)
        """
        frangi_resp = torch.zeros((len(self.sigmas),) + img.shape, dtype=torch.float32, device=img.device)
        for i in range(len(self.sigmas)):
            xx = self.gaus_filters[i][0](img)
            yy = self.gaus_filters[i][1](img)
            xy = self.gaus_filters[i][2](img)
            vness, _, _, _, _ = self._calc_frangi_response(xx, yy, xy)
            frangi_resp[i] = vness
        max_frangi_resp = (torch.softmax(frangi_resp, dim=0) * frangi_resp).sum(dim=0)
        return max_frangi_resp


import sys
import os
sys.path.append(os.getcwd()+'/Python/')
from APIRequests import *
from logger import logger

patchdate = datetime.date.today() - datetime.timedelta(days=7)

def getmatchIds(amount = 1000):
    matchIds = []
    s = summoners()
    while len(matchIds)<amount:
        mh = matchhistory(s.getid())
        try:
            mh.sendrequest()
        except IOError as e:
            logger.warning('%s, skipping to next', str(e))
            continue
        try:
            for record in mh.getmatch():
                if datetime.date.fromtimestamp(record['matchCreation']/1000) > patchdate and record['matchId'] not in matchIds and record['queueType']=='RANKED_SOLO_5x5':
                    matchIds.append(record['matchId'])
                    if len(matchIds) % 50 == 0:
                        logger.info('Currently have %d matchIds', len(matchIds))
        except IndexError as e:
            logger.warning('%s, skipping to next', str(e))
            continue ##If there are no matches available for this summoner, skip to next
    logger.info('Ready to get match data, have %d matchIds', len(matchIds))
    return matchIds

def getgamesmongo(matchIds):
    """Pass a list of matchIds to add each match to mongodb"""
    fullcounter = 0
    parsedcounter = 0
    for matchId in matchIds:
        m = match(matchId)
        try:
            m.fetchdata()
        except IOError as e:
            logger.warning('%s. Cannot get match data, skipping to next', str(e))
            continue
        try:
            m.fetchparsed()
        except IOError as e:
            logger.warning('%s. Cannot get parsed data, will add full data if possible', str(e))
        except AssertionError as e: ##If there is no timeline data, skip this
            logger.warning('%s. Skipping to next', str(e))
            continue

        ##Try to add to mongodb, will output error if it already is there
        try:
            m.addtomongo('full')
            fullcounter += 1
        except IOError as e:
            logger.warning('%s. Cannot add full data, will add parsed data if possible', str(e))
        try:
            m.addtomongo('parsed')
            parsedcounter += 1
        except IOError as e:
            logger.warning('%s, Cannot add parsed data, moving on to next match', str(e))
        if fullcounter % 50 == 0:
            logger.info('Added %d games to MongoDB full collection and %d games to MongoDB parsed collection so far this session', fullcounter, parsedcounter)

    logger.info('Operation completed successfully, added %d games to MongoDB full collection and %d games to MongoDB parsed collection', fullcounter, parsedcounter)

##Also consider making this function and the one above part of the above classes. or maybe subclasses

script, amount = sys.argv

logger.info('Getting %s games after date of %s', amount, patchdate)
matchIds = getmatchIds(int(amount))
getgamesmongo(matchIds)


from pymongo.mongo_client import MongoClient
import json
import os

with open(str(os.path.dirname(__file__)) + '/details.json') as file:
    details = json.load(file)

uri = details['mongodb_uri']
client = MongoClient(uri)

def mongo():
    try:
        db = client["tixelscrapy"]
        collection = db["tickets"]
        print("Pinged your deployment. You successfully connected to MongoDB!")
    except Exception as e:
        print(e)
    return db


from abc import ABC, abstractmethod
import argparse
import datetime
import json
import logging
import itertools
import os
import os.path
import time
import traceback
import typing
import uuid

import psutil

from constants import (
    REPORTS_DIR,
    CONFIG_FILE_PATH,
    ALERT_SNOOZE_TIME_SECONDS,
)
from utils.data_cache import (
    read_saved_value,
    write_saved_value,
    DataCacheMissError,
    DataCacheKeyExpired,
)
from utils.discord import create_discord_message
from utils.logger import spawn_logger
from utils.pid import (
    create_pid_file,
    remove_pid_file,
    PIDFileExistsException,
)
from utils.helpers import md5sum

# Config validation
class InvalidConfigError(Exception):
    pass

def validate_config(config: typing.Dict):
    if 'max_memory_percent' not in config:
        raise InvalidConfigError("expected config value 'max_memory_percent'")
    if 'max_cpu_percent' not in config:
        raise InvalidConfigError("expected config value 'max_cpu_percent'")
    if 'max_disk_percent' not in config:
        raise InvalidConfigError("expected config value 'max_disk_percent'")

    def _validate_percent_value(v: float, name: str):
        if not isinstance(v, float) or v < 0 or v > 1:
            raise InvalidConfigError(f'field {v} has invalid value')

    _validate_percent_value(config['max_memory_percent'], 'max_memory_percent')
    _validate_percent_value(config['max_cpu_percent'], 'max_cpu_percent')
    _validate_percent_value(config['max_disk_percent'], 'max_disk_percent')

    if "user_names_to_ignore" not in config or not isinstance(config['user_names_to_ignore'], list):
        raise InvalidConfigError(f'expected field user_names_to_ignore to be a list')

    if "process_names_to_ignore" not in config or not isinstance(config['process_names_to_ignore'], list):
        raise InvalidConfigError(f'expected field process_names_to_ignore to be a list')

    if "discord_bot" in config:
        if "channel_id" not in config["discord_bot"]:
            raise InvalidConfigError(
                "discord_bot object expected key channel_id"
            )
        if not config["discord_bot"]['channel_id'] or len(config["discord_bot"]['channel_id']) < 5:
            raise InvalidConfigError(
                "discord_bot object contains invalid key channel_id"
            )
        if "bot_token" not in config["discord_bot"]:
            raise InvalidConfigError(
                "discord_bot object expected key bot_token"
            )
        if not config["discord_bot"]['bot_token'] or len(config["discord_bot"]['bot_token']) < 5:
            raise InvalidConfigError(
                "discord_bot object contains invalid key bot_token"
            )

# Alert type definitions # # # # # #

class AlertType:
    PROCESSES = "PROCESSES"
    RESOURCES = "RESOURCES"

class ResourceNames:
    MEMORY = "MEMORY"
    CPU = "CPU"
    DISK = "DISK"

class BaseAlert(ABC):

    alert_type = NotImplemented
    notice_fields = NotImplemented

    @abstractmethod
    def to_key_name(self) -> str:
        pass

    def to_email_string(self) -> str:
        parts = []
        for field in self.notice_fields:
            parts.append(f"{field}: {getattr(self, field)}")
        return "\n".join(parts)

class SystemResourceUsageAlert(BaseAlert):

    alert_type = AlertType.RESOURCES
    notice_fields = [
        'resource_name',
        'resource_usage_percent',
    ]

    def __init__(
        self,
        resource_name: str,
        resource_usage_percent: float,
    ):
        self.resource_name = resource_name
        self.resource_usage_percent = resource_usage_percent

    def to_key_name(self):
        return self.resource_name

    def __str__(self):
        return f"<ResourceAlert {self.resource_name} @ {round(self.resource_usage_percent, 4)}>"

class ProcessAlert(BaseAlert):

    alert_type = AlertType.PROCESSES
    notice_fields = [
        'process_id',
        'user_uid',
        'user_name',
        'command',
        'priority',
        'created_at',
    ]

    def __init__(
        self,
        process_id: str,
        user_uid: int,
        user_name: str,
        command: str,
        priority: str,
        created_at: str,
    ):
        self.process_id = process_id
        self.user_uid = user_uid
        self.user_name = user_name
        self.command = command
        self.priority = priority
        self.created_at = created_at

    def to_key_name(self):
        return md5sum(f"{self.process_id}-{self.user_uid}-{self.command}")

    def __str__(self):
        return f"<ProcessAlert pid{self.process_id} uid{self.user_uid}>"

# # Check for resource alerts # # # # # #

def get_system_resource_usage_alerts(logger, config) -> typing.List[SystemResourceUsageAlert]:
    # Memory
    memory_used_percent = psutil.virtual_memory().percent / 100
    logger.info(f"memory_used_percent {memory_used_percent}")
    # CPU
    cpu_useage_percent = psutil.cpu_percent(interval=0.5) / 100
    logger.info(f"cpu_useage_percent {cpu_useage_percent}")
    # Disk
    disk_useage_percent = psutil.disk_usage("/").percent / 100
    logger.info(f"disk_useage_percent {disk_useage_percent}")

    alerts = []
    if memory_used_percent > config['max_memory_percent']:
        alerts.append(SystemResourceUsageAlert(
            ResourceNames.MEMORY,
            memory_used_percent
        ))
        logger.warning(f"creating alert for MEMORY usage")
    if cpu_useage_percent > config['max_cpu_percent']:
        alerts.append(SystemResourceUsageAlert(
            ResourceNames.CPU,
            cpu_useage_percent
        ))
        logger.warning(f"creating alert for CPU usage")
    if disk_useage_percent > config['max_disk_percent']:
        alerts.append(SystemResourceUsageAlert(
            ResourceNames.DISK,
            disk_useage_percent
        ))
        logger.warning(f"creating alert for DISK usage")

    return alerts

# # Check for processes alerts # # # # # #

def get_processes_alerts(logger, config) -> typing.List[ProcessAlert]:
    alerts = []
    for proc in psutil.process_iter():
        with proc.oneshot():
            pid = proc.pid
            user_uid = proc.uids().real
            user_name = proc.username()
            command = f"{proc.name()} ({proc.exe()})"
            created_at =  datetime.datetime.fromtimestamp(proc.create_time()).strftime("%Y-%m-%d %H:%M:%S")
            priority = proc.nice()

        if user_name in config['user_names_to_ignore']:
            continue
        if proc.name() in config['process_names_to_ignore']:
            if priority >= 0:
                # We expect this to be 0 or higher.
                # if < 0 then process has elevated priority
                continue

        logger.warning(f"creating alert for process")
        alerts.append(ProcessAlert(
            pid, user_uid, user_name, command, priority, created_at
        ))

    return alerts

# # # Script Entry Point # # # # # # # # #

def main(logger: logging.Logger, config: typing.Dict) -> None:
    logger.debug("main method running")

    logger.debug("checking for alerts")
    resource_alerts = get_system_resource_usage_alerts(logger, config)
    process_alerts = get_processes_alerts(logger, config)
    if len(resource_alerts) == 0 and len(process_alerts) == 0:
        logger.debug("no alerts found")
        return

    if len(resource_alerts):
        logger.warning(f"found {len(resource_alerts)} resource alert(s)")
    if len(process_alerts):
        logger.warning(f"found {len(process_alerts)} process alert(s)")

    message_text = []
    for alert in itertools.chain(resource_alerts, process_alerts):

        alert_cache_key = alert.to_key_name()
        try:
            # Check cache to see if we've sent an alert for this event recently.
            # We don't want to spam alerts.
            read_saved_value(alert_cache_key)
        except (DataCacheMissError, DataCacheKeyExpired):
            send_alert = True
            logger.debug(f"cache miss, writing key to cache: {alert_cache_key}")
            write_saved_value(alert_cache_key, None, ALERT_SNOOZE_TIME_SECONDS)
        else:
            logger.debug("cache hit")
            logger.warning(f"not sending alert {alert}, cache key found")
            send_alert = False

        if send_alert:
            logger.warning(f" **** SENDING ALERT **** {alert}")
            message_text.append(alert.to_email_string())

    if len(message_text):
        file_report_path = os.path.join(REPORTS_DIR, str(uuid.uuid4()) + ".report")
        # Send messages
        if "discord_bot" in config and not config['discord_bot'].get("skip"):
            logger.debug("sending discord message")
            max_discord_messages = 5
            chunk_size = 12
            for ix, i in enumerate(range(0, len(message_text), chunk_size)):
                if i > 0:
                    time.sleep(1.5)
                create_discord_message(
                    "\n\n".join(message_text[i: i+chunk_size]),
                    config['discord_bot']['bot_token'],
                    config['discord_bot']['channel_id'],
                    logger,
                )
                if ix >= max_discord_messages:
                    # write the rest to disk
                    logger.debug("creating incident report file")
                    with open(file_report_path, "w") as f:
                        f.write("\n\n".join(message_text[i+chunk_size:]))
                    create_discord_message(
                        "Additional indicent details written to " + file_report_path,
                        config['discord_bot']['bot_token'],
                        config['discord_bot']['channel_id'],
                        logger,
                    )
                    break

        else:
            logger.debug("creating incident report file")
            with open(file_report_path, "w") as f:
                f.write("\n\n".join(message_text))

    else:
        logger.warning("no messages to send")

if __name__ == "__main__":

    # check if production. If prod = True, no console logging and send alerts
    #                      If prod = False, log to console, show alerts in console.
    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--production', action='store_true')
    args = parser.parse_args()

    logger = spawn_logger(args.production)

    # Load config
    try:
        with open(CONFIG_FILE_PATH) as f:
            config = json.load(f)
        validate_config(config)
    except IOError:
        logger.error("could not find config file")
        logger.error(traceback.format_exc())
        raise
    except InvalidConfigError:
        logger.error("invalid config file")
        logger.error(traceback.format_exc())
        raise

    # Create pid file right before calling main
    try:
        create_pid_file()
    except PIDFileExistsException:
        logger.error("found PID file, exiting")
        raise

    # Call main method and log any errors.
    try:
        main(logger, config)
    except Exception as e:
        logger.error(f"An Error Occured :(: {e}")
        logger.error(traceback.format_exc())
        raise
    finally:
        remove_pid_file()
        logger.debug("goodbye")


import json
import os
import random
import sys
from contextlib import closing

from boto3 import Session
from botocore.exceptions import BotoCoreError, ClientError

session = Session()
polly = session.client('polly')
outputFormat = 'mp3'
book = 'caterpillar'#'peter'
#book = 'peter'
voices = ('Amy', 'Emma', 'Joanna', 'Ivy', 'Joey', 'Justin', 'Kendra', 'Kimberly', 'Salli', 'Brian')
voices = ('Emma',)
base_path = os.path.join('catalyst', 'static', 'catalyst', book)
colors = [
    '#4A90E2', # Blue
    '#D0021B', # Red
    '#F5A623', # Orange
    '#F8E71C', # Yellow
    '#7ED321', # Green
    '#BD10E0', # Pink
    '#9013FE', # Purple
    '#50E3C2', # Teal
]
with open(os.path.join(base_path, 'english.json')) as f:
    english_texts = json.load(f)
with open(os.path.join(base_path, 'images.json')) as f:
    images = json.load(f)
for voiceId in voices:
    voice_path = os.path.join(base_path, voiceId)
    audio_path = os.path.join(voice_path, 'audio')
    for path in (voice_path, audio_path):
        if not os.path.exists(path):
            os.makedirs(path)
    pages = []
    audio_count = 1
    for page_count, text in enumerate(english_texts):
        image = images[page_count]
        page = {
            'text': text,
            'audio': '',
            'audio_info': [],
            'image': image,
            'color': '' if (text or image) else random.choice(colors),
        }
        pages.append(page)
        if text:
            response = polly.synthesize_speech(Text=text,
                                               VoiceId=voiceId,
                                               OutputFormat=outputFormat)
            with closing(response['AudioStream']) as stream:
                body = stream.read()
                with open(os.path.join(audio_path, '{}.mp3'.format(audio_count)), 'wb') as audio_file:
                    audio_file.write(body)
                page['audio'] = '{}.mp3'.format(audio_count)
                audio_count += 1
            response = polly.synthesize_speech(Text=page['text'],
                                               VoiceId=voiceId,
                                               OutputFormat='json',
                                               SpeechMarkTypes=['word'])
            with closing(response['AudioStream']) as stream:
                body = stream.read()
                # audio_info_dict = {}
                audio_info_list = []
                for audio_info_str in body.decode().split():
                    audio_info = json.loads(audio_info_str)
                    audio_info_list.append(audio_info)
                    # audio_info_dict[audio_info['time']] = audio_info
                page['audio_info'] = audio_info_list
                print(audio_info_list)

    with open(os.path.join(voice_path, 'meta.json'), 'w') as f:
        json.dump(pages, f)

    # with open(os.path.join(path, 'meta_audio.json'), 'w') as f:
    #     json.dump(audio_meta, f)
    # with open(os.path.join(path, 'meta.json'), 'w') as f:
    #     json.dump(meta, f)


import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from datetime import datetime

# Import classes
import sys
sys.path.insert(0,"..")
from linear_regression import Regression
from stochastic_gradient_descent import StochasticDescent
from logistic_regression import MultinomialRegression
from load_mnist import LoadMNIST

# Load MNIST dataset
dat = LoadMNIST()

# Logistic regression using sklearn  -----------------------
logreg = LogisticRegression()
logreg.fit(dat.x_train, dat.y_train)
pred = logreg.predict(dat.x_test)

# Checking accuracy
score = logreg.score(dat.x_test, dat.y_test)
print(score)

# Using our own logistic regression ------------------------

# Define parameters for SGD
batch_size = 2**6
epochs = 100
eta = 0.01
lam = 0.1

# Use our Logistic method to make prediction
mulreg = MultinomialRegression(dat.x_train, dat.y_train, batch_size=batch_size, epochs=epochs, t0=eta, lam=lam)
mulreg.fit()
pred2 = mulreg.predict(dat.x_test)

# Checking accuracy
score2 = mulreg.accuracy_score(pred2, dat.y_test)

print(dat.y_test[:30])
print(pred2[:30])
print(score2)

# Random sampling
def random_search(iterations, print_vals=False):
    """
    Performs random search for a given amount of iterations
    and writes out the best result to logistic_mark.txt.
    """
    # Set up parameter lists
    epochs = 500
    batch_sizes = [4, 16, 32, 64, 128]
    learning_rates = np.logspace(-2, 1, 10)
    gamma_array = np.linspace(0, 0.99, 10)
    penalties = np.logspace(-5, -1, 10)

    # Storage of parameters used and accuracy score
    eta_arr         = np.zeros(iterations)
    batch_size_arr  = np.zeros(iterations)
    gamma_arr       = np.zeros(iterations)
    lam_arr         = np.zeros(iterations)
    scores          = np.zeros(iterations)

    current_iteration = 1
    for i in range(iterations):
        # Fetch random values
        eta_arr[i]         = np.random.choice(learning_rates)
        batch_size_arr[i]  = np.random.choice(batch_sizes)
        gamma_arr[i]       = np.random.choice(gamma_array)
        lam_arr[i]         = np.random.choice(penalties)

        # Print for debugging
        if print_vals:
            print("eta: ", eta_arr[i])
            print("B: ", batch_size_arr[i])
            print("gamma: ", gamma_arr[i])
            print("lam: ", lam_arr[i])

        # Do logistic regression
        mulreg = MultinomialRegression(dat.x_train, dat.y_train,
        batch_size=batch_size_arr[i], epochs=epochs, t0=eta_arr[i], lam=lam_arr[i], gamma=gamma_arr[i])
        mulreg.fit()
        pred = mulreg.predict(dat.x_test)

        # Calculate accuracy
        scores[i] =  mulreg.accuracy_score(pred, dat.y_test)

        # Print progress
        print("Iteration: %i/%i"%(current_iteration, iterations))
        current_iteration += 1

    # Set nan values to 0
    scores = np.nan_to_num(scores)
    idx = np.argmax(scores)

    # Store best values in benchmark file
    filename = "../../benchmarks/logistic_mark.txt"
    outfile = open(filename, 'a')
    outfile.write("---- Random search: %i iterations ----" %iterations)
    outfile.write("\nAccuracy score: %f"    %scores[idx])
    outfile.write("\nLearning rate: %.3e"   %eta_arr[idx])
    outfile.write("\nBatch size: %i"        %batch_size_arr[idx])
    outfile.write("\nGamma: %.3f"           %gamma_arr[idx])
    outfile.write("\nPenalty: %.3e"         %lam_arr[idx])
    outfile.write("\n\n")

    print("Accuracy score: ", scores[idx])
    print("Learning rate: ", eta_arr[idx])
    print("Batch size: ", batch_size_arr[idx])
    print("Gamma: ", gamma_arr[idx])
    print("Penalty: ", lam_arr[idx])

random_search(100)


import datetime

from django.utils import timezone
from fora.tests.base import BaseTestCase
from forums.models import Forum
from mock import patch
from posts.models import Post
from threads.models import Thread

class PostModelTestCase(BaseTestCase):

    def setUp(self):
        self.forum = Forum.objects.create(name='Forum', description='Testing')
        self.thread = Thread.objects.create(subject='Thread', forum=self.forum)
        self.post = Post.objects.create(
            message='Post', thread=self.thread, parent=None)
        super(PostModelTestCase, self).setUp()

    def test_new_post(self):
        self.assertEqual(Post.objects.count(), 1)
        self.assertEqual(Post.objects.get(pk=1), self.post)

    def test_post_to_string(self):
        self.assertEqual(str(self.post), "o4")

    def test_post_to_unicode(self):
        self.assertEqual(unicode(self.post), u'o4')

    @patch.object(timezone, 'now', return_value=datetime.datetime(2015, 1, 1))
    def test_thread_created_date_and_updated_date(self, mock_now):
        p = Post.objects.create(
            message='Post', thread=self.thread, parent=None)
        self.assertEqual(p.created, datetime.datetime(2015, 1, 1))
        self.assertEqual(p.updated, datetime.datetime(2015, 1, 1))

    def test_post_depth(self):
        self.assertEqual(self.post.depth, 0)

        p = Post.objects.create(
            message='A Child', thread=self.thread, parent=self.post)
        self.assertEqual(p.depth, 1)


import pytest
import requests

from pathlib import Path
from pytest_mock import MockerFixture
from typing import Any
from unittest.mock import call as MockCall

from ahriman.core.configuration import Configuration
from ahriman.core.upload.github import GitHub

def test_github_release_tag(configuration: Configuration) -> None:
    """
    must correctly define GitHub release tag
    """
    _, repository_id = configuration.check_loaded()

    instance = GitHub(repository_id, configuration, "github:x86_64")
    assert instance.github_release_tag == instance.github_release_tag_name == repository_id.architecture

    configuration.set_option("github:x86_64", "use_full_release_name", "yes")
    instance = GitHub(repository_id, configuration, "github:x86_64")
    assert instance.github_release_tag == f"{repository_id.name}-{repository_id.architecture}"
    assert instance.github_release_tag_name == f"{repository_id.name} {repository_id.architecture}"

def test_asset_remove(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must remove asset from the release
    """
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    github.asset_remove(github_release, "asset_name")
    request_mock.assert_called_once_with("DELETE", "asset_url")

def test_asset_remove_unknown(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must not fail if no asset found
    """
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    github.asset_remove(github_release, "unknown_asset_name")
    request_mock.assert_not_called()

def test_asset_upload(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must upload asset to the repository
    """
    mocker.patch("pathlib.Path.open")
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    remove_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_remove")

    github.asset_upload(github_release, Path("/root/new.tar.xz"))
    request_mock.assert_called_once_with("POST", "upload_url", params=[("name", "new.tar.xz")],
                                         data=pytest.helpers.anyvar(int),
                                         headers={"Content-Type": "application/x-tar"})
    remove_mock.assert_not_called()

def test_asset_upload_with_removal(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must remove existing file before upload
    """
    mocker.patch("pathlib.Path.open")
    mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    remove_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_remove")

    github.asset_upload(github_release, Path("asset_name"))
    github.asset_upload(github_release, Path("/root/asset_name"))
    remove_mock.assert_has_calls([
        MockCall(github_release, "asset_name"),
        MockCall(github_release, "asset_name"),
    ])

def test_asset_upload_empty_mimetype(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must upload asset to the repository with empty mime type if the library cannot guess it
    """
    mocker.patch("pathlib.Path.open")
    mocker.patch("ahriman.core.upload.github.GitHub.asset_remove")
    mocker.patch("mimetypes.guess_type", return_value=(None, None))
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")

    github.asset_upload(github_release, Path("/root/new.tar.xz"))
    request_mock.assert_called_once_with("POST", "upload_url", params=[("name", "new.tar.xz")],
                                         data=pytest.helpers.anyvar(int),
                                         headers={"Content-Type": "application/octet-stream"})

def test_get_local_files(github: GitHub, resource_path_root: Path, mocker: MockerFixture) -> None:
    """
    must get all local files recursively
    """
    walk_mock = mocker.patch("ahriman.core.util.walk")
    github.get_local_files(resource_path_root)
    walk_mock.assert_called()

def test_files_remove(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must remove files from the remote
    """
    remove_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_remove")
    github.files_remove(github_release, {Path("a"): "a"}, {"a": "a", "b": "b"})
    remove_mock.assert_called_once_with(github_release, "b")

def test_files_remove_empty(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must remove nothing if nothing changed
    """
    remove_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_remove")
    github.files_remove(github_release, {Path("a"): "a"}, {"a": "a"})
    remove_mock.assert_not_called()

def test_files_upload(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must upload files to the remote
    """
    upload_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_upload")
    github.files_upload(github_release, {Path("a"): "a", Path("b"): "c", Path("c"): "c"}, {"a": "a", "b": "b"})
    upload_mock.assert_has_calls([
        MockCall(github_release, Path("b")),
        MockCall(github_release, Path("c")),
    ])

def test_files_upload_empty(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must upload nothing if nothing changed
    """
    upload_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_upload")
    github.files_upload(github_release, {Path("a"): "a"}, {"a": "a"})
    upload_mock.assert_not_called()

def test_release_create(github: GitHub, mocker: MockerFixture) -> None:
    """
    must create release
    """
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    github.release_create()
    request_mock.assert_called_once_with(
        "POST", pytest.helpers.anyvar(str, True),
        json={"tag_name": github.github_release_tag, "name": github.github_release_tag_name})

def test_release_get(github: GitHub, mocker: MockerFixture) -> None:
    """
    must get release
    """
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    github.release_get()
    request_mock.assert_called_once_with("GET", pytest.helpers.anyvar(str, True))

def test_release_get_empty(github: GitHub, mocker: MockerFixture) -> None:
    """
    must return nothing in case of 404 status code
    """
    response = requests.Response()
    response.status_code = 404
    mocker.patch("ahriman.core.upload.github.GitHub.make_request",
                 side_effect=requests.HTTPError(response=response))
    assert github.release_get() is None

def test_release_get_exception(github: GitHub, mocker: MockerFixture) -> None:
    """
    must re-raise non HTTPError exception
    """
    mocker.patch("ahriman.core.upload.github.GitHub.make_request", side_effect=Exception())
    with pytest.raises(Exception):
        github.release_get()

def test_release_get_exception_http_error(github: GitHub, mocker: MockerFixture) -> None:
    """
    must re-raise HTTPError exception with code differs from 404
    """
    exception = requests.HTTPError(response=requests.Response())
    mocker.patch("ahriman.core.upload.github.GitHub.make_request", side_effect=exception)
    with pytest.raises(requests.HTTPError):
        github.release_get()

def test_release_update(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must update release
    """
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    github.release_update(github_release, "body")
    request_mock.assert_called_once_with("POST", "release_url", json={"body": "body"})

def test_release_sync(github: GitHub, mocker: MockerFixture) -> None:
    """
    must run sync command
    """
    release_get_mock = mocker.patch("ahriman.core.upload.github.GitHub.release_get", return_value={})
    get_hashes_mock = mocker.patch("ahriman.core.upload.github.GitHub.get_hashes", return_value={})
    get_local_files_mock = mocker.patch("ahriman.core.upload.github.GitHub.get_local_files", return_value={})
    files_upload_mock = mocker.patch("ahriman.core.upload.github.GitHub.files_upload")
    files_remove_mock = mocker.patch("ahriman.core.upload.github.GitHub.files_remove")
    release_update_mock = mocker.patch("ahriman.core.upload.github.GitHub.release_update")

    github.sync(Path("local"), [])
    release_get_mock.assert_called_once_with()
    get_hashes_mock.assert_called_once_with("")
    get_local_files_mock.assert_called_once_with(Path("local"))
    files_upload_mock.assert_called_once_with({}, {}, {})
    files_remove_mock.assert_called_once_with({}, {}, {})
    release_update_mock.assert_called_once_with({}, pytest.helpers.anyvar(int))

def test_release_sync_create_release(github: GitHub, mocker: MockerFixture) -> None:
    """
    must create release in case if it does not exist
    """
    mocker.patch("ahriman.core.upload.github.GitHub.release_get", return_value=None)
    mocker.patch("ahriman.core.upload.github.GitHub.get_hashes")
    mocker.patch("ahriman.core.upload.github.GitHub.get_local_files")
    mocker.patch("ahriman.core.upload.github.GitHub.files_upload")
    mocker.patch("ahriman.core.upload.github.GitHub.files_remove")
    mocker.patch("ahriman.core.upload.github.GitHub.release_update")
    release_create_mock = mocker.patch("ahriman.core.upload.github.GitHub.release_create")

    github.sync(Path("local"), [])
    release_create_mock.assert_called_once_with()


import numpy as np

class HMC_Forward_Backward_Restorator:

    def __init__(self, Omega_X, Pi, A, B):
        self.Omega_X = Omega_X
        self.N = len(Omega_X)
        self.Pi = Pi
        self.A = A
        self.B = B

    def forward(self, Y):
        T = len(Y)
        alpha = np.zeros((T, len(self.Omega_X)))

        alpha[0] = self.compute_alpha_1(Y[0])
        for t in range(0, T - 1):
            alpha[t + 1] = self.compute_alpha_t_plus_1(Y[t + 1], alpha[t])

        return alpha

    def backward(self, Y, default = 0):
        T = len(Y)
        beta = np.zeros((T, self.N))

        beta[T - 1] = self.compute_beta_T()
        for t in range(T - 2, -1, -1):
            beta[t] = self.compute_beta_t(Y[t + 1], beta[t + 1])

        return beta

    def compute_gamma(self, alpha, beta):
        gamma = alpha * beta
        gamma = gamma/np.sum(gamma, axis = 1, keepdims = True)
        return gamma

    def restore_X(self, Y):
        alpha = self.forward(Y)
        beta = self.backward(Y)
        gamma = self.compute_gamma(alpha, beta)

        indexes = np.argmax(gamma, axis = 1)
        X = [self.Omega_X[index] for index in indexes]
        return X

    ########################
    ### COMPUTING ALPHA ####
    ########################

    def compute_alpha_1(self, y0):
        alpha_1 = np.zeros(self.N)
        #vérification de la nullité des bi
        test = 0
        for idx,i in enumerate(self.Omega_X):
            test += self.B[i].get(y0,0)
        if (test == 0):
            for idx,i in enumerate(self.Omega_X):
                self.B[i][y0] = 1

        for idx,i in enumerate(self.Omega_X):
            alpha_1[idx] = self.Pi[i] * self.B[i].get(y0,0)
        return alpha_1

    def compute_alpha_t_plus_1(self, yt_plus_1, alpha_t):
        alpha_t_plus_1 = np.zeros(self.N)
        #vérification de la nullité des bi
        test = 0
        for idx,i in enumerate(self.Omega_X):
            test += self.B[i].get(yt_plus_1,0)
        if (test == 0):
            for idx,i in enumerate(self.Omega_X):
                self.B[i][yt_plus_1] = 1

        #calcul des alpha
        for idx,i in enumerate(self.Omega_X):
            sum = 0 #Somme des at(j)at(i)
            for jdx,j in enumerate(self.Omega_X):
                sum += alpha_t[jdx] * self.A[j].get(i,0)
            alpha_t_plus_1[idx] = self.B[i].get(yt_plus_1,0) * sum
        alpha_t_plus_1 /= np.sum(alpha_t_plus_1)
        return alpha_t_plus_1

    ######################
    ### COMPUTING BETA ###
    ######################

    def compute_beta_T(self):
        beta_T = np.zeros(self.N)
        # code à compléter
        for idx,i in enumerate(self.Omega_X):
            beta_T[idx] = 1
        beta_T = beta_T/np.sum(beta_T)
        return beta_T

    def compute_beta_t(self, yt_plus_1, beta_t_plus_1):
        beta_t = np.zeros(self.N)

        #vérification de la nullité des bi
        test = 0
        for idx,i in enumerate(self.Omega_X):
            test += self.B[i].get(yt_plus_1,0)
        if (test == 0):
            for idx,i in enumerate(self.Omega_X):
                self.B[i][yt_plus_1] = 1

        for idx,i in enumerate(self.Omega_X):
            sum = 0
            for jdx,j in enumerate(self.Omega_X):
                sum += beta_t_plus_1[jdx] * self.A[i].get(j,0) * self.B[j].get(yt_plus_1,0)
            beta_t[idx] = sum
        beta_t /= np.sum(beta_t)
        return beta_t


import os
import random
import os.path as osp
import tarfile
import zipfile
from collections import defaultdict
import gdown

from dassl.utils import check_isfile

class Datum:
    """Data instance which defines the basic attributes.

    Args:
        impath (str): image path.
        label (int): class label.
        domain (int): domain label.
        classname (str): class name.
    """

    def __init__(self, impath="", label=0, domain=0, classname=""):
        assert isinstance(impath, str)
        assert check_isfile(impath)

        self._impath = impath
        self._label = label
        self._domain = domain
        self._classname = classname

    @property
    def impath(self):
        return self._impath

    @property
    def label(self):
        return self._label

    @property
    def domain(self):
        return self._domain

    @property
    def classname(self):
        return self._classname

class DatasetBase:
    """A unified dataset class for
    1) domain adaptation
    2) domain generalization
    3) semi-supervised learning
    """

    dataset_dir = ""  # the directory where the dataset is stored
    domains = []  # string names of all domains

    def __init__(self, train_x=None, train_u=None, val=None, test=None):
        self._train_x = train_x  # labeled training data
        self._train_u = train_u  # unlabeled training data (optional)
        self._val = val  # validation data (optional)
        self._test = test  # test data
        self._num_classes = self.get_num_classes(train_x)
        self._lab2cname, self._classnames = self.get_lab2cname(train_x)

    @property
    def train_x(self):
        return self._train_x

    @property
    def train_u(self):
        return self._train_u

    @property
    def val(self):
        return self._val

    @property
    def test(self):
        return self._test

    @property
    def lab2cname(self):
        return self._lab2cname

    @property
    def classnames(self):
        return self._classnames

    @property
    def num_classes(self):
        return self._num_classes

    @staticmethod
    def get_num_classes(data_source):
        """Count number of classes.

        Args:
            data_source (list): a list of Datum objects.
        """
        label_set = set()
        for item in data_source:
            label_set.add(item.label)
        return max(label_set) + 1

    @staticmethod
    def get_lab2cname(data_source):
        """Get a label-to-classname mapping (dict).

        Args:
            data_source (list): a list of Datum objects.
        """
        container = set()
        for item in data_source:
            container.add((item.label, item.classname))
        mapping = {label: classname for label, classname in container}
        labels = list(mapping.keys())
        labels.sort()
        classnames = [mapping[label] for label in labels]
        return mapping, classnames

    def check_input_domains(self, source_domains, target_domains):
        assert len(source_domains) > 0, "source_domains (list) is empty"
        assert len(target_domains) > 0, "target_domains (list) is empty"
        self.is_input_domain_valid(source_domains)
        self.is_input_domain_valid(target_domains)

    def is_input_domain_valid(self, input_domains):
        for domain in input_domains:
            if domain not in self.domains:
                raise ValueError(
                    "Input domain must belong to {}, "
                    "but got [{}]".format(self.domains, domain)
                )

    def download_data(self, url, dst, from_gdrive=True):
        if not osp.exists(osp.dirname(dst)):
            os.makedirs(osp.dirname(dst))

        if from_gdrive:
            gdown.download(url, dst, quiet=False)
        else:
            raise NotImplementedError

        print("Extracting file ...")

        if dst.endswith(".zip"):
            zip_ref = zipfile.ZipFile(dst, "r")
            zip_ref.extractall(osp.dirname(dst))
            zip_ref.close()

        elif dst.endswith(".tar"):
            tar = tarfile.open(dst, "r:")
            tar.extractall(osp.dirname(dst))
            tar.close()

        elif dst.endswith(".tar.gz"):
            tar = tarfile.open(dst, "r:gz")
            tar.extractall(osp.dirname(dst))
            tar.close()

        else:
            raise NotImplementedError

        print("File extracted to {}".format(osp.dirname(dst)))

    def generate_fewshot_dataset(
        self, *data_sources, num_shots=-1, repeat=False
    ):
        """Generate a few-shot dataset (typically for the training set).

        This function is useful when one wants to evaluate a model
        in a few-shot learning setting where each class only contains
        a small number of images.

        Args:
            data_sources: each individual is a list containing Datum objects.
            num_shots (int): number of instances per class to sample.
            repeat (bool): repeat images if needed (default: False).
        """
        if num_shots < 1:
            if len(data_sources) == 1:
                return data_sources[0]
            return data_sources

        print(f"Creating a {num_shots}-shot dataset")

        output = []

        for data_source in data_sources:
            tracker = self.split_dataset_by_label(data_source)
            dataset = []

            for label, items in tracker.items():
                if len(items) >= num_shots:
                    sampled_items = random.sample(items, num_shots)
                else:
                    if repeat:
                        sampled_items = random.choices(items, k=num_shots)
                    else:
                        sampled_items = items
                dataset.extend(sampled_items)

            output.append(dataset)

        if len(output) == 1:
            return output[0]

        return output

    def split_dataset_by_label(self, data_source):
        """Split a dataset, i.e. a list of Datum objects,
        into class-specific groups stored in a dictionary.

        Args:
            data_source (list): a list of Datum objects.
        """
        output = defaultdict(list)

        for item in data_source:
            output[item.label].append(item)

        return output

    def split_dataset_by_domain(self, data_source):
        """Split a dataset, i.e. a list of Datum objects,
        into domain-specific groups stored in a dictionary.

        Args:
            data_source (list): a list of Datum objects.
        """
        output = defaultdict(list)

        for item in data_source:
            output[item.domain].append(item)

        return output