import inspect
import io
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Iterable,
    List,
    Optional,
    Sequence,
    Type,
    TypeVar,
    Union,
)

import click
import cloup

if TYPE_CHECKING:  # pragma: no cover
    import click.shell_completion
    from cloup.constraints import BoundConstraintSpec

    from .core import TyperCommand, TyperGroup
    from .main import Typer  # noqa

NoneType = type(None)

AnyType = Type[Any]

Required = ...

class Context(cloup.Context):
    def __init__(
        self,
        # click.Context
        command: "click.Command",
        parent: Optional["click.Context"] = None,
        info_name: Optional[str] = None,
        obj: Optional[Any] = None,
        auto_envvar_prefix: Optional[str] = None,
        default_map: Optional[Dict[str, Any]] = None,
        terminal_width: Optional[int] = None,
        max_content_width: Optional[int] = None,
        resilient_parsing: bool = False,
        allow_extra_args: Optional[bool] = None,
        allow_interspersed_args: Optional[bool] = None,
        ignore_unknown_options: Optional[bool] = None,
        help_option_names: Optional[List[str]] = None,
        token_normalize_func: Optional[Callable[[str], str]] = None,
        color: Optional[bool] = None,
        show_default: Optional[bool] = None,
        # cloup.Context
        align_option_groups: Optional[bool] = None,
        align_sections: Optional[bool] = None,
        show_subcommand_aliases: Optional[bool] = None,
        show_constraints: Optional[bool] = None,
        check_constraints_consistency: Optional[bool] = None,
        formatter_settings: Dict[str, Any] = {},
    ):
        if auto_envvar_prefix is None:
            if (
                parent is not None
                and parent.auto_envvar_prefix is not None
                and info_name is not None
            ):
                auto_envvar_prefix = info_name.upper()
                if parent.auto_envvar_prefix:
                    auto_envvar_prefix = (
                        f"{parent.auto_envvar_prefix}_{auto_envvar_prefix}"
                    )

        super().__init__(
            command=command,
            parent=parent,
            info_name=info_name,
            obj=obj,
            auto_envvar_prefix=auto_envvar_prefix,
            default_map=default_map,
            terminal_width=terminal_width,
            max_content_width=max_content_width,
            resilient_parsing=resilient_parsing,
            allow_extra_args=allow_extra_args,
            allow_interspersed_args=allow_interspersed_args,
            ignore_unknown_options=ignore_unknown_options,
            help_option_names=help_option_names,
            token_normalize_func=token_normalize_func,
            color=color,
            show_default=show_default,
            align_option_groups=align_option_groups,
            align_sections=align_sections,
            show_subcommand_aliases=show_subcommand_aliases,
            show_constraints=show_constraints,
            check_constraints_consistency=check_constraints_consistency,
            formatter_settings=formatter_settings,
        )

class FileText(io.TextIOWrapper):
    pass

class FileTextWrite(FileText):
    pass

class FileBinaryRead(io.BufferedReader):
    pass

class FileBinaryWrite(io.BufferedWriter):
    pass

class CallbackParam(click.Parameter):
    pass

class DefaultPlaceholder:
    """
    You shouldn't use this class directly.

    It's used internally to recognize when a default value has been overwritten, even
    if the new value is `None`.
    """

    def __init__(self, value: Any):
        self.value = value

    def __bool__(self) -> bool:
        return bool(self.value)

DefaultType = TypeVar("DefaultType")

CommandFunctionType = TypeVar("CommandFunctionType", bound=Callable[..., Any])

def Default(value: DefaultType) -> DefaultType:
    """
    You shouldn't use this function directly.

    It's used internally to recognize when a default value has been overwritten, even
    if the new value is `None`.
    """
    return DefaultPlaceholder(value)  # type: ignore

class CommandInfo:
    def __init__(
        self,
        name: Optional[str] = None,
        *,
        cls: Optional[Type["TyperCommand"]] = None,
        context_settings: Optional[Dict[Any, Any]] = None,
        callback: Optional[Callable[..., Any]] = None,
        help: Optional[str] = None,
        epilog: Optional[str] = None,
        short_help: Optional[str] = None,
        options_metavar: str = "[OPTIONS]",
        add_help_option: bool = True,
        no_args_is_help: bool = False,
        hidden: bool = False,
        deprecated: bool = False,
        section: Optional[cloup.Section] = None,
        constraints: Sequence["BoundConstraintSpec"] = (),
        show_constraints: Optional[bool] = None,
        align_option_groups: Optional[bool] = None,
        aliases: Optional[Iterable[str]] = None,
        formatter_settings: Optional[Dict[str, Any]] = None,
    ):
        self.name = name
        self.cls = cls
        self.context_settings = context_settings
        self.callback = callback
        self.help = help
        self.epilog = epilog
        self.short_help = short_help
        self.options_metavar = options_metavar
        self.add_help_option = add_help_option
        self.no_args_is_help = no_args_is_help
        self.hidden = hidden
        self.deprecated = deprecated
        self.section = section
        self.constraints = constraints
        self.show_constraints = show_constraints
        self.align_option_groups = align_option_groups
        self.aliases = aliases
        self.formatter_settings = formatter_settings

class TyperInfo:
    def __init__(
        self,
        typer_instance: Optional["Typer"] = Default(None),
        *,
        name: Optional[str] = Default(None),
        cls: Optional[Type["TyperGroup"]] = Default(None),
        invoke_without_command: bool = Default(False),
        no_args_is_help: bool = Default(False),
        subcommand_metavar: Optional[str] = Default(None),
        chain: bool = Default(False),
        result_callback: Optional[Callable[..., Any]] = Default(None),
        section: Optional[cloup.Section] = Default(None),
        align_sections: Optional[bool] = Default(None),
        show_subcommand_aliases: Optional[bool] = Default(None),
        # Command
        context_settings: Optional[Dict[Any, Any]] = Default(None),
        callback: Optional[Callable[..., Any]] = Default(None),
        help: Optional[str] = Default(None),
        epilog: Optional[str] = Default(None),
        short_help: Optional[str] = Default(None),
        options_metavar: str = Default("[OPTIONS]"),
        add_help_option: bool = Default(True),
        hidden: bool = Default(False),
        deprecated: bool = Default(False),
        constraints: Sequence["BoundConstraintSpec"] = Default(()),
        show_constraints: Optional[bool] = Default(None),
        align_option_groups: Optional[bool] = Default(None),
        aliases: Optional[Iterable[str]] = Default(None),
        formatter_settings: Optional[Dict[str, Any]] = Default(None),
    ):
        self.typer_instance = typer_instance
        self.name = name
        self.cls = cls
        self.invoke_without_command = invoke_without_command
        self.no_args_is_help = no_args_is_help
        self.subcommand_metavar = subcommand_metavar
        self.chain = chain
        self.result_callback = result_callback
        self.section = section
        self.align_sections = align_sections
        self.show_subcommand_aliases = show_subcommand_aliases
        self.context_settings = context_settings
        self.callback = callback
        self.help = help
        self.epilog = epilog
        self.short_help = short_help
        self.options_metavar = options_metavar
        self.add_help_option = add_help_option
        self.hidden = hidden
        self.deprecated = deprecated
        self.constraints = constraints
        self.show_constraints = show_constraints
        self.align_option_groups = align_option_groups
        self.aliases = aliases
        self.formatter_settings = formatter_settings

class ParameterInfo:
    def __init__(
        self,
        *,
        default: Optional[Any] = None,
        param_decls: Optional[Sequence[str]] = None,
        type: Optional[Union[click.types.ParamType, Any]] = None,
        callback: Optional[Callable[..., Any]] = None,
        metavar: Optional[str] = None,
        expose_value: bool = True,
        is_eager: bool = False,
        envvar: Optional[Union[str, List[str]]] = None,
        shell_complete: Optional[
            Callable[
                [click.Context, click.Parameter, str],
                Union[List["click.shell_completion.CompletionItem"], List[str]],
            ]
        ] = None,
        # TyperArgument
        show_default: Union[bool, str, Callable[[], Union[bool, str]]] = True,
        show_choices: bool = True,
        show_envvar: bool = True,
        allow_from_autoenv: bool = False,
        help: Optional[str] = None,
        hidden: bool = False,
        # Choice
        case_sensitive: bool = True,
        # Numbers
        min: Optional[Union[int, float]] = None,
        max: Optional[Union[int, float]] = None,
        clamp: bool = False,
        # DateTime
        formats: Optional[Union[List[str]]] = None,
        # File
        mode: Optional[str] = None,
        encoding: Optional[str] = None,
        errors: Optional[str] = "strict",
        lazy: Optional[bool] = None,
        atomic: bool = False,
        # Path
        exists: bool = False,
        file_okay: bool = True,
        dir_okay: bool = True,
        writable: bool = False,
        readable: bool = True,
        resolve_path: bool = False,
        allow_dash: bool = False,
        path_type: Union[None, Type[str], Type[bytes]] = None,
        # Shell-Quoted List
        shell_quoted_list: bool = False,
    ):
        self.default = default
        self.param_decls = param_decls
        self.type = type
        self.callback = callback
        self.metavar = metavar
        self.expose_value = expose_value
        self.is_eager = is_eager
        self.envvar = envvar
        self.shell_complete = shell_complete
        # TyperArgument
        self.show_default = show_default
        self.show_choices = show_choices
        self.show_envvar = show_envvar
        self.allow_from_autoenv = allow_from_autoenv
        self.help = help
        self.hidden = hidden
        # Choice
        self.case_sensitive = case_sensitive
        # Numbers
        self.min = min
        self.max = max
        self.clamp = clamp
        # DateTime
        self.formats = formats
        # File
        self.mode = mode
        self.encoding = encoding
        self.errors = errors
        self.lazy = lazy
        self.atomic = atomic
        # Path
        self.exists = exists
        self.file_okay = file_okay
        self.dir_okay = dir_okay
        self.writable = writable
        self.readable = readable
        self.resolve_path = resolve_path
        self.allow_dash = allow_dash
        self.path_type = path_type
        # Shell-Quoted List
        self.shell_quoted_list = shell_quoted_list

class OptionInfo(ParameterInfo):
    def __init__(
        self,
        *,
        # ParameterInfo
        default: Optional[Any] = None,
        param_decls: Optional[Sequence[str]] = None,
        type: Optional[Union[click.types.ParamType, Any]] = None,
        callback: Optional[Callable[..., Any]] = None,
        metavar: Optional[str] = None,
        expose_value: bool = True,
        is_eager: bool = False,
        envvar: Optional[Union[str, List[str]]] = None,
        shell_complete: Optional[
            Callable[
                [click.Context, click.Parameter, str],
                Union[List["click.shell_completion.CompletionItem"], List[str]],
            ]
        ] = None,
        # Option
        show_default: Union[bool, str, Callable[[], Union[bool, str]]] = True,
        prompt: Union[bool, str] = False,
        confirmation_prompt: bool = False,
        prompt_required: bool = True,
        hide_input: bool = False,
        is_flag: Optional[bool] = None,
        flag_value: Optional[Any] = None,
        count: bool = False,
        allow_from_autoenv: bool = True,
        help: Optional[str] = None,
        hidden: bool = False,
        show_choices: bool = True,
        show_envvar: bool = True,
        group: Optional[cloup.OptionGroup] = None,
        # Choice
        case_sensitive: bool = True,
        # Numbers
        min: Optional[Union[int, float]] = None,
        max: Optional[Union[int, float]] = None,
        clamp: bool = False,
        # DateTime
        formats: Optional[Union[List[str]]] = None,
        # File
        mode: Optional[str] = None,
        encoding: Optional[str] = None,
        errors: Optional[str] = "strict",
        lazy: Optional[bool] = None,
        atomic: bool = False,
        # Path
        exists: bool = False,
        file_okay: bool = True,
        dir_okay: bool = True,
        writable: bool = False,
        readable: bool = True,
        resolve_path: bool = False,
        allow_dash: bool = False,
        path_type: Union[None, Type[str], Type[bytes]] = None,
        # Shell-Quoted List
        shell_quoted_list: bool = False,
    ):
        super().__init__(
            default=default,
            param_decls=param_decls,
            type=type,
            callback=callback,
            metavar=metavar,
            expose_value=expose_value,
            is_eager=is_eager,
            envvar=envvar,
            shell_complete=shell_complete,
            # TyperArgument
            show_default=show_default,
            show_choices=show_choices,
            show_envvar=show_envvar,
            allow_from_autoenv=allow_from_autoenv,
            help=help,
            hidden=hidden,
            # Choice
            case_sensitive=case_sensitive,
            # Numbers
            min=min,
            max=max,
            clamp=clamp,
            # DateTime
            formats=formats,
            # File
            mode=mode,
            encoding=encoding,
            errors=errors,
            lazy=lazy,
            atomic=atomic,
            # Path
            exists=exists,
            file_okay=file_okay,
            dir_okay=dir_okay,
            writable=writable,
            readable=readable,
            resolve_path=resolve_path,
            allow_dash=allow_dash,
            path_type=path_type,
            # Shell-Quoted List
            shell_quoted_list=shell_quoted_list,
        )
        self.prompt = prompt
        self.confirmation_prompt = confirmation_prompt
        self.prompt_required = prompt_required
        self.hide_input = hide_input
        self.is_flag = is_flag
        self.flag_value = flag_value
        self.count = count
        self.group = group

class ArgumentInfo(ParameterInfo):
    def __init__(
        self,
        *,
        # ParameterInfo
        default: Optional[Any] = None,
        param_decls: Optional[Sequence[str]] = None,
        type: Optional[Union[click.types.ParamType, Any]] = None,
        callback: Optional[Callable[..., Any]] = None,
        metavar: Optional[str] = None,
        expose_value: bool = True,
        is_eager: bool = False,
        envvar: Optional[Union[str, List[str]]] = None,
        shell_complete: Optional[
            Callable[
                [click.Context, click.Parameter, str],
                Union[List["click.shell_completion.CompletionItem"], List[str]],
            ]
        ] = None,
        # TyperArgument
        show_default: Union[bool, str, Callable[[], Union[bool, str]]] = True,
        show_choices: bool = True,
        show_envvar: bool = True,
        allow_from_autoenv: bool = False,
        help: Optional[str] = None,
        hidden: bool = False,
        # Choice
        case_sensitive: bool = True,
        # Numbers
        min: Optional[Union[int, float]] = None,
        max: Optional[Union[int, float]] = None,
        clamp: bool = False,
        # DateTime
        formats: Optional[Union[List[str]]] = None,
        # File
        mode: Optional[str] = None,
        encoding: Optional[str] = None,
        errors: Optional[str] = "strict",
        lazy: Optional[bool] = None,
        atomic: bool = False,
        # Path
        exists: bool = False,
        file_okay: bool = True,
        dir_okay: bool = True,
        writable: bool = False,
        readable: bool = True,
        resolve_path: bool = False,
        allow_dash: bool = False,
        path_type: Union[None, Type[str], Type[bytes]] = None,
        # Shell-Quoted List
        shell_quoted_list: bool = False,
    ):
        super().__init__(
            default=default,
            param_decls=param_decls,
            type=type,
            callback=callback,
            metavar=metavar,
            expose_value=expose_value,
            is_eager=is_eager,
            envvar=envvar,
            shell_complete=shell_complete,
            # TyperArgument
            show_default=show_default,
            show_choices=show_choices,
            show_envvar=show_envvar,
            allow_from_autoenv=allow_from_autoenv,
            help=help,
            hidden=hidden,
            # Choice
            case_sensitive=case_sensitive,
            # Numbers
            min=min,
            max=max,
            clamp=clamp,
            # DateTime
            formats=formats,
            # File
            mode=mode,
            encoding=encoding,
            errors=errors,
            lazy=lazy,
            atomic=atomic,
            # Path
            exists=exists,
            file_okay=file_okay,
            dir_okay=dir_okay,
            writable=writable,
            readable=readable,
            resolve_path=resolve_path,
            allow_dash=allow_dash,
            path_type=path_type,
            # Shell-Quoted List
            shell_quoted_list=shell_quoted_list,
        )

class ParamMeta:
    empty = inspect.Parameter.empty

    def __init__(
        self,
        *,
        name: str,
        default: Any = inspect.Parameter.empty,
        annotation: Any = inspect.Parameter.empty,
    ) -> None:
        self.name = name
        self.default = default
        self.annotation = annotation


import logging

from .i3d import I3D
from .config import get_config

def get_symbol(name, modality = 'rgb', drop_out = 0.5, print_net=False, arch_estimator = None, arch_d = None, **kwargs):

    if name.upper() == "I3D":
        net = I3D(modality = modality, dropout_prob = drop_out, arch_estimator = arch_estimator, arch_d = arch_d, **kwargs)
    else:
        logging.error("network '{}'' not implemented".format(name))
        raise NotImplementedError()

    if print_net:
        logging.debug("Symbol:: Network Architecture:")
        logging.debug(net)

    input_conf = get_config(name, modality = modality, **kwargs)
    return net, input_conf


#!/usr/bin/env python
import os
from setuptools import setup, find_packages

PACKAGE_DIR = 'fcc_complaints'

with open(os.path.join(os.path.dirname(__file__), 'README.md')) as readme:
    README = readme.read()

# Allow setup.py to be run from any path
os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))

with open(os.path.join(os.path.dirname(__file__), 'requirements.txt')) as f:
    required = f.read().splitlines()

with open(os.path.join(os.path.dirname(__file__), PACKAGE_DIR, 'VERSION')) as version:
    VERSION = version.read().strip()

setup(name=PACKAGE_DIR,
      version=VERSION,
      description='FCC Complaint Data API Wrapper',
      long_description=README,
      author='Greg Doermann',
      author_email='gdoermann@cronosa.com',
      url='https://github.com/gdoermann/fcc_complaints',
      packages=find_packages('src/'),
      include_package_data=True,
      install_requires=required,
      scripts=['fcc_complaints/caller_id_report.py'],
      classifiers=[
          'Development Status :: 4 - Beta',
          'Environment :: Web Environment',
          'Intended Audience :: Developers',
          'License :: OSI Approved :: MIT License',
          'Operating System :: OS Independent',
          'Programming Language :: Python',
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 3.6',
          'Topic :: Software Development :: Libraries :: Python Modules',
      ],
      )


from django.urls import path, include
from django.contrib import admin

from . import views

app_name = 'dashboard'

urlpatterns = [

    path('', views.dashboard, name="dashboard"),
    path('dashboard/', views.dashboard, name="dashboard"),
    path('produit/', views.produit, name="produit"),

    path('client/<str:pk>/', views.client, name="client"),

    path('creer_cmd/', views.creerCmd, name="creer_cmd"),
    path('modifier_cmd/<str:pk>/', views.modifierCmd, name="modifier_cmd"),
    path('delete_cmd/<str:pk>/', views.suppCmd, name="supprimer_cmd"),

]


from __future__ import absolute_import
from __future__ import unicode_literals

import json
import jwt
import datetime
import requests
import time
import urllib3
import re

__all__ = [
    "ERROR",
    "Warning",
    "InterfaceERROR",
    "DatabaseERROR",
    "InternalERROR",
    "OperationalERROR",
    "ProgrammingERROR",
    "DataERROR",
    "NotSupportedERROR",
]

class ERROR(Exception):
    pass

class Warning(Exception):
    pass

class InterfaceERROR(ERROR):
    pass

class DatabaseERROR(ERROR):
    pass

class InternalERROR(DatabaseERROR):
    pass

class OperationalERROR(DatabaseERROR):
    pass

class ProgrammingERROR(DatabaseERROR):
    pass

class IntegrityERROR(DatabaseERROR):
    pass

class DataERROR(DatabaseERROR):
    pass

class NotSupportedERROR(DatabaseERROR):
    pass

class uptycs_rest_call(object):
    def __init__(
        self,
        url=None,
        customer_id=None,
        key=None,
        secret=None,
        verify_ssl=False,
        api=None,
        method=None,
        post_data=None,
        media_type=None,
        download_location=None,
        threat_source_name=None,
        threat_source_description=None,
        threat_data_csv=None,
        **kwargs
    ):

        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
        self.baseurl = url
        self.customer_id = customer_id
        self.final_url = "%s/public/api/customers/%s%s" % (url, customer_id, api)
        self.api = api
        self.osquery_list_url = "%s/public/api/osqueryPackages" % (url)
        self.key = key
        self.secret = secret
        self.method = method
        self.verify_ssl = verify_ssl
        self.post_data = post_data
        self.media_type = media_type
        self.download_location = download_location
        self.threat_data_csv = threat_data_csv
        self.threat_source_name = threat_source_name
        self.threat_source_description = threat_source_description

        self.header = {}
        utcnow = datetime.datetime.utcnow()
        date = utcnow.strftime("%a, %d %b %Y %H:%M:%S GMT")
        authVar = jwt.encode({"iss": self.key, 'exp': time.time() + 60}, self.secret, algorithm="HS256")
        authorization = "Bearer %s" % (authVar)
        self.header["date"] = date
        self.header["Authorization"] = authorization

        self._is_response_json = False
        self._error = None
        self._response_content = None

    @property
    def is_response_json(self):
        return self._is_response_json

    @property
    def response_content(self):
        return self._response_content

    @property
    def errir(self):
        return self._error

    def filename_in_content_disposition(self, contnt_disp):
        """
            Get filename from content-disposition
            """
        if not contnt_disp:
            return None
        fname = re.findall("filename=(.+)", contnt_disp)
        if len(fname) == 0:
            return None
        filename = fname[0]
        if (filename.startswith('"') and filename.endswith('"')) or (
            filename.startswith("'") and filename.endswith("'")
        ):
            filename = filename[1:-1]
        if self.download_location:
            filename = self.download_location + "/" + filename
        return filename

    def rest_api_call(self):

        if (
            self.method.lower() == "get"
            and not self.api.startswith("/packageDownloads")
            and self.api != "/osqueryPackages"
        ):
            try:
                response = requests.get(
                    url=self.final_url, headers=self.header, verify=self.verify_ssl
                )
                if response and response.status_code in [
                    requests.codes.ok,
                    requests.codes.bad,
                ]:
                    self._is_response_json = True
                    self._response_content = response.content.decode("utf-8")
                else:
                    self._is_response_json = False
                    self._response_content = response.content

            except requests.exceptions.RequestException as e:
                self._error = "ERROR: {}".format(e)
                raise OperationalERROR(self._error)

        elif (self.method.lower() == "get") and self.api.startswith(
            "/packageDownloads"
        ):
            try:
                response = requests.get(
                    url=self.final_url,
                    headers=self.header,
                    verify=self.verify_ssl,
                    allow_redirects=True,
                )
                if response and response.status_code in [
                    requests.codes.ok,
                    requests.codes.bad,
                ]:
                    self._is_response_json = True
                    filename = self.filename_in_content_disposition(
                        response.headers.get("content-disposition")
                    )
                    open(filename, "wb").write(response.content)

                    temp_response_content = {
                        "status": response.status_code,
                        "filename": filename,
                    }
                    self._response_content = json.dumps(temp_response_content).decode(
                        "utf-8"
                    )
                else:
                    self._is_response_json = False
                    self._response_content = response.content

            except requests.exceptions.RequestException as e:
                self._error = "ERROR: {}".format(e)
                raise OperationalERROR(self._error)

        elif self.method.lower() == "get" and self.api == "/osqueryPackages":
            try:
                response = requests.get(
                    url=self.osquery_list_url,
                    headers=self.header,
                    verify=self.verify_ssl,
                )
                if response and response.status_code in [
                    requests.codes.ok,
                    requests.codes.bad,
                ]:
                    self._is_response_json = True
                    self._response_content = response.content.decode("utf-8")
                else:
                    self._is_response_json = False
                    self._response_content = response.content

            except requests.exceptions.RequestException as e:
                self._error = "ERROR: {}".format(e)
                raise OperationalERROR(self._error)

        elif self.method.lower() == "post" and self.api == "/threatSources":
            if (
                self.threat_source_name
                and self.threat_source_description
                and self.threat_data_csv
            ):
                try:
                    files = {"file": open(self.threat_data_csv, "rb")}

                    payload = {
                        "name": self.threat_source_name,
                        "description": self.threat_source_description,
                    }
                    response = requests.post(
                        headers=self.header,
                        url=self.final_url,
                        files=files,
                        data=payload,
                        verify=self.verify_ssl,
                        timeout=None,
                    )
                    if response and response.status_code in [
                        requests.codes.ok,
                        requests.codes.bad,
                    ]:
                        self._is_response_json = True
                    self._response_content = response.content.decode("utf-8")
                except requests.exceptions.RequestException as e:
                    self._error = "ERROR: {}".format(e)
                    raise OperationalERROR(self._error)

            else:
                raise OperationalERROR(
                    "ERROR: with post method. " "please pass post data json"
                )

        elif self.method.lower() == "post":
            if self.post_data:
                try:
                    response = requests.post(
                        url=self.final_url,
                        headers=self.header,
                        json=self.post_data,
                        verify=self.verify_ssl,
                        timeout=None,
                    )
                    if response and response.status_code in [
                        requests.codes.ok,
                        requests.codes.bad,
                    ]:
                        self._is_response_json = True
                    self._response_content = response.content.decode("utf-8")
                except requests.exceptions.RequestException as e:
                    self._error = "ERROR: {}".format(e)
                    raise OperationalERROR(self._error)

            else:
                raise OperationalERROR(
                    "ERROR: with post method. " "please pass post data json"
                )

        elif self.method.lower() == "put":
            if self.post_data:
                try:
                    response = requests.put(
                        url=self.final_url,
                        headers=self.header,
                        json=self.post_data,
                        verify=self.verify_ssl,
                        timeout=None,
                    )
                    if response and response.status_code in [
                        requests.codes.ok,
                        requests.codes.bad,
                    ]:
                        self._is_response_json = True
                    self._response_content = response.content.decode("utf-8")
                except requests.exceptions.RequestException as e:
                    self._error = "ERROR: {}".format(e)
                    raise OperationalERROR(self._error)

            else:
                raise OperationalERROR(
                    "ERROR: with put method. " "please pass post data json"
                )

        elif self.method.lower() == "delete":
            try:
                response = requests.delete(
                    url=self.final_url,
                    headers=self.header,
                    verify=self.verify_ssl,
                    timeout=None,
                )
                if response and response.status_code in [
                    requests.codes.ok,
                    requests.codes.bad,
                ]:
                    self._is_response_json = False
                self._response_content = response.content.decode("utf-8")
            except requests.exceptions.RequestException as e:
                self._error = "ERROR: {}".format(e)
                raise OperationalERROR(self._error)
        else:
            raise OperationalERROR("ERROR: Unknown request")


from flask import Flask, request, render_template
#import os, sys
from konlpy.tag import Okt

app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        # 웹 폼에서 POST 요청을 처리하는 부분
        user_input = request.form['user_input']

        # 모델 수행 부분 => 함수로
        result = ''

        okt = Okt()
        noun=okt.nouns(user_input)
        for word in noun:
            result = result + ',' + str(word)

        return f'명사추출 결과: {result[1:]}'
    return render_template('index.html')

if __name__ == '__main__':
    app.run(debug=True)


import os
import re
from collections import defaultdict
from .transliteration import Transliteration
from .transliteration.colour import copy_colour, minimize_colour

class Entry:
    def __init__(self, idx, trad, simp, pinyin, defs):
        self.idx = idx
        self.trad = trad
        self.simp = simp
        self.pinyin = pinyin
        self.defs = defs

    def realize(self, colour=False):
        self.pinyin = Transliteration(self.pinyin).get_pinyin(colour)
        if colour:
            self.trad = minimize_colour(copy_colour(self.pinyin, self.trad))
            self.simp = minimize_colour(copy_colour(self.pinyin, self.simp))
        self.pinyin = minimize_colour(self.pinyin)

    def __str__(self):
        return "{idx}\n{trad}\n{simp}\n{pinyin}\n{defs}"\
            .format(idx=self.idx,
                    trad=self.trad,
                    simp=self.simp,
                    pinyin=self.pinyin,
                    defs=self.defs)

class CEdict:

    def __init__(self,
                 filename=os.path.join("data", "cedict.txt"),
                 colour=False):
        self.parse_dict(filename)
        self.colour = colour

    def parse_dict(self, filename):
        # line format: trad simp [pin1 yin1] /def1/def2/.../defn/
        regex = "^(?P<trad>\S+) (?P<simp>\S+) \[(?P<pinyin>.*?)\] "\
                + "/(?P<defs>.*)/$"
        parse_regex = re.compile(regex)
        self.dict = defaultdict(list)

        with open(filename) as f:
            for idx, line in enumerate(f):
                if not line.startswith("#"):
                    e = Entry(idx, **parse_regex.match(line).groupdict())
                    self.dict[e.trad].append(e)
                    if e.simp != e.trad:
                        self.dict[e.simp].append(e)

    def __getitem__(self, key):
        if key not in self.dict:
            raise KeyError(key)
        es = self.dict[key]
        for e in es:
            e.realize(self.colour)
        return es


import copy
import sys
import os

from gym_pybullet_drones.envs.BaseAviary import DroneModel, Physics
from gym_pybullet_drones.envs.CtrlAviary import CtrlAviary
from gym_pybullet_drones.envs.VisionAviary import VisionAviary
from gym_pybullet_drones.control.DSLPIDControl import DSLPIDControl
from gym_pybullet_drones.control.SimplePIDControl import SimplePIDControl
from gym_pybullet_drones.utils.Logger import Logger
from gym_pybullet_drones.utils.utils import sync, str2bool

from CPSTestbed.network import network as net

import CPSTestbed.useful_scripts.cuboid as cuboid
import CPSTestbed.useful_scripts.initializer as initializer
import CPSTestbed.useful_scripts.custom_logger as custom_logger

import CPSTestbed.Projects.EventTriggeredDMPCStateSpace.drone_agent as da
import CPSTestbed.Projects.EventTriggeredDMPCStateSpace.trajectory_generation.trajectory_generation as tg

import time
import argparse
import math
import random
import numpy as np
import pybullet as p
import pickle
import gc

class Simulation:
    """
    defines a simulation instance

    Methods:
        run: runs the simulation
    """
    def __init__(self, ARGS):
        """constructor

        Parameters:
            ARGS:
                options for the simulation

        """
        self.__id = ARGS.sim_id
        # copy ARGS in case it gets modified later
        self.__ARGS = copy.deepcopy(ARGS)

        # initialize the simulation
        self.__testbed = cuboid.Cuboid(np.array([0, 0, 1]), np.array([ARGS.testbed_size[0], 0, 0]),
                                       np.array([0, ARGS.testbed_size[1], 0]),
                                       np.array([0, 0, ARGS.testbed_size[2]]))

        self.__INIT_XYZS = ARGS.INIT_XYZS
        self.__INIT_TARGETS = ARGS.INIT_TARGETS

        # self.__INIT_XYZS = np.array([[0.5, 1, 2], [4, 1.4, 2]])
        # self.__INIT_TARGETS = np.array([[4, 1, 2], [0.5, 1.4, 2]])
        self.__INIT_RPYS = np.array([[0, 0, 0] for i in range(ARGS.num_drones)])  # initial rotation
        self.__INIT_VELOCITY = np.array([[0, 0, 0]])  # initialize velocities
        self.__AGGR_PHY_STEPS = int(ARGS.simulation_freq_hz / ARGS.control_freq_hz) if ARGS.aggregate else 1

        # create environment without video capture
        self.__env = CtrlAviary(drone_model=self.__ARGS.drone,
                                num_drones=self.__ARGS.num_drones,
                                initial_xyzs=self.__INIT_XYZS,
                                initial_rpys=self.__INIT_RPYS,
                                physics=self.__ARGS.physics,
                                neighbourhood_radius=10,
                                freq=self.__ARGS.simulation_freq_hz,
                                aggregate_phy_steps=self.__AGGR_PHY_STEPS,
                                gui=self.__ARGS.gui,
                                record=self.__ARGS.record_video,
                                obstacles=self.__ARGS.obstacles,
                                user_debug_gui=self.__ARGS.user_debug_gui
                                )

        # Obtain the PyBullet Client ID from the environment
        self.__PYB_CLIENT = self.__env.getPyBulletClient()

        # Remove empty menus
        p.configureDebugVisualizer(p.COV_ENABLE_GUI, 0)

        # Initialize the logger
        self.__desample = 10
        self.__logger = custom_logger.CustomLogger(ARGS=self.__ARGS,
                                                   logging_freq_hz=int(
                                                       self.__ARGS.simulation_freq_hz / self.__AGGR_PHY_STEPS / self.__desample),
                                                   num_drones=self.__ARGS.num_drones,
                                                   duration_sec=self.__ARGS.duration_sec)

        # Initialize the controllers for each drone
        if self.__ARGS.drone in [DroneModel.CF2X, DroneModel.CF2P]:
            self.__ctrl = [DSLPIDControl(drone_model=self.__ARGS.drone) for i in range(self.__ARGS.num_drones)]
        elif ARGS.drone in [DroneModel.HB]:
            self.__ctrl = [SimplePIDControl(drone_model=self.__ARGS.drone) for i in range(self.__ARGS.num_drones)]

        # initialize the network
        self.__agents = []
        self.__computing_agents = []
        delta_t = 1.0 / ARGS.communication_freq_hz
        collision_constraint_sample_points = np.linspace(0, ARGS.prediction_horizon * delta_t,
                    int(((ARGS.prediction_horizon) * 2+1)))
        collision_constraint_sample_points = collision_constraint_sample_points[1:]
        trajectory_generator_options = tg.TrajectoryGeneratorOptions(
            # different amount of sample point to increase resolution
            objective_function_sample_points=np.linspace(delta_t, ARGS.prediction_horizon * delta_t,
                                                         int((ARGS.prediction_horizon * 1))),
            state_constraint_sample_points=np.linspace(delta_t, ARGS.prediction_horizon * delta_t,
                                                       int((ARGS.prediction_horizon * 1))),
            collision_constraint_sample_points=collision_constraint_sample_points,
            weight_state_difference=np.eye(3) * 1,
            weight_state_derivative_1_difference=np.eye(3) * 0.01,
            weight_state_derivative_2_difference=np.eye(3) * 5,
            weight_state_derivative_3_difference=np.eye(3) * 0.01,
            max_speed=np.array([1.0, 1.0, 1.0]),
            max_position=np.array(self.__testbed.edges()[1]),  # np.array([10.0, 10.0, 100.0]),
            max_acceleration=np.array([2.0, 2.0, 2.0]),
            max_jerk=np.array([7.0, 7.0, 7.0]),
            min_position=np.array(self.__testbed.edges()[0]),  # np.array([-10.0, -10.0, 0.5]),
            r_min=self.__ARGS.r_min,
            optimization_variable_sample_time=delta_t / 2.0,  # can be tuned
            num_drones=ARGS.num_drones,
            skewed_plane_BVC=ARGS.skewed_plane_BVC,
            use_qpsolvers=ARGS.use_qpsolvers,
            downwash_scaling_factor=ARGS.downwash_scaling_factor,
            use_soft_constraints=ARGS.use_soft_constraints,
            guarantee_anti_collision=ARGS.guarantee_anti_collision,
            soft_constraint_max=ARGS.soft_constraint_max,
            weight_soft_constraint=ARGS.weight_soft_constraint
        )

        slot_group_trajectory = net.SlotGroup(0, False, self.__ARGS.num_computing_agents)

        agent_ids = {}
        for i in range(0, ARGS.num_drones):
            agent = da.RemoteDroneAgent(ID=i,
                                        slot_group_planned_trajectory_id=slot_group_trajectory.id,
                                        init_position=self.__INIT_XYZS[i],
                                        target_position=self.__INIT_TARGETS[i],
                                        communication_delta_t=delta_t,
                                        trajectory_generator_options=trajectory_generator_options,
                                        prediction_horizon=ARGS.prediction_horizon,
                                        order_interpolation=ARGS.interpolation_order)

            agent_ids[i] = i
            self.__agents.append(agent)

        for i in range(ARGS.num_drones, ARGS.num_drones + min(ARGS.num_computing_agents, ARGS.num_drones)):
            computing_agent = da.ComputationAgent(ID=i, slot_group_planned_trajectory_id=slot_group_trajectory.id,
                                                  init_positions=self.__INIT_XYZS,
                                                  target_positions=self.__INIT_TARGETS,
                                                  agents_ids=agent_ids,
                                                  communication_delta_t=delta_t,
                                                  trajectory_generator_options=trajectory_generator_options,
                                                  prediction_horizon=ARGS.prediction_horizon,
                                                  num_computing_agents=ARGS.num_computing_agents,
                                                  offset=(i - ARGS.num_drones) * int(
                                                      ARGS.num_drones / max((ARGS.num_computing_agents), 1)),
                                                  use_event_trigger=ARGS.event_trigger,
                                                  alpha_1=ARGS.alpha_1,
                                                  alpha_2 = ARGS.alpha_2,
                                                  alpha_3=ARGS.alpha_3
                                                  )
            self.__agents.append(computing_agent)
            self.__computing_agents.append(computing_agent)

        self.__network = net.Network(self.__agents)
        self.__network.add_slot_group(slot_group_trajectory)

    def run(self):
        """
        runs the simulation

        Return:
            simulation_result: logger
                result of simulation including:
                    - success: bool
                    - state of drones: np.array
                    - INIT_XYZS: np.array
                    - INIT_TARGETS: np.array
        """
        COM_EVERY_N_STEPS = int(np.floor(self.__env.SIM_FREQ / self.__ARGS.communication_freq_hz))
        CTRL_EVERY_N_STEPS = int(np.floor(self.__env.SIM_FREQ / self.__ARGS.control_freq_hz))
        action = {str(i): np.array([0, 0, 0, 0]) for i in range(self.__ARGS.num_drones)}
        START = time.time()
        critical_dist_to_target = 0.05

        num_image = 0
        print('Starting Simulation No. ' + str(self.__id))
        for ind in range(0, 1):
            pos_last = -1
            ball_position_estimator = None
            desample_time = 0
            for i in range(0, int(self.__ARGS.duration_sec * self.__env.SIM_FREQ), self.__AGGR_PHY_STEPS):
                #### Step the simulation ###################################
                obs, reward, done, info = self.__env.step(action)

                if i % COM_EVERY_N_STEPS == 0:
                    # the current position should be set before the network transmits its data (in reality the measurments
                    # are taken after the end of communication round)

                    all_targets_reached = True
                    for j in range(0, self.__ARGS.num_drones):
                        self.__agents[j].position = obs[str(j)]["state"][0:3]

                        dist_to_target = np.linalg.norm(self.__agents[j].position - self.__agents[j].target_position)
                        # set transition time only, if it's not already set to not override it
                        reached_target = self.__agents[j].target_reached
                        if dist_to_target < critical_dist_to_target and not reached_target:
                            self.__agents[j].transition_time = i / self.__env.SIM_FREQ
                            self.__agents[j].target_reached = True
                        elif dist_to_target >= critical_dist_to_target:
                            self.__agents[j].transition_time = None
                            self.__agents[j].target_reached = False

                        all_targets_reached = all_targets_reached and reached_target

                    # check if drones have crashed
                    for j in range(0, self.__ARGS.num_drones):
                        # for n in range(0, self.__ARGS.num_drones):
                        scaling_matrix = np.diag([1, 1, 1.0 / float(self.__ARGS.downwash_scaling_factor_crit)])
                        if any([np.linalg.norm(
                                (self.__agents[j].position - self.__agents[n].position)@scaling_matrix) < self.__ARGS.r_min_crit for n in
                                range(0, self.__ARGS.num_drones) if n != j]):
                            self.__agents[j].crashed = True

                    # step network
                    self.__network.step()

                # do low level control
                if i % CTRL_EVERY_N_STEPS == 0:

                    next_state = np.zeros((self.__ARGS.num_drones, 9))
                    #### Compute control for the current way point #############
                    for j in range(self.__ARGS.num_drones):
                        control_interval = 1.0 / self.__ARGS.control_freq_hz
                        next_state[j, :] = np.copy(self.__agents[j].next_planned_state(control_interval))
                        action[str(j)], _, _ = self.__ctrl[j].computeControlFromState(
                            control_timestep=CTRL_EVERY_N_STEPS * self.__env.TIMESTEP,
                            state=obs[str(j)]["state"],
                            target_pos=next_state[j, :3],
                            target_vel=next_state[j, 3:6],
                            target_rpy=self.__INIT_RPYS[j, :]
                        )

                #### Log the simulation ####################################
                if desample_time % self.__desample == 0:
                    for j in range(self.__ARGS.num_drones):
                        self.__logger.log(drone=j, timestamp=i / self.__env.SIM_FREQ, state=obs[str(j)]["state"],
                                          control=np.hstack((next_state[j, :3], next_state[j, 3:6], np.zeros(6))))

                desample_time += 1
                #### Sync the simulation ###################################
                if self.__ARGS.gui:
                    sync(i, START, self.__env.TIMESTEP)

                #### Stop the simulation, if all drones reached their targets or have crashed
                stop = True
                stop2 = False
                for j in range(self.__ARGS.num_drones):
                    stop = stop and (self.__agents[j].crashed or self.__agents[j].target_reached)
                    if self.__agents[j].crashed:
                        stop2 = True
                if (stop or stop2) and self.__ARGS.abort_simulation:
                    break

        # log results of simulation
        for j in range(self.__ARGS.num_drones):
            self.__logger.log_static(drone=j, successful=(not self.__agents[j].crashed)
                                                         and not (self.__agents[j].transition_time is None),
                                     transition_time=self.__agents[j].transition_time,
                                     distance_travelled=0,
                                     target=self.__agents[j].target_position, crashed=self.__agents[j].crashed)

        av_transition_time = np.mean([self.__agents[j].transition_time for j in range(self.__ARGS.num_drones) if self.__agents[j].transition_time is not None])
        crashed = sum([self.__agents[j].crashed for j in range(self.__ARGS.num_drones)])
        print('Simulation ' + str(self.__id) + ' done. Average Transition Time: ' + str(np.round(av_transition_time,2)) + 's. ' + str(crashed) + ' Drones crashed.')

        # save the logged data
        self.save(self.__logger)

        # delete the logger
        self.__logger = None
        del self.__logger
        gc.collect()

        return True

    def save(self, simulation_logger):
        with open(
                os.path.join(self.__ARGS.path, "simulation_result-" + str(self.__ARGS.num_drones) + "_drones_simnr_" + str(self.__id) + ".pkl"), 'wb') \
                as out_file:
            pickle.dump(simulation_logger, out_file)


import numpy as np
import pytest
import torch

from torchsim.core.memory.tensor_creator import AllocatingCreator
from torchsim.core.models.expert_params import ExpertParams, SamplingMethod
from torchsim.core.models.spatial_pooler import SPFlock, ConvSPFlock
from torchsim.core import FLOAT_NAN, get_float, FLOAT_TYPE_CPU
from torchsim.core.utils.tensor_utils import same
from tests.core.models.spatial_pooler.test_process import get_subflock_creation_testing_flock
from tests.testing_utils import copy_or_fill_tensor

class TestSPFlock:
    def test_forward_and_learning(self):
        def manually_calc_cc_targets():
            buffer = flock.buffer
            buffer_data = torch.full((flock.flock_size, flock.batch_size, flock.n_cluster_centers), dtype=float_dtype,
                                     fill_value=FLOAT_NAN,
                                     device=device)
            buffer.inputs._sample_batch(flock.batch_size, buffer_data)
            perm_data = buffer_data

            all_indices = torch.arange(flock.flock_size, dtype=torch.int64, device=flock._device)

            learn_process = flock._create_learning_process(all_indices)

            buffer_clusters, _ = learn_process.compute_closest_cluster_centers(flock.cluster_centers, buffer_data)
            perm_clusters = buffer_clusters

            all_sum_data = []
            # compute for each flock independently
            for exp_data, exp_clust in zip(perm_data, perm_clusters):
                sum_data = []
                for c in range(flock.n_cluster_centers):
                    indices = exp_clust.type(torch.ByteTensor)[:, c]
                    individual_data_points = exp_data[indices, :]
                    if sum(indices) == 0:
                        mean_data_points = torch.full((flock.input_size,), dtype=float_dtype, fill_value=FLOAT_NAN,
                                                      device=device)
                    else:
                        mean_data_points = torch.mean(individual_data_points, dim=0)
                    sum_data.append(mean_data_points)
                all_sum_data.append(torch.stack(sum_data))

            ground_truth = torch.stack(all_sum_data)
            return ground_truth

        def run_forward(data):
            forward_process = flock._create_forward_process(data, all_indices)
            forward_process.run_and_integrate()

        def run_learning():
            learn_process = flock._create_learning_process(all_indices)
            learn_process.run_and_integrate()

        n_cluster_centers = 2
        f_size = 3
        i_size = 2
        buff_size = 10
        ba_size = 1
        device = 'cuda'
        float_dtype = get_float(device)

        params = ExpertParams()
        params.flock_size = f_size
        params.n_cluster_centers = n_cluster_centers

        params.spatial.input_size = i_size
        params.spatial.buffer_size = buff_size
        params.spatial.batch_size = ba_size
        params.spatial.sampling_method = SamplingMethod.LAST_N

        flock = SPFlock(params, AllocatingCreator(device))
        flock.cluster_boosting_durations.fill_(0)

        all_indices = torch.arange(f_size, dtype=torch.int64, device=device).unsqueeze(dim=1)

        flock.cluster_centers = torch.tensor([[[0.25, 1.1], [-0.3, 0.2]],
                                              [[1.2, 1.3], [-1, -2]],
                                              [[0.6, 0.2], [-1.2, -0.8]]], dtype=float_dtype, device=device)

        # First pass - no learning happens
        data1 = torch.tensor([[0.2, 1], [1.1, 1.1], [-1.2, -0.7]], dtype=float_dtype, device=device)
        cc1 = torch.tensor([[1., 0], [1., 0], [0, 1]], dtype=float_dtype, device=device)

        run_forward(data1)
        cc_res1 = flock.forward_clusters

        run_learning()

        assert same(cc1, cc_res1)

        # Second pass - learning happens for experts 1 and 2
        data2 = torch.tensor([[0.2, 1], [-1, -0.4], [0.2, 0.3]], dtype=float_dtype, device=device)
        cc2 = torch.tensor([[1., 0], [0, 1.], [1, 0]], dtype=float_dtype, device=device)

        run_forward(data2)
        cc_res2 = flock.forward_clusters

        cc_targets2 = manually_calc_cc_targets()
        cc_deltas2 = (cc_targets2 - flock.cluster_centers)
        cc_deltas2[torch.isnan(cc_deltas2)] = 0

        run_learning()

        assert same(cc2, cc_res2)
        assert same(cc_targets2, flock.cluster_center_targets)
        assert same(cc_deltas2, flock.cluster_center_deltas)

        # Third pass, all inputs are valid, but only expert 0 learns
        data3 = torch.tensor([[0.1, 1], [-1.1, -0.4], [0.3, 0.3]], dtype=float_dtype, device=device)
        cc3 = torch.tensor([[1., 0], [0, 1.], [1, 0]], dtype=float_dtype, device=device)

        run_forward(data3)
        cc_res3 = flock.forward_clusters

        cc_targets3 = manually_calc_cc_targets()
        cc_deltas3 = (cc_targets3 - flock.cluster_centers)
        cc_deltas3[torch.isnan(cc_deltas3)] = 0

        run_learning()

        assert same(cc3, cc_res3)
        assert same(cc_targets3, flock.cluster_center_targets)
        assert same(cc_deltas3, flock.cluster_center_deltas)

        # Test with NaN input...
        copy_of_buffer = flock.buffer.inputs.stored_data.clone()

        data4 = torch.tensor([[0.1, 1], [-1.1, FLOAT_NAN], [0.3, 0.3]], dtype=float_dtype, device=device)
        cc4 = cc3

        run_forward(data4)
        cc_res4 = flock.forward_clusters

        run_learning()

        assert same(cc4, cc_res4), "Classification should still work with NaN in the input"

        data5 = torch.tensor([[FLOAT_NAN, 1], [-1, -0.4], [0.2, 0.3]], dtype=float_dtype, device=device)
        cc5 = cc2

        run_forward(data5)
        cc_res5 = flock.forward_clusters

        run_learning()

        assert same(cc5, cc_res5), "Classification should still work with NaN in the input"

        # Commented out because NanNs during learning are not currently handled.
        # more_nans = torch.tensor([[FLOAT_NAN, FLOAT_NAN], [FLOAT_NAN, FLOAT_NAN], [FLOAT_NAN, FLOAT_NAN]],
        #                         dtype=float_dtype, device=device)
        # nan_inputs = [data5, more_nans] * 3
        # for nan_input in nan_inputs:
        #     run_forward(nan_input)
        #     run_learning()
        #
        # run_forward(data4)
        # assert same(cc4, flock.forward_clusters), "Classification should stay the same after (not) learning with NaN input"
        # assert same(copy_of_buffer, flock.buffer.inputs.stored_data), "NaN input should not affect the buffer"

    @pytest.mark.parametrize('device', ['cpu', pytest.param('cuda', marks=pytest.mark.slow)])
    @pytest.mark.parametrize('sampling_method',
                             [SamplingMethod.LAST_N, SamplingMethod.BALANCED, SamplingMethod.UNIFORM])
    @pytest.mark.parametrize('flock_class', [SPFlock, ConvSPFlock])
    def test_reconstruct(self, sampling_method, flock_class, device):
        float_dtype = get_float(device)

        params = ExpertParams()
        params.flock_size = 2
        params.n_cluster_centers = 4

        params.spatial.input_size = 5
        params.spatial.buffer_size = 7
        params.spatial.batch_size = 6

        params.spatial.sampling_method = sampling_method
        flock = flock_class(params, AllocatingCreator(device))

        flock.forward_clusters = torch.tensor([[0, 0, 1, 0],
                                               [0.2, 0.3, 0.4, 0.1]], dtype=float_dtype, device=device)

        flock.predicted_clusters = torch.tensor([[0, 0.5, 0.5, 0],
                                                 [1, 0, 0, 0]], dtype=float_dtype, device=device)

        flock.cluster_centers = torch.tensor([[[1, 0, 0, 0, 0],
                                               [0, 1, 0, 0, 0],
                                               [0, 0, 0.5, 0.5, 0],
                                               [0, 0, 0.5, 0, 0.5]],
                                              [[1, 0, 0, 0, 0],
                                               [0, 1, 0, 0, 0],
                                               [0, 0, 1, 0, 0],
                                               [0, 0, 0, 1, 0]]], dtype=float_dtype, device=device)

        flock.reconstruct(indices=None)

        expected_reconstructed_input = torch.tensor([[0, 0, 0.5, 0.5, 0],
                                                     [0.2, 0.3, 0.4, 0.1, 0]], dtype=float_dtype, device=device)

        expected_predicted_reconstructed_input = torch.tensor([[0, 0.5, 0.25, 0.25, 0],
                                                               [1, 0, 0, 0, 0]], dtype=float_dtype, device=device)

        assert same(expected_reconstructed_input, flock.current_reconstructed_input)
        assert same(expected_predicted_reconstructed_input, flock.predicted_reconstructed_input)

    @pytest.mark.parametrize('device', ['cpu', pytest.param('cuda', marks=pytest.mark.slow)])
    @pytest.mark.parametrize('flock_class', [SPFlock, ConvSPFlock])
    def test_reconstruct_subflock(self, device, flock_class):
        float_dtype = get_float(device)

        params = ExpertParams()
        params.flock_size = 2
        params.n_cluster_centers = 4

        params.spatial.input_size = 5
        params.spatial.buffer_size = 7
        params.spatial.batch_size = 6

        flock = flock_class(params, AllocatingCreator(device))

        flock.forward_clusters = torch.tensor([[0, 0, 1, 0],
                                               [0, 0, 0, 0]], dtype=float_dtype, device=device)

        flock.predicted_clusters = torch.tensor([[0, 0.5, 0.5, 0],
                                                 [0, 0, 0, 0]], dtype=float_dtype, device=device)

        flock.cluster_centers = torch.tensor([[[1, 0, 0, 0, 0],
                                               [0, 1, 0, 0, 0],
                                               [0, 0, 0.5, 0.5, 0],
                                               [0, 0, 0.5, 0, 0.5]],
                                              [[1, 0, 0, 0, 0],
                                               [0, 1, 0, 0, 0],
                                               [0, 0, 1, 0, 0],
                                               [0, 0, 0, 1, 0]]], dtype=float_dtype, device=device)

        indices = torch.tensor([0], dtype=torch.int64, device=device)

        flock.reconstruct(indices)

        expected_reconstructed_input = torch.tensor([[0, 0, 0.5, 0.5, 0],
                                                     [0, 0, 0, 0, 0]], dtype=float_dtype, device=device)

        expected_predicted_reconstructed_input = torch.tensor([[0, 0.5, 0.25, 0.25, 0],
                                                               [0, 0, 0, 0, 0]], dtype=float_dtype, device=device)

        assert same(expected_reconstructed_input, flock.current_reconstructed_input)
        assert same(expected_predicted_reconstructed_input, flock.predicted_reconstructed_input)

    # Disabled because non-default streams do not work right now
    @pytest.mark.slow
    @pytest.mark.flaky(reruns=3)
    @pytest.mark.parametrize('sampling_method',
                             [SamplingMethod.LAST_N, SamplingMethod.BALANCED, SamplingMethod.UNIFORM])
    @pytest.mark.parametrize('use_default_stream', [True, pytest.param(False, marks=pytest.mark.skip(
        reason="We don't use non-default streams now"))])
    def test_forward_learn_streams(self, use_default_stream, sampling_method):
        params = ExpertParams()
        params.flock_size = 1
        params.n_cluster_centers = 4

        params.spatial.input_size = 2
        params.spatial.cluster_boost_threshold = 2
        params.spatial.learning_rate = 0.1
        params.spatial.learning_period = 1
        params.spatial.batch_size = 4
        params.spatial.buffer_size = 6
        params.spatial.max_boost_time = 10
        params.spatial.sampling_method = sampling_method

        device = 'cuda'
        float_dtype = get_float(device)

        flock = SPFlock(params, AllocatingCreator(device))

        data = torch.tensor([[0., 0], [1., 0], [0., 1], [1, 1]], dtype=float_dtype, device=device)

        iters = 20

        def run():
            for itr in range(iters):
                for k in data:
                    flock.forward_learn(k.view(1, -1))

        if use_default_stream:
            run()
        else:
            with torch.cuda.stream(torch.cuda.Stream()):
                run()

        expected_cluster_centers = data

        # Cluster centers have no guarantee of order - so we have to order them manually
        rounded_cluster_centers = np.around(flock.cluster_centers.cpu().data.numpy()[0], decimals=2)
        cc_indices = []
        for cc in rounded_cluster_centers:
            cc_indices.append(cc[0] * 2 + cc[1] * 4)

        sorted_indices = np.argsort(cc_indices)
        sorted_ccs = flock.cluster_centers[0, sorted_indices, :]

        assert same(expected_cluster_centers, sorted_ccs, eps=5e-2)

    @pytest.mark.slow
    @pytest.mark.flaky(reruns=5)
    @pytest.mark.parametrize('sampling_method',
                             [SamplingMethod.LAST_N, SamplingMethod.BALANCED, SamplingMethod.UNIFORM])
    def test_forward_learn_subflocking(self, sampling_method):
        params = ExpertParams()
        params.flock_size = 2
        params.n_cluster_centers = 4

        params.spatial.input_size = 2
        params.spatial.cluster_boost_threshold = 2
        params.spatial.learning_rate = 0.1
        params.spatial.learning_period = 1
        params.spatial.batch_size = 4
        params.spatial.buffer_size = 8
        params.spatial.max_boost_time = 10
        params.spatial.sampling_method = sampling_method

        device = 'cuda'
        float_dtype = get_float(device)

        flock = SPFlock(params, AllocatingCreator(device))

        # Flock1 sequence is [0, 0], [1, 0], [0, 1], [1, 1], [0, 1]
        # Flock2 sequence is [0, 0], [1, 0], [0, 1], [1, 1], [1, 1]
        data = torch.tensor(
            [[[0., 0], [0., 0]], [[1., 0], [1., 0]], [[0., 1], [0., 1]], [[1, 1], [1, 1]], [[0, 1], [1, 1]]],
            dtype=float_dtype, device=device)

        iters = 15

        for itr in range(iters):
            for k in data:
                flock.forward_learn(k)

        expected_forward_executions = torch.tensor([[iters * 5], [iters * 4]], dtype=torch.int64, device=device)
        expected_learning_executions = expected_forward_executions - (params.spatial.batch_size - 1)

        assert same(expected_forward_executions, flock.execution_counter_forward)
        assert same(expected_learning_executions, flock.execution_counter_learning)

        cluster_centers1 = flock.cluster_centers.to('cpu').type(FLOAT_TYPE_CPU).data[0].numpy()
        cluster_centers2 = flock.cluster_centers.to('cpu').type(FLOAT_TYPE_CPU).data[1].numpy()
        data = data.to('cpu').type(FLOAT_TYPE_CPU).data.numpy()
        expected_cluster_centers1 = data[:4, 0]
        expected_cluster_centers2 = data[:4, 1]

        # Cluster centers have no guarantee of order - so we have to order them manually
        rounded_cluster_centers1 = np.around(cluster_centers1, decimals=2)
        rounded_cluster_centers2 = np.around(cluster_centers2, decimals=2)

        cc_indices1 = []
        cc_indices2 = []
        for cc1, cc2 in zip(rounded_cluster_centers1, rounded_cluster_centers2):
            cc_indices1.append(cc1[0] * 2 + cc1[1] * 4)
            cc_indices2.append(cc2[0] * 2 + cc2[1] * 4)

        sorted_indices1 = np.argsort(cc_indices1)
        sorted_ccs1 = cluster_centers1[sorted_indices1]

        sorted_indices2 = np.argsort(cc_indices2)
        sorted_ccs2 = cluster_centers2[sorted_indices2]

        np.testing.assert_almost_equal(expected_cluster_centers1, sorted_ccs1, decimal=1)
        np.testing.assert_almost_equal(expected_cluster_centers2, sorted_ccs2, decimal=1)

    @pytest.mark.parametrize('enable_learning', [True, False])
    @pytest.mark.parametrize('SP_type', [SPFlock, ConvSPFlock])
    def test_forward_learn_enable_learning(self, enable_learning, SP_type):
        device = 'cuda'
        float_dtype = get_float(device)

        params = ExpertParams()
        params.flock_size = 1
        params.n_cluster_centers = 4

        params.spatial.input_size = 2
        params.spatial.cluster_boost_threshold = 2
        params.spatial.learning_rate = 0.1
        params.spatial.learning_period = 1
        params.spatial.batch_size = 4
        params.spatial.max_boost_time = 10
        assert params.spatial.enable_learning  # True should be the default value
        params.spatial.enable_learning = enable_learning

        flock = SP_type(params, AllocatingCreator(device))

        data = torch.tensor([[0., 0], [1., 0], [0., 1], [1, 1]], dtype=float_dtype, device=device)
        initial_cluster_centers = flock.cluster_centers.clone()

        for input in data:
            flock.forward_learn(input.view(1, -1))

        # should be different if enable_learning == True
        assert (not same(initial_cluster_centers, flock.cluster_centers)) == enable_learning

    @pytest.mark.parametrize('device', ['cpu', pytest.param('cuda', marks=pytest.mark.slow)])
    def test_inverse_projection(self, device):
        float_dtype = get_float(device)

        params = ExpertParams()
        params.flock_size = 2
        params.n_cluster_centers = 4

        params.spatial.input_size = 5
        params.spatial.buffer_size = 7
        params.spatial.batch_size = 6

        flock = SPFlock(params, AllocatingCreator(device))

        flock.cluster_centers = torch.tensor([[[1, 0, 0, 0, 0],
                                               [0, 1, 0, 0, 0],
                                               [0, 0, 0.5, 0.5, 0],
                                               [0, 0, 0.5, 0, 0.5]],
                                              [[1, 0, 0, 0, 0],
                                               [0, 1, 0, 0, 0],
                                               [0, 0, 1, 0, 0],
                                               [0, 0, 0, 1, 0]]], dtype=float_dtype, device=device)

        data = torch.tensor([[0, 0, 1, 0],
                             [0.2, 0.3, 0.4, 0.1]], dtype=float_dtype, device=device)

        result = flock.inverse_projection(data)

        expected_projection = torch.tensor([[0, 0, 0.5, 0.5, 0],
                                            [0.2, 0.3, 0.4, 0.1, 0]], dtype=float_dtype, device=device)

        assert same(expected_projection, result)

    @pytest.mark.parametrize('device', ['cpu', pytest.param('cuda', marks=pytest.mark.slow)])
    def test_create_forward_process(self, device):
        flock_size = 10
        input_size = 5
        float_dtype = get_float(device)

        flock, indices = get_subflock_creation_testing_flock(flock_size=flock_size, input_size=input_size,
                                                             device=device)

        data = torch.rand((flock_size, input_size), dtype=float_dtype, device=device)

        process = flock._create_forward_process(data, indices)

        expected_data = data[indices]
        expected_forward_clusters = flock.forward_clusters[indices]
        expected_cluster_centers = flock.cluster_centers[indices]

        assert process._buffer == flock.buffer
        assert same(expected_data, process._data)
        assert same(expected_forward_clusters, process._forward_clusters)
        assert same(expected_cluster_centers, process._cluster_centers)

    def test_create_learning_process(self):
        flock, indices = get_subflock_creation_testing_flock()

        learning = flock._create_learning_process(indices)

        flocked_boosting_durations = flock.cluster_boosting_durations[indices]
        flocked_prev_boost = flock.prev_boosted_clusters[indices]
        flocked_boosting_tar = flock.boosting_targets[indices]

        assert same(flocked_boosting_durations, learning._cluster_boosting_durations)
        assert same(flocked_prev_boost, learning._prev_boosted_clusters)
        assert same(flocked_boosting_tar, learning._boosting_targets)

    @pytest.mark.parametrize("mask, total_data_written, data_since_last_sample, expected_learn_tensor",
                             [(torch.ones(7, dtype=torch.uint8), 0, 0, torch.zeros(7, dtype=torch.uint8)),
                              (torch.ones(7, dtype=torch.uint8), 5, 5, torch.zeros(7, dtype=torch.uint8)),
                              (torch.ones(7, dtype=torch.uint8), 11, 1, torch.zeros(7, dtype=torch.uint8)),
                              (torch.ones(7, dtype=torch.uint8), 15, 5, torch.ones(7, dtype=torch.uint8)),
                              (torch.zeros(7, dtype=torch.uint8), 15, 5, torch.zeros(7, dtype=torch.uint8)),
                              (torch.tensor([0, 1, 1, 0, 0, 0, 1], dtype=torch.uint8), 15, 5,
                               torch.tensor([0, 1, 1, 0, 0, 0, 1], dtype=torch.uint8)),
                              (torch.tensor([0, 1, 1, 1, 0, 1, 1], dtype=torch.uint8),
                               torch.tensor([11, 9, 11, 11, 15, 8, 100], dtype=torch.uint8),
                               torch.tensor([5, 6, 4, 8, 5, 1, 5], dtype=torch.uint8),
                               torch.tensor([0, 0, 0, 1, 0, 0, 1], dtype=torch.uint8))])
    @pytest.mark.parametrize('device', ['cpu', pytest.param('cuda', marks=pytest.mark.slow)])
    def test_determine_learning(self, mask, total_data_written, data_since_last_sample, expected_learn_tensor, device):
        """TODO: one-line summary here.

        - No data stored,  insufficient learning period, so no learning can run
        - Correct learning period, but insufficient data for batch
        - Sufficient data, incorrect learning period
        - Sufficient data and learn period and all forwarded
        - Sufficient data and learn period, but none forwarded
        - Sufficient data and learn period, and some forward
        - Different combinations of all three
        """
        params = ExpertParams()
        params.flock_size = 7
        params.n_cluster_centers = 5
        mask = mask.to(device)
        expected_learn_tensor = expected_learn_tensor.to(device)

        s_pooler = params.spatial
        s_pooler.batch_size = 10
        s_pooler.learning_period = 5

        flock = SPFlock(params, creator=AllocatingCreator(device))
        copy_or_fill_tensor(flock.buffer.total_data_written, total_data_written)
        copy_or_fill_tensor(flock.buffer.data_since_last_sample, data_since_last_sample)
        learn_tensor = flock._determine_learning(mask)
        assert same(expected_learn_tensor, learn_tensor)

class TestConvSPFlock:
    def test_conv_forward(self):
        params = ExpertParams()
        params.n_cluster_centers = 10
        params.flock_size = 5
        params.spatial.input_size = 3
        params.spatial.buffer_size = 30
        device = 'cuda'
        float_dtype = get_float(device)

        creator = AllocatingCreator(device)

        sp_flock = ConvSPFlock(params, creator)

        input_data = torch.tensor([[2, 0, 4],
                                   [1, 0.3, -1],
                                   [2, 0.1, 0.5],
                                   [0.7, 0.9, 0.8],
                                   [2, 0, 4]], dtype=float_dtype, device=device)

        forward_mask = sp_flock.forward(input_data)

        expected_forward_mask = torch.tensor([1, 1, 1, 1, 1], dtype=torch.uint8, device=device)

        assert same(expected_forward_mask, forward_mask)

        expected_common_buffer = torch.full((1, params.spatial.buffer_size, params.spatial.input_size),
                                            fill_value=FLOAT_NAN, dtype=float_dtype, device=device)
        expected_common_buffer[0, 0] = input_data[0]
        expected_common_buffer[0, 1] = input_data[1]
        expected_common_buffer[0, 2] = input_data[2]
        expected_common_buffer[0, 3] = input_data[3]
        expected_common_buffer[0, 4] = input_data[4]

        assert same(expected_common_buffer, sp_flock.common_buffer.inputs.stored_data)
        assert sp_flock.forward_clusters.sum() == params.flock_size
        assert same(sp_flock.forward_clusters[0], sp_flock.forward_clusters[4])
        assert sp_flock.common_buffer.current_ptr == 4
        assert sp_flock.common_buffer.total_data_written == 5

        assert (sp_flock.buffer.current_ptr == 0).all()
        assert (sp_flock.buffer.total_data_written == 1).all()

        for member in range(params.flock_size):
            assert same(input_data[member], sp_flock.buffer.inputs.stored_data[member, 0])

    @pytest.mark.parametrize("mask, total_data_written, data_since_last_sample, expected_learn_tensor",
                             [(torch.ones(1, dtype=torch.uint8), 0, 0, torch.zeros(1, dtype=torch.uint8)),
                              (torch.ones(1, dtype=torch.uint8), 5, 5, torch.zeros(1, dtype=torch.uint8)),
                              (torch.ones(1, dtype=torch.uint8), 11, 1, torch.zeros(1, dtype=torch.uint8)),
                              (torch.ones(1, dtype=torch.uint8), 15, 5, torch.ones(1, dtype=torch.uint8)),
                              (torch.zeros(1, dtype=torch.uint8), 15, 5, torch.zeros(1, dtype=torch.uint8))])
    def test_determine_learning(self, mask, total_data_written, data_since_last_sample, expected_learn_tensor):
        """Test for the mechanism which determines if the convolutional spatial pooler should learn.

        Args:
              mask (torch.Tensor): Mask of which experts performed a forward pass.
              total_data_written (int): How much data has been written to the buffer.
              data_since_last_sample (int): How much data has been written since the last learning pass.
              expected_learn_tensor (torch.Tensor): What mask we expect for the learning pass.

        - No data stored,  insufficient learning period, so no learning can run
        - Correct learning period, but insufficient data for batch
        - Sufficient data, incorrect learning period
        - Sufficient data and learn period and all forwarded
        - Sufficient data and learn period, but none forwarded
        """
        device = 'cpu'
        params = ExpertParams()
        params.flock_size = 7
        params.n_cluster_centers = 5

        s_pooler = params.spatial
        s_pooler.batch_size = 10
        s_pooler.learning_period = 5

        flock = ConvSPFlock(params, AllocatingCreator(device))

        flock.common_buffer.total_data_written.fill_(total_data_written)
        flock.common_buffer.data_since_last_sample.fill_(data_since_last_sample)
        learn_tensor = flock._determine_learning(mask)

        assert same(expected_learn_tensor, learn_tensor)

    @pytest.mark.slow
    @pytest.mark.flaky(reruns=3)
    @pytest.mark.parametrize('sampling_method',
                             [SamplingMethod.LAST_N, SamplingMethod.BALANCED, SamplingMethod.UNIFORM])
    def test_forward_learn_subflocking(self, sampling_method):
        params = ExpertParams()
        params.flock_size = 2
        params.n_cluster_centers = 4

        params.spatial.input_size = 2
        params.spatial.cluster_boost_threshold = 2
        params.spatial.learning_rate = 0.1
        params.spatial.learning_period = 1
        params.spatial.batch_size = 8
        params.spatial.buffer_size = 10
        params.spatial.max_boost_time = 10
        params.spatial.sampling_method = sampling_method

        device = 'cuda'
        float_dtype = get_float(device)

        flock = ConvSPFlock(params, AllocatingCreator(device))

        # Flock1 sequence is [0, 0], [1, 0], [0, 1], [1, 1], [0, 1]
        # Flock2 sequence is [0, 0], [1, 0], [0, 1], [1, 1], [1, 1]
        data = torch.tensor(
            [[[0., 0], [0., 0]], [[1., 0], [1., 0]], [[0., 1], [0., 1]], [[1, 1], [1, 1]], [[0, 1], [1, 1]]],
            dtype=float_dtype, device=device)

        iters = 15

        for itr in range(iters):
            for k in data:
                flock.forward_learn(k)

        expected_forward_executions = torch.tensor([[iters * 5], [iters * 4]], dtype=torch.int64, device=device)
        expected_learning_executions = torch.tensor([[iters * 5]], dtype=torch.int64, device=device) - 3
        expected_learning_executions = expected_learning_executions.expand(params.flock_size, 1)

        assert same(expected_forward_executions, flock.execution_counter_forward)
        assert same(expected_learning_executions, flock.execution_counter_learning)

        cluster_centers1 = flock.cluster_centers.to('cpu').data[0].numpy()
        cluster_centers2 = flock.cluster_centers.to('cpu').data[1].numpy()
        data = data.to('cpu').data.numpy()
        expected_cluster_centers1 = data[:4, 0]
        expected_cluster_centers2 = data[:4, 1]

        # Cluster centers have no guarantee of order - so we have to order them manually
        rounded_cluster_centers1 = np.around(cluster_centers1, decimals=2)
        rounded_cluster_centers2 = np.around(cluster_centers2, decimals=2)

        cc_indices1 = []
        cc_indices2 = []
        for cc1, cc2 in zip(rounded_cluster_centers1, rounded_cluster_centers2):
            cc_indices1.append(cc1[0] * 2 + cc1[1] * 4)
            cc_indices2.append(cc2[0] * 2 + cc2[1] * 4)

        sorted_indices1 = np.argsort(cc_indices1)
        sorted_ccs1 = cluster_centers1[sorted_indices1]

        sorted_indices2 = np.argsort(cc_indices2)
        sorted_ccs2 = cluster_centers2[sorted_indices2]

        np.testing.assert_almost_equal(expected_cluster_centers1, sorted_ccs1, decimal=1)
        np.testing.assert_almost_equal(expected_cluster_centers2, sorted_ccs2, decimal=1)


import enum
import copy
import claripy
from claripy.operations import infix, prefix

class BASIC_OP_TYPE(enum.Enum):
    MEM = 0,
    REG = 1,
    CONST = 2,
    TEMP = 3,

_g_type_dict = {
    'mem': BASIC_OP_TYPE.MEM,
    'reg': BASIC_OP_TYPE.REG,
    'BVV': BASIC_OP_TYPE.CONST,
    'tmp': BASIC_OP_TYPE.TEMP
}

def str2type(s: str):
    if s in _g_type_dict:
        return _g_type_dict[s]
    return s

class FormulaNode:

    def __init__(self, op, name, size, depth, args=None):
        self.op = op
        self.name = name
        self.size = size
        self.depth = depth
        self.args = args

    def set_args(self, args):
        self.args = args

    def to_basic_key(self):
        """
        This key can easily crash, merely used for register, memory, const
        """
        assert self.args is None
        return self.op, self.name, self.size, self.depth

    def copy(self):
        return FormulaNode(self.op,
                           self.name,
                           self.size,
                           self.depth,
                           copy.deepcopy(self.args))

    def to_deref(self, ptr_info):
        assert self.op == BASIC_OP_TYPE.MEM
        self.name = 'deref'
        self.depth = ptr_info.depth + 1
        self.args = (ptr_info,)

    def __str__(self):
        if self.op == BASIC_OP_TYPE.REG:
            return self.name
        elif self.op == BASIC_OP_TYPE.MEM:
            if self.name == 'deref':
                return '(*%s)[0,%d]' % (str(self.args[0]), self.size)
            else:
                return '(*%x)[0,%d]' % (self.name, self.size)
        elif self.op == BASIC_OP_TYPE.CONST:
            return '0x%x#%d' % (self.name, self.size)
        elif self.op in infix:
            tmp = map(lambda a: '(%s)' % str(a), self.args)
            tmp = (' %s ' % infix[self.op]).join(tmp)
            return tmp
        elif self.op in prefix:
            return '%s(%s)' % (str(self.args[0]), prefix[self.op])
        elif self.op == 'Extract':
            return '%s[%s:%s]' % (str(self.args[2]), str(self.args[0]), str(self.args[1]))
        elif self.op == 'Reverse':
            return 'Reverse%s' % str(self.args)
        elif self.op == 'ZeroExt':
            return "(0#%d .. %s)" % (self.args[0], self.args[1])
        else:
            tmp = self.op + '('
            for arg in self.args:
                tmp += str(arg) + ', '
            tmp += ')'
            return tmp
            # raise Exception('Unknow op! %s, %s' % (str(self.op), str(self.args)))

    def __repr__(self):
        return str(self)

    @staticmethod
    def get_depth_1_node(expr):
        assert expr.depth == 1, "the depth is not 1!"
        # tmp = str(expr._encoded_name, encoding='utf-8').split('_')
        if expr.symbolic:
            tmp = expr.args[0].split('_')
            op = tmp[0]
            if op == 'mem':
                # for mem, the name is address value
                name = int(tmp[1], 16)
            else:
                name = tmp[1]
            size = expr.length
        else:
            op = expr.op
            name = expr.args[0]
            size = expr.length
        op = str2type(op)
        return FormulaNode(op, name, size, 1)

    @staticmethod
    def get_formula_tree(expr):
        """
        We give up when the expression is too complex (extra-ordinary depth)
        """
        if not hasattr(expr, 'depth'):
            return expr
        if expr.depth == 1:
            return FormulaNode.get_depth_1_node(expr)
        if expr.depth > 100:
            return None
        depth = expr.depth
        op = expr.op
        # TODO: change the operation to readable format
        name = op
        args = []
        for arg in expr.args:
            args.append(FormulaNode.get_formula_tree(arg))
        if op == 'Extract':
            size = args[0] - args[1] + 1
        elif op in ['ZeroExt', 'SignExt']:
            size = args[0] + args[1].size
        else:
            size = args[0].size
        return FormulaNode(op, name, size, depth, tuple(args))

    @staticmethod
    def _rebuild(formula, ptr_map, done_map):
        if not isinstance(formula, FormulaNode):
            return formula
        if formula.depth == 1:
            if formula.to_basic_key() in done_map.keys():
                return done_map[formula.to_basic_key()].copy()
            elif formula.op == BASIC_OP_TYPE.REG:
                return formula
            if formula.op == BASIC_OP_TYPE.MEM:
                if formula.name in ptr_map.keys():
                    # this could be a multi-deref process
                    tmp = ptr_map[formula.name][0]
                    new_tmp = FormulaNode._rebuild(tmp, ptr_map, done_map)
                    new_formula = formula.copy()
                    new_formula.to_deref(new_tmp)
                    done_map[formula.to_basic_key()] = new_formula
                    return new_formula
                else:
                    # TODO: we may fail to handle it, or it simply loads data from a exact memory block
                    return formula
            else:
                return formula
        else:
            args = []
            depth = formula.depth
            for arg in formula.args:
                tmp = FormulaNode._rebuild(arg, ptr_map, done_map)
                if isinstance(tmp, FormulaNode):
                    depth = max(tmp.depth + 1, depth)
                args.append(tmp)
            return FormulaNode(formula.op, formula.name, formula.size, depth, tuple(args))

    @staticmethod
    def ptr_constraints_to_ptr_map(ptr_constraints):
        """
        return the dictionary with items (ptr_addr, (being_concretized_symbol, const_node))
        """
        ptr_map = dict()
        for c in ptr_constraints:
            symbol = FormulaNode.get_formula_tree(c.args[0])
            value = FormulaNode.get_depth_1_node(c.args[1])
            ptr_map[value.name] = (symbol, value)
        return ptr_map

    @staticmethod
    def rebuild_formula_without_concretization(formula, ptr_map, done_map):
        """
        The root node
        see FormulaNode.ptr_constraints_to_ptr_map
        the transfer map (a block can have multiple formulas, reuse it)
        """
        if formula is None:
            return None
        # this should be a recursive process
        return FormulaNode._rebuild(formula, ptr_map, done_map)


import argparse
import os
import parser
from functools import partial

import dataset
import evaluate
import longrange
import lovely_tensors as lt
import matplotlib.pyplot as plt
import numpy as np
import torch
import util
import wandb
from tqdm import tqdm
from tqdm.auto import tqdm

lt.monkey_patch()

def train_transformer(ds, net, n_batches=10, batch_size=32, seq_len=100, lr=1e-3, device='cpu', wandb=None, tqdm=None):
    net.to(device).train()
    opt = torch.optim.Adam(net.parameters(), lr=lr)

    pbar = dataset.get_seq_batches(ds, n_batches, batch_size, n_seqs=1, seq_len=seq_len, min_dist=0, max_dist=1, unbind=True)
    pbar = pbar if tqdm is None else tqdm(pbar, total=n_batches)
    for (batch1,), (batch1fchar,) in pbar:
        batch1 = batch1.to(device).long()

        loss = longrange.loss_fn(net, batch1)
        opt.zero_grad()
        loss.backward()
        opt.step()

        grad_main = torch.cat([p.grad.flatten() for p in net.blocks_main.parameters()]).norm().item()

        data = dict(loss=loss.item(), grad_main=grad_main)
        if tqdm is not None:
            pbar.set_postfix({k: v for k, v in data.items() if isinstance(v, float)})
        if wandb is not None:
            wandb.log(data)

def train_longrange(net, ds, n_batches, batch_size,
                    n_seqs, seq_len, min_dist, max_dist,
                    lr=1e-3, device='cpu', wandb=None, tqdm=None):
    net.to(device).train()
    opt = torch.optim.Adam(net.parameters(), lr=lr)
    # scheduler = torch.optim.MultiStepLR(opt, milestones=[1000], gamma=0.1)

    pbar = dataset.get_seq_batches(ds, n_batches, batch_size, n_seqs, seq_len,
                                   min_dist, max_dist, unbind=False, device=device)
    pbar = pbar if tqdm is None else tqdm(pbar, total=n_batches)
    for ids, fbins in pbar:
        # ids: (bs, n_seqs, seq_len), fbins: (bs, n_seqs, seq_len)
        losses = longrange.loss_fn_longrange(net, ids) #: (bs, n_seqs, seq_len-1)
        loss = losses.mean()

        opt.zero_grad()
        loss.backward()
        opt.step()

        grad_main = torch.cat([p.grad.flatten() for p in net.blocks_main.parameters()]).norm()

        data = dict(ppl=loss.exp().item())
        data.update({f'ppl_seq_{seq_i}': losses_seq_i.mean().exp().item()
                     for seq_i, losses_seq_i in enumerate(losses.unbind(dim=-2))})
        data.update(grad_main=grad_main.item())
        if net.config['use_memory']:
            grad_mc = torch.cat([p.grad.flatten() for p in net.blocks_memory_creator.parameters()]).norm()
            grad_mr = torch.cat([p.grad.flatten() for p in net.blocks_memory_retriever.parameters()]).norm()
            data.update(grad_mc=grad_mc.item(), grad_mr=grad_mr.item())

        if tqdm is not None:
            pbar.set_postfix({k: v for k, v in data.items() if isinstance(v, float)})
        if wandb is not None:
            wandb.log(data)

# def log_network_stats(akl, net):
#     grad_mag = [block.attn.out_proj.weight.grad.norm().item() for block in net.blocks_main]
#     akl.put(grad_blocks_main=grad_mag)
#     grad_mag = [block.attn.out_proj.weight.grad.norm().item() for block in net.blocks_memory_retriever]
#     akl.put(grad_blocks_retriever=grad_mag)
#     grad_mag = [block.attn.out_proj.weight.grad.norm().item() for block in net.blocks_memory_creator]
#     akl.put(grad_blocks_creator=grad_mag)
#     if akl.ts%20==0:
#         for name, module in net.named_modules():
#             if isinstance(module, longrange.Block):
#                 key = f'attn_mat_{name}'
#                 akl.put(**{key: module.attn_weights.detach().cpu().numpy()})

# def print_network_gradient_stats(net):
#     print('Main blocks gradient mag:')
#     for block in net.blocks_main:
#         gradmag = block.attn.out_proj.weight.grad.norm().item()
#         print(f'{gradmag: .3e}', end=' ')
#     print()
#     print('Memory Retrieval blocks gradient mag:')
#     for block in net.blocks_memory_retriever:
#         gradmag = block.attn.out_proj.weight.grad.norm().item()
#         print(f'{gradmag: .3e}', end=' ')
#     print()
#     print('Memory Creator blocks gradient mag:')
#     for block in net.blocks_memory_creator:
#         gradmag = block.attn.out_proj.weight.grad.norm().item()
#         print(f'{gradmag: .3e}', end=' ')
#     print()

def main(args):
    print(f'Starting training with args: {args}')
    print('Loading dataset...')
    ds_train, ds_test = dataset.load_dataset(tqdm=tqdm)
    print('Done!')
    np.random.seed(args.seed); torch.manual_seed(args.seed)
    config_net = longrange.get_config(args.model, max_context_length=args.seq_len)
    net = longrange.LongRangeGPT(**config_net)
    print(f'Model: {args.model}')
    print(f'Model config: {config_net}')
    print(f'Model # of parameters: {util.count_params(net)}')

    config = vars(args)
    config['config_net'] = config_net

    if args.track:
        run = wandb.init(config=config, name=args.name, save_code=True)

    n_seqs_train = 2 if net.config['use_memory'] else 1
    n_seqs_test = 4 if net.config['use_memory'] else 1
    train_longrange(net=net, ds=ds_train, n_batches=args.n_batches_train, batch_size=args.batch_size,
                    n_seqs=n_seqs_train, seq_len=args.seq_len, min_dist=args.min_dist, max_dist=args.max_dist,
                    lr=args.lr, device=args.device, wandb=wandb if args.track else None, tqdm=tqdm)
    loss_mean, loss_count = evaluate.evaluate_longrange(net=net, ds=ds_test, n_batches=args.n_batches_test, batch_size=args.batch_size,
                                                        n_seqs=n_seqs_test, seq_len=args.seq_len, min_dist=0, max_dist=1,
                                                        device=args.device, wandb=wandb if args.track else None, tqdm=tqdm)

    if args.track:
        for i_seq in range(len(loss_mean)):
            fig = evaluate.viz_losses(loss_mean[i_seq], loss_count[i_seq])
            plt.suptitle(f'Losses for sequence {i_seq}')
            wandb.log({f'viz loss': fig})

        net.eval().cpu()

        # make the log directory recursively
        os.makedirs(f'./results/{run.name}', exist_ok=True)
        torch.save(net, f'./results/{run.name}/model.pt')
        torch.save((loss_mean, loss_count), f'./results/{run.name}/losses.pt')

        run.finish()
        plt.close('all')

parser = argparse.ArgumentParser(description='Train a model.')

parser.add_argument('--model', type=str, default='longrange')
parser.add_argument('--n_batches_train', type=int, default=5000)
parser.add_argument('--n_batches_test', type=int, default=5000)
parser.add_argument('--batch_size', type=int, default=64)
parser.add_argument('--seq_len', type=int, default=8)
parser.add_argument('--min_dist', type=int, default=0)
parser.add_argument('--max_dist', type=int, default=1)
parser.add_argument('--n_latent_tokens', type=int, default=None)

parser.add_argument('--lr', type=float, default=1e-3)

parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--device', type=str, default='cpu')
parser.add_argument('--track', action=argparse.BooleanOptionalAction, default=False)
parser.add_argument('--name', type=str, default=None)

if __name__ == '__main__':
    args = parser.parse_args()
    main(args)

"""
# Test

python train.py --model transformer --n_batches_train 0100 --n_batches_test 0100 --batch_size 64 --seq_len 8 --seed 0 --lr 1e-3 --device cuda:0 --track --name testing
python train.py --model transformer --batch_size 64 --seq_len 8 --seed 0 --lr 1e-3 --device cuda:0 --track --name testing

# Learning Rate Sweep
python train.py --model transformer --batch_size 64 --seq_len 8 --seed 0 --lr 3e-3 --device cuda:0 --track --name lr_3e-3 &
python train.py --model transformer --batch_size 64 --seq_len 8 --seed 0 --lr 1e-3 --device cuda:0 --track --name lr_1e-3 &
python train.py --model transformer --batch_size 64 --seq_len 8 --seed 0 --lr 6e-4 --device cuda:1 --track --name lr_6e-4 &
python train.py --model transformer --batch_size 64 --seq_len 8 --seed 0 --lr 3e-4 --device cuda:1 --track --name lr_3e-4 &
python train.py --model transformer --batch_size 64 --seq_len 8 --seed 0 --lr 1e-4 --device cuda:2 --track --name lr_1e-4 &
python train.py --model transformer --batch_size 64 --seq_len 8 --seed 0 --lr 6e-5 --device cuda:2 --track --name lr_6e-5 &
python train.py --model transformer --batch_size 64 --seq_len 8 --seed 0 --lr 3e-5 --device cuda:3 --track --name lr_3e-5 &
python train.py --model transformer --batch_size 64 --seq_len 8 --seed 0 --lr 1e-5 --device cuda:3 --track --name lr_1e-5 &
wait

# Transformer Model Size Sweep
python train.py --model  transformer_small --batch_size 64 --seq_len 8 --seed 0 --lr 3e-4 --device cuda:0 --track --name transformer__small &
python train.py --model transformer_medium --batch_size 64 --seq_len 8 --seed 0 --lr 3e-4 --device cuda:1 --track --name transformer_medium &
python train.py --model  transformer_large --batch_size 64 --seq_len 8 --seed 0 --lr 3e-4 --device cuda:2 --track --name transformer__large &
python train.py --model transformer_xlarge --batch_size 64 --seq_len 8 --seed 0 --lr 3e-4 --device cuda:3 --track --name transformer_xlarge &
wait

# Longrange Model Size Sweep
python train.py --model  longrange_small --batch_size 64 --seq_len 8 --seed 0 --lr 3e-4 --device cuda:0 --track --name longrange__small &
python train.py --model longrange_medium --batch_size 64 --seq_len 8 --seed 0 --lr 3e-4 --device cuda:1 --track --name longrange_medium &
python train.py --model  longrange_large --batch_size 64 --seq_len 8 --seed 0 --lr 3e-4 --device cuda:2 --track --name longrange__large &
python train.py --model longrange_xlarge --batch_size 64 --seq_len 8 --seed 0 --lr 3e-4 --device cuda:3 --track --name longrange_xlarge &
wait

# Transformer Seq Length Sweep
python train.py --model transformer --batch_size 64 --seq_len 002 --seed 0 --lr 3e-4 --device cuda:0 --track --name transformer_002 &
python train.py --model transformer --batch_size 64 --seq_len 004 --seed 0 --lr 3e-4 --device cuda:1 --track --name transformer_004 &
python train.py --model transformer --batch_size 64 --seq_len 008 --seed 0 --lr 3e-4 --device cuda:2 --track --name transformer_008 &
python train.py --model transformer --batch_size 64 --seq_len 016 --seed 0 --lr 3e-4 --device cuda:3 --track --name transformer_016 &
python train.py --model transformer --batch_size 64 --seq_len 032 --seed 0 --lr 3e-4 --device cuda:4 --track --name transformer_032 &
python train.py --model transformer --batch_size 64 --seq_len 064 --seed 0 --lr 3e-4 --device cuda:0 --track --name transformer_064 &
wait

# Longrange Seq Length Sweep
python train.py --model longrange --batch_size 64 --seq_len 002 --seed 0 --lr 3e-4 --device cuda:0 --track --name longrange_002 &
python train.py --model longrange --batch_size 64 --seq_len 004 --seed 0 --lr 3e-4 --device cuda:1 --track --name longrange_004 &
python train.py --model longrange --batch_size 64 --seq_len 008 --seed 0 --lr 3e-4 --device cuda:2 --track --name longrange_008 &
python train.py --model longrange --batch_size 64 --seq_len 016 --seed 0 --lr 3e-4 --device cuda:3 --track --name longrange_016 &
python train.py --model longrange --batch_size 64 --seq_len 032 --seed 0 --lr 3e-4 --device cuda:4 --track --name longrange_032 &
python train.py --model longrange --batch_size 64 --seq_len 064 --seed 0 --lr 3e-4 --device cuda:0 --track --name longrange_064 &
wait

python train.py --device cuda:0 --model transformer --seq_len 64 --seed 0

python train.py --device cuda:1 --model perceiver --seq_len 128 --seed 0
python train.py --device cuda:2 --model perceiver --seq_len 256 --seed 0
python train.py --device cuda:0 --model perceiver --seq_len 512 --seed 0

python train.py --device cuda:3 --model longrange1 --seq_len 64 --batch_size 64 --seed 0

python main.py --device cuda:0 --model transformer --seq_len 002 --track --name transformer-002
python main.py --device cuda:0 --model transformer --seq_len 004 --track --name transformer-004
python main.py --device cuda:0 --model transformer --seq_len 008 --track --name transformer-008
python main.py --device cuda:0 --model transformer --seq_len 016 --track --name transformer-016

python main.py --device cuda:1 --model transformer --seq_len 032 --track --name transformer-032
python main.py --device cuda:1 --model transformer --seq_len 064 --track --name transformer-064

python main.py --device cuda:2 --model  longrange1 --seq_len 002 --track --name  longrange1-002
python main.py --device cuda:2 --model  longrange1 --seq_len 004 --track --name  longrange1-004
python main.py --device cuda:2 --model  longrange1 --seq_len 008 --track --name  longrange1-008
python main.py --device cuda:2 --model  longrange1 --seq_len 016 --track --name  longrange1-016

python main.py --device cuda:3 --model  longrange1 --seq_len 032 --track --name  longrange1-032
python main.py --device cuda:3 --model  longrange1 --seq_len 064 --track --name  longrange1-064

"""