"""
Test the top-level ansible module
"""

# Standard
from contextlib import contextmanager
from unittest import mock
import functools
import os
import tempfile

# Third Party
from openshift.dynamic import DynamicClient, EagerDiscoverer
import pytest

# First Party
import alog

# Local
from oper8 import Controller, constants
from oper8.log_format import Oper8JsonFormatter
from oper8.test_helpers.helpers import (
    DummyController,
    ModuleExit,
    TestRecorder,
    maybe_temp_dir,
    mock_config_file,
    mock_sys_exit,
)
from oper8.test_helpers.kub_mock import mock_kub_client_constructor
from oper8.utils import merge_configs

## Helpers #####################################################################

log = alog.use_channel("TEST")

DEFAULT_CONTROLLER_IMPORT = "test_k8s_application.TestController"

class AppTestRecorder(TestRecorder):
    def __init__(self, mock_client=None, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.mock_client = (
            DynamicClient(mock_client, discoverer=EagerDiscoverer)
            if mock_client
            else None
        )

@contextmanager
def mock_app_calls(recorder, module_params, patch_controller_class):
    if not patch_controller_class:
        yield
    else:
        import_name = module_params.get("controller_class", DEFAULT_CONTROLLER_IMPORT)

        # Disable reimporting the controller so mocks work
        with mock.patch(
            "oper8.reconcile.ReconcileManager.__init__.__defaults__",
            (None, None, None, False),
        ):

            with mock.patch(
                ".".join([import_name, "log_constructed"]),
                lambda _, config_defaults: recorder.add_event(
                    recorder.CONSTRUCTED, config_defaults
                ),
            ):
                with mock.patch(
                    ".".join([import_name, "setup_components"]),
                    lambda _, deploy_config: recorder.add_event(
                        recorder.SETUP_COMPONENTS, deploy_config
                    ),
                ):
                    with mock.patch(
                        ".".join([import_name, "finalize_components"]),
                        lambda _, deploy_config: recorder.add_event(
                            recorder.FINALIZE_COMPONENTS, deploy_config
                        ),
                    ):
                        log.debug("Yielding from patched app calls")
                        yield

DEFAULT_VERSION = "v1.2.3.test"

def make_cr(overrides=None):
    overrides = overrides or {}
    return merge_configs(
        {
            "apiVersion": "test.foo.bar.com/v1",
            "kind": "Foo",
            "metadata": {
                "name": "foo",
                "namespace": "bar",
                "uid": "12345678",
            },
            "spec": {"version": DEFAULT_VERSION},
        },
        overrides,
    )

class TestController(Controller):
    """We aren't testing the controller stack, so this is a minimal stub"""

    group = "test.com"
    version = "v2"
    kind = "Test"

    # This is not a test class!
    __test__ = False

    def __init__(self, config_defaults=None, **kwargs):
        super().__init__(config_defaults=config_defaults, **kwargs)
        self.log_constructed(config_defaults)

    def log_constructed(self, config_defaults):
        pass

    def setup_components(self, session):
        pass

    def finalize_components(self, session):
        pass

class BrokenTestController(TestController):
    """Test controller that will fail during rollout"""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def reconcile(self, *args, **kwargs):
        raise RuntimeError("Oh no!")

    def log_constructed(self, config_defaults):
        pass

    def setup_components(self, session):
        pass

    def finalize_components(self, session):
        pass

class TestControllerWithResources(DummyController):
    """Dummy controller that deploys some api resources"""

    # This is not a test class!
    __test__ = False

    def log_constructed(self, config_defaults):
        pass

    def setup_components(self, session):
        pass

    def finalize_components(self, session):
        pass

    def __init__(self, **kwargs):
        super().__init__(
            components=[
                {
                    "name": "foo-comp",
                    "api_objects": [
                        ("foo", {"kind": "Foo", "metadata": {"namespace": "bar"}})
                    ],
                }
            ],
            **kwargs,
        )

    def setup_components(self, session):
        log.debug("TestControllerWithResources.setup_components")
        super().setup_components(session)

def mock_alog_configure(rec, *args, **kwargs):
    rec.add_event("alog_configure", (args, kwargs))

def run_module(
    module_params=None,
    add_module_param_defaults=True,
    cr_overrides=None,
    config_defaults=None,
    recorder=None,
    cluster_state=None,
    mock_logging=True,
    patch_controller_class=True,
):
    # Local
    from oper8.watch_manager.ansible_watch_manager.modules.k8s_application import main

    with maybe_temp_dir() as temp_dir:
        log.info("Temp Dir: %s", temp_dir)

        cr = make_cr(cr_overrides)

        # Add the CR to the default cluster state
        cluster_state = cluster_state or {}
        cluster_state.setdefault(cr["metadata"].get("namespace"), {}).setdefault(
            cr["kind"], {}
        ).setdefault(cr["apiVersion"], {}).setdefault(cr["metadata"].get("name"), cr)

        # If this is a dry run, go with it, otherwise mock the k8s api
        with mock_kub_client_constructor(cluster_state=cluster_state) as mock_client:

            # Mock sys.exit and keep track of the exit code
            log.info("Mocking sys.exit")
            recorder = recorder or AppTestRecorder(mock_client)
            with mock_sys_exit(recorder):

                # Add the defaults to send output to the temp dir
                module_params = module_params or {}
                log.info("Module Params: %s", module_params)
                if add_module_param_defaults:
                    module_params["working_dir"] = os.path.join(temp_dir, "working")
                    module_params.setdefault(
                        "controller_class",
                        DEFAULT_CONTROLLER_IMPORT,
                    )
                    module_params.setdefault("version", DEFAULT_VERSION)
                    module_params.setdefault("full_cr", str(cr))

                # Wire the recorder into the test app
                log.info("Mocking app calls")
                with mock_app_calls(recorder, module_params, patch_controller_class):

                    # Create the config file for the module
                    config_object = {"ANSIBLE_MODULE_ARGS": module_params}

                    # Inject the config file as sys.argv[1]
                    log.info("Mocking the config file input")
                    with mock_config_file(config_object):

                        # run it!
                        try:
                            log.info("------------------")
                            if mock_logging:
                                with mock.patch(
                                    "alog.configure",
                                    functools.partial(mock_alog_configure, recorder),
                                ):
                                    main()
                            else:
                                main()
                        except ModuleExit:
                            pass

                        # Merge events from controller
                        return recorder

## Happy Path ##################################################################

@pytest.mark.ansible
def test_run_module_reconcile():
    """Test that a straightforward execution of the module works as expected"""
    recorder = run_module()
    assert recorder.retcode == 0
    assert recorder.events == [
        "alog_configure",
        AppTestRecorder.CONSTRUCTED,
        AppTestRecorder.SETUP_COMPONENTS,
        AppTestRecorder.EXITED,
    ]

@pytest.mark.ansible
def test_run_module_finalize():
    """Test that a straightforward execution of the module as a finalizer works
    as expected
    """
    recorder = run_module(module_params={"operation": "remove"})
    assert recorder.retcode == 0
    assert recorder.events == [
        "alog_configure",
        AppTestRecorder.CONSTRUCTED,
        AppTestRecorder.FINALIZE_COMPONENTS,
        AppTestRecorder.EXITED,
    ]

@pytest.mark.ansible
def test_paused():
    """Test if the paused annotation is present, the controller is not executed"""
    recorder = run_module(
        cr_overrides={
            "metadata": {"annotations": {constants.PAUSE_ANNOTATION_NAME: "true"}}
        }
    )
    assert recorder.retcode == 0
    assert recorder.events == ["alog_configure", AppTestRecorder.EXITED]

@pytest.mark.ansible
def test_log_config():
    recorder = AppTestRecorder()
    annotations = {
        constants.LOG_DEFAULT_LEVEL_NAME: "debug1",
        constants.LOG_FILTERS_NAME: "FOO:debug4",
        constants.LOG_THREAD_ID_NAME: "true",
        constants.LOG_JSON_NAME: "true",
    }

    with mock.patch("alog.configure", functools.partial(mock_alog_configure, recorder)):
        recorder = run_module(
            recorder=recorder, cr_overrides={"metadata": {"annotations": annotations}}
        )
        assert recorder.retcode == 0
        assert "alog_configure" in recorder.events
        config_args, config_kwargs = recorder.event_details["alog_configure"]
        assert not config_args
        assert (
            config_kwargs["default_level"]
            == annotations[constants.LOG_DEFAULT_LEVEL_NAME]
        )
        assert config_kwargs["filters"] == annotations[constants.LOG_FILTERS_NAME]
        assert config_kwargs["thread_id"] == True
        assert isinstance(config_kwargs["formatter"], Oper8JsonFormatter)

@pytest.mark.ansible
def test_log_file():
    """Make sure that setting a log_file results in a callable handler_generator"""
    recorder = AppTestRecorder()
    with mock.patch("alog.configure", functools.partial(mock_alog_configure, recorder)):
        with tempfile.TemporaryDirectory() as temp_log_dir:
            recorder = run_module(
                recorder=recorder,
                module_params={"log_file": os.path.join(temp_log_dir, "foo.bar.log")},
            )
            assert recorder.retcode == 0
            assert "alog_configure" in recorder.events
            config_args, config_kwargs = recorder.event_details["alog_configure"]
            assert not config_args
            assert config_kwargs["handler_generator"] is not None
            assert callable(config_kwargs["handler_generator"])

## Error Cases #################################################################

@pytest.mark.ansible
def test_required_args():
    """Test missing required arguments are caught and cause a non-zero exit"""
    recorder = run_module(
        add_module_param_defaults=False, module_params={"full_cr": "{}"}
    )
    assert recorder.retcode != 0

@pytest.mark.ansible
def test_unparsable_full_cr():
    """Test an un-parsable value for a full_cr exits with an error"""
    recorder = run_module(module_params={"full_cr": "not a dict"})
    assert recorder.retcode != 0

@pytest.mark.ansible
def test_incomplete_full_cr():
    """Test a CR which doesn't have the kubernetes basics"""
    recorder = run_module(module_params={"full_cr": "{}"})
    assert recorder.retcode == 0

@pytest.mark.ansible
def test_bad_controller_module():
    """Test that a bad module in controller_class causes an error exit"""
    recorder = run_module(
        module_params={"controller_class": "foo.bar.Baz"},
        patch_controller_class=False,
    )
    assert recorder.retcode == 0

@pytest.mark.ansible
def test_missing_controller_class():
    """Test that a class name that is missing from the module causes an error
    exit
    """
    recorder = run_module(
        module_params={"controller_class": "test_k8s_controller.NotThere"},
        patch_controller_class=False,
    )
    assert recorder.retcode == 0

@pytest.mark.ansible
def test_bad_controller_class():
    """Test that a class name that is not a class causes an error exit"""
    recorder = run_module(
        module_params={
            "controller_class": "test_k8s_controller.test_bad_controller_class",
        },
        patch_controller_class=False,
    )
    assert recorder.retcode == 0

@pytest.mark.ansible
def test_controller_exception():
    """Test that an exception in the Controller causes an error exit"""
    recorder = run_module(
        module_params={"controller_class": "test_k8s_application.BrokenTestController"}
    )
    assert recorder.retcode == 0

@pytest.mark.ansible
def test_owner_references():
    """Test that running the module creates resources with owner references"""
    recorder = run_module(
        cluster_state={"bar": {"Foo": {"v1": {}}}},
        module_params={
            "controller_class": "test_k8s_application.TestControllerWithResources"
        },
        # We don't patch the controller class here because we want the kube mock
        # to be able to fetch the deployed resources and they don't get deployed
        # if the patches take effect
        patch_controller_class=False,
    )
    assert recorder.retcode == 0

    # Fetch the content of the deployed component
    foos = recorder.mock_client.resources.get(kind="Foo", api_version="v1")
    resource = foos.get(name="foo", namespace="bar").to_dict()
    assert resource
    log.debug(resource)
    assert len(resource["metadata"]["ownerReferences"]) == 1


import random
import math

print('---------------------------------1----------------------------------------')
# Задайте список из нескольких чисел. Напишите программу, которая найдёт сумму элементов списка, стоящих на нечётной позиции.
# [2, 3, 5, 9, 3] -> на нечётных позициях элементы 3 и 9, ответ: 12

list = [2, 3, 5, 9, 3]

def sumOfOdd(value: list) -> int:
    tempInt = 0
    if (len(value) >= 2):
        for i in range(1, len(value), 2):
            print(f'i={i} элемент {value[i]}')
            tempInt += value[i]
    return tempInt

print(f'для массива {list} ответ {sumOfOdd(list)}')

print('---------------------------------2----------------------------------------')
# Напишите программу, которая найдёт произведение пар чисел списка. Парой считаем первый и последний элемент, второй и предпоследний и т.д.
# [2, 3, 4, 5, 6] => [12, 15, 16];
# [2, 3, 5, 6] => [12, 15]
list = [2, 3, 4, 5, 6]
#list = [2, 3,  5, 6];

def sumPair(value: list) -> list:
    tempList = []
    if (len(value) >= 2):
        for i in range(0, int(len(value)/2+0.5), 1):
            tempList.append(value[i]*value[len(value)-1-i])
    return tempList

print(f'для массива {list} ответ {sumPair(list)}')

print('---------------------------------3----------------------------------------')
# Задайте список из вещественных чисел. Напишите программу, которая найдёт разницу между максимальным и минимальным значением дробной части элементов.
# [1.1, 1.2, 3.1, 5, 10.01] => 0.19

list = [1.1, 1.2, 3.1, 5, 10.01]

def subtractMaxMin(value: list) -> int:
    tempMin = 0
    tempMax = 0
    for i in range(0, len(value), 1):
        # если убрать окгругление отсет будет 0.199999999
        tempNum = round(value[i] - int(value[i]), 2)
        if (tempMin > tempNum):
            tempMin = tempNum
        if (tempMax < tempNum):
            tempMax = tempNum
            print(f'----tempMax = {tempMax}')
    print(f'tempMin = {tempMin}')
    print(f'tempMax = {tempMax}')
    return tempMax - tempMin

print(f'для массива {list} ответ {subtractMaxMin(list)}')

print('---------------------------------4----------------------------------------')
# Напишите программу, которая будет преобразовывать десятичное число в двоичное.
# 45 -> 101101
# 3 -> 11
# # 2 -> 10

num = 45

def toBinary(value: int) -> str:
    tempStr = ''
    while value >= 1:
        print(f'{value}  {value//2}  {value%2}')
        tempStr = str(value % 2) + tempStr
        value //= 2
    return tempStr

print(f'чисто {num} в двоичном {toBinary(num)}')

print('---------------------------------5----------------------------------------')
# Задайте число. Составьте список чисел Фибоначчи, в том числе для отрицательных индексов.
# для k = 8 список будет выглядеть так: [-21 ,13, -8, 5, −3, 2, −1, 1, 0, 1, 1, 2, 3, 5, 8, 13, 21] [Негафибоначчи]

num = 8

# без рекурсии тут вызовов можно было бы сделать меньше. То так как это у нас часть курса оставл.
def fibonacci(value: int) -> int:
    if value == 0:
        return 0;
    elif value == 1:
        return 1
    else:
        return fibonacci(value-1)+fibonacci(value-2)

tempList = []
for i in range(-num, num+1, 1):
    if (i < 0):
        tempList.append(fibonacci(-i)*(-1**(-i)))
    else:
        tempList.append(fibonacci(i))

print(f'для числа  {num} ответ {tempList}')


import unittest
import numpy as np
from op_test import OpTest

class TestConv2dOp(OpTest):
    def setUp(self):
        self.init_groups()
        self.op_type = "conv2d"
        batch_size = 2
        input_channels = 3
        input_height = 5
        input_width = 5
        output_channels = 6
        filter_height = 3
        filter_width = 3
        stride = 1
        padding = 0
        output_height = (input_height - filter_height + 2 * padding
                         ) / stride + 1
        output_width = (input_width - filter_width + 2 * padding) / stride + 1
        input = np.random.random((batch_size, input_channels, input_height,
                                  input_width)).astype("float32")

        filter = np.random.random(
            (output_channels, input_channels / self.groups, filter_height,
             filter_width)).astype("float32")
        output = np.ndarray(
            (batch_size, output_channels, output_height, output_width))

        self.inputs = {'Input': input, 'Filter': filter}
        self.attrs = {
            'strides': [1, 1],
            'paddings': [0, 0],
            'groups': self.groups
        }

        output_group_channels = output_channels / self.groups
        input_group_channels = input_channels / self.groups
        for batchid in xrange(batch_size):
            for group in xrange(self.groups):
                for outchannelid in range(group * output_group_channels,
                                          (group + 1) * output_group_channels):
                    for rowid in xrange(output_height):
                        for colid in xrange(output_width):
                            start_h = (rowid * stride) - padding
                            start_w = (colid * stride) - padding
                            output_value = 0.0
                            for inchannelid in range(
                                    group * input_group_channels,
                                (group + 1) * input_group_channels):
                                for frowid in xrange(filter_height):
                                    for fcolid in xrange(filter_width):
                                        input_value = 0.0
                                        inrowid = start_h + frowid
                                        incolid = start_w + fcolid
                                        if ((inrowid >= 0 and
                                             inrowid < input_height) and
                                            (incolid >= 0 and
                                             incolid < input_width)):
                                            input_value = input[batchid][
                                                inchannelid][inrowid][incolid]
                                        filter_value = filter[outchannelid][
                                            inchannelid % input_group_channels][
                                                frowid][fcolid]
                                        output_value += input_value * filter_value
                            output[batchid][outchannelid][rowid][
                                colid] = output_value

        self.outputs = {'Output': output}

    def test_check_output(self):
        self.check_output()

    def test_check_grad(self):
        self.check_grad(
            set(['Input', 'Filter']), 'Output', max_relative_error=0.05)

    def test_check_grad_no_filter(self):
        self.check_grad(
            ['Input'],
            'Output',
            max_relative_error=0.05,
            no_grad_set=set(['Filter']))

    def test_check_grad_no_input(self):
        self.check_grad(
            ['Filter'],
            'Output',
            max_relative_error=0.05,
            no_grad_set=set(['Input']))

    def init_groups(self):
        self.groups = 1

class TestWithGroup(TestConv2dOp):
    def init_groups(self):
        self.groups = 3

if __name__ == '__main__':
    unittest.main()


# First, we check if these are integer number?
try:
    number = int(input())
# Secondly, we check if the number is parity
    if number % 2:
        parity_indication = 'odd'
    else:
        # Zero is even number
        parity_indication = 'even'
# Thirdly, we check the sign of positivity
    if number > 0:
        positivity_indication = 'positive'
    elif number < 0:
        positivity_indication = 'negative'
    else:
        positivity_indication = 'zero'
    print(number, positivity_indication, parity_indication, 'number')
# Yes, yes, if not numbers then an exception
except:
    print('Dear user, you have entered invalid data!')


#!python
import argparse
from eth_utils import keccak, to_bytes, to_hex

def to_bytes32(value: int) -> bytes:
    v = to_bytes(value).rjust(32, b'\x00')
    assert len(v) == 32, \
            "{} must be less than 2**256".format(value)
    return v

def create_token_id(randomizer, serial) -> hex:
    return to_hex(keccak(to_bytes32(serial) + to_bytes32(randomizer)))

if __name__ == '__main__':
    ap = argparse.ArgumentParser("Generate Token Hash for Silencer")
    ap.add_argument("randomizer", type=int, help="Firearm Serial Number")
    ap.add_argument("serial", type=int, help="Firearm Token Randomizer")
    args = ap.parse_args()
    print(create_token_id(args.randomizer, args.serial))


# -*- coding: utf-8 -*-
import numpy as np

from arpym.statistics.simulate_normal import simulate_normal

def simulate_bm(x_tnow, delta_t_m, mu, sig2, j_):
    """For details, see here.

    Parameters
    ----------
        x_t_now : array, shape(n_, )
        delta_t_m : array, shape(m_, )
        mu : array, shape(n_, )
        sig2 : array, shape(n_,n_)
        j_ : int

    Returns
    -------
        x_tnow_thor : array, shape(j_, m_ + 1, n_)

    """
    x_tnow, delta_t_m, mu, sig2 = \
        np.atleast_1d(x_tnow), np.atleast_1d(delta_t_m), np.atleast_1d(mu), np.atleast_2d(sig2)

    n_ = x_tnow.shape[0]
    m_ = delta_t_m.shape[0]

    x_tnow_thor = np.zeros((j_, m_ + 1, n_))

    # Step 1: Compute invariants

    x_tnow_thor[:, 0, :] = x_tnow

    for m in range(m_):
        epsi_delta_t_m = simulate_normal(mu * delta_t_m[m], sig2 * delta_t_m[m], j_)

        # Step 2: Compute projected path

        x_tnow_thor[:, m + 1, :] = x_tnow_thor[:, m, :] + epsi_delta_t_m.reshape(j_, n_)

    return x_tnow_thor


import torch
import logging
import tqdm
import time
import math
from anagen.dataset import collate
from anagen.utils import batch_to_device

from torch.utils.data import DataLoader, Dataset, SequentialSampler, RandomSampler

DEFAULT_TRAIN_BATCH_SIZE=1
logger = logging.getLogger(__name__)

def parse_train_args(parser):
    # data input
    parser.add_argument("--train_input_file", type=str)
    parser.add_argument("--eval_input_file", type=str)
    parser.add_argument("--train_batch_size", type=int, default=16)
    parser.add_argument("--eval_batch_size", type=int, default=16)
    parser.add_argument("--max_span_width", type=int, default=10)
    parser.add_argument("--max_num_ctxs_in_batch", type=int, default=8)
    parser.add_argument("--max_segment_len", type=int, default=256)
    parser.add_argument("--shuffle_examples", action="store_true")
    parser.add_argument("--data_augment", type=str, choices=[None, "null_from_l0"])
    parser.add_argument("--data_augment_max_span_width", type=int, default=10)
    parser.add_argument("--train_data_augment_file", type=str)
    parser.add_argument("--eval_data_augment_file", type=str)

    # trained model save/load
    parser.add_argument("--model_load_path", type=str)
    parser.add_argument("--model_save_path", type=str)

    # gpt2 model settings
    parser.add_argument("--gpt2_model_dir", type=str, default=None)

    # training settings
    parser.add_argument("--gpu", action="store_true")
    parser.add_argument("--random_seed", type=int, default=39393)
    parser.add_argument("--unfreeze_gpt2", action="store_true")
    parser.add_argument("--learning_rate", type=float, default=0.001)
    parser.add_argument("--train_epochs", type=int, default=1)
    parser.add_argument("--log_steps", type=int, default=100)
    parser.add_argument("--eval_and_save_by_epoch", type=float)
    parser.add_argument("--save_optimizer_state", action="store_true")
    parser.add_argument("--save_latest_state", action="store_true")

    # model settings
    parser.add_argument("--gpt2_hidden_size", type=int, default=768)
    parser.add_argument("--sum_start_end_emb", action="store_true")
    parser.add_argument("--use_speaker_info", action="store_true")
    parser.add_argument("--use_distance_info", action="store_true")
    parser.add_argument("--distance_groups", action="store_true", default=32)
    parser.add_argument("--metadata_emb_size", type=int, default=20)

    parser.add_argument("--param_init_stdev", type=float, default=0.1)
    parser.add_argument("--rnn_num_layers", type=int, default=1)

    return parser.parse_args()

# based on transformers/run_lm_finetuning
def train(args, model, train_dataset, eval_dataset):
    device = torch.device("cuda" if args.gpu else "cpu")
    train_sampler = RandomSampler(train_dataset)
    train_dataloader = DataLoader(train_dataset, sampler=train_sampler,
                                  batch_size=DEFAULT_TRAIN_BATCH_SIZE,
                                  collate_fn=collate)

    # not sure what the following does, copied from run_lm_finetuning code
    """
    if args.max_steps > 0:
        t_total = args.max_steps
        args.num_train_epochs = args.max_steps \
            // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
    else:
        t_total = len(train_dataloader) // \
            args.gradient_accumulation_steps * args.num_train_epochs
    """

    if args.model_load_path:
        # Load in model and optimizer states
        print("***** Loading model from %s *****" % args.model_load_path)
        ckpt = torch.load(args.model_load_path)
        model.load_state_dict(ckpt["model_state_dict"])

    if not args.unfreeze_gpt2:
        model.freeze_gpt2()
    else:
        print("***** Unfreezing gpt2 parameters *****")
        model.unfreeze_gpt2()

    # load state dict before transferring to GPU to save memory
    model.to(device)

    optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
    global_step = 0

    if args.model_load_path:
        if not (ckpt["args"].unfreeze_gpt2 ^ args.unfreeze_gpt2)\
            and "optimizer_state_dict" in ckpt:
            print("***** Loading state of optimizer *****")
            optimizer.load_state_dict(ckpt["optimizer_state_dict"])
        else:
            print("***** Not loading state of optimizer *****")

    num_batches = len(train_dataset)
    # start training
    print("***** Running training *****")
    print("  Num epochs = %d" % args.train_epochs)
    print("  Num batches per epoch = %d" % num_batches)
    print("  Batch size = %d" % args.train_batch_size)
    print("  Logging every %d steps" % args.log_steps)
    if args.eval_and_save_by_epoch:
        print("  Evaluating and saving model every %.2f%% of every epoch" % (args.eval_and_save_by_epoch * 100))

    if args.eval_and_save_by_epoch and args.eval_and_save_by_epoch < 1.0:
        eval_and_save_by_steps = math.floor(num_batches * args.eval_and_save_by_epoch)
    else:
        eval_and_save_by_steps = None

    if args.model_save_path:
        # keep track of best loss for early stopping
        best_loss = float("inf")
        best_step = 0

    total_training_time = 0.0
    epoch = 0
    training_steps_in_this_session = 0
    stepped_in_epoch = None

    if args.model_load_path:
        epoch = ckpt["epoch"]
        global_step = ckpt["global_step"]
        if args.model_save_path and "best_loss" in ckpt:
            best_loss = ckpt["best_loss"]
        # stepped_in_epoch = ckpt["step_in_epoch"]
        # deal with case where one session may not finish one full epoch
        if num_batches == ckpt["num_batches_in_epoch"] \
            and ckpt["step_in_epoch"] != 0 \
            and ckpt["step_in_epoch"] != ckpt["num_batches_in_epoch"] - 1:
            stepped_in_epoch = ckpt["step_in_epoch"]
            print("Fastforwarding to step %d in epoch %d" % (stepped_in_epoch, epoch))

        if ckpt["step_in_epoch"] == ckpt["num_batches_in_epoch"] - 1:
            epoch = ckpt["epoch"] + 1

        # clean up checkpoint to save memory
        del ckpt
        torch.cuda.empty_cache()

    # data to monitor memory usage
    ctx_ids_dim0_sum = 0
    ctx_ids_dim1_sum = 0
    anaphor_ids_dim0_sum = 0
    anaphor_ids_dim1_sum = 0
    start_time = time.time()
    while epoch < args.train_epochs:
        print("*** Epoch %d ***" % epoch)
        for step, batch in enumerate(train_dataloader):
            # fast-forward
            if stepped_in_epoch and step <= stepped_in_epoch:
                continue
            elif stepped_in_epoch:
                stepped_in_epoch = None

            # monitor memory usage
            ctx_ids_dim0_sum += batch["ctx_ids"].shape[0]
            ctx_ids_dim1_sum += batch["ctx_ids"].shape[1]
            anaphor_ids_dim0_sum += batch["anaphor_ids"].shape[0]
            anaphor_ids_dim1_sum += batch["anaphor_ids"].shape[1]

            batch = batch_to_device(batch, device)
            model.zero_grad()
            model.train()

            res_dict = model(batch)
            loss = res_dict["loss"]

            loss.backward()

            optimizer.step()
            global_step += 1
            training_steps_in_this_session += 1

            loss = loss.item()
            del res_dict

            if training_steps_in_this_session < 20 or global_step % args.log_steps == 0:
                avg_time_per_batch = (time.time() - start_time) / training_steps_in_this_session
                estimated_time = (num_batches - (step+1)) * avg_time_per_batch
                print("  step %d/%d, global_step %d, batch loss = %.6f" \
                      % (step+1, num_batches, global_step, loss))
                print("  avg time per batch = %.2f, est %.2f mins left for this epoch" \
                      % (avg_time_per_batch, estimated_time / 60))

            if training_steps_in_this_session < 20 or global_step % (args.log_steps * 5) == 0:
                print("  [tensor dims] ctx_ids [%d, %d], avg [%.2f, %.2f], anaphor_ids [%d, %d], avg [%.2f, %.2f]" \
                      % (batch["ctx_ids"].shape[0], batch["ctx_ids"].shape[1],
                         ctx_ids_dim0_sum / training_steps_in_this_session,
                         ctx_ids_dim1_sum / training_steps_in_this_session,
                         batch["anaphor_ids"].shape[0], batch["anaphor_ids"].shape[1],
                         anaphor_ids_dim0_sum / training_steps_in_this_session,
                         anaphor_ids_dim1_sum / training_steps_in_this_session))

            if eval_and_save_by_steps and step % eval_and_save_by_steps == 0 \
                and step > 0 and step < num_batches - eval_and_save_by_steps:
                best_loss = eval_and_save_checkpoint(args, epoch, eval_dataset,
                    best_loss, step, num_batches, global_step, model,
                    optimizer if args.save_optimizer_state else None,
                    args.save_latest_state)

        best_loss = eval_and_save_checkpoint(args, epoch, eval_dataset,
            best_loss, step, num_batches, global_step, model,
            optimizer if args.save_optimizer_state else None,
            args.save_latest_state)
        epoch += 1

def eval_and_save_checkpoint(args, epoch, eval_dataset, best_loss, step_in_epoch,
                             num_batches_in_epoch, global_step,
                             model, optimizer, save_latest_state):
    eval_results = evaluate(args, model, eval_dataset, global_step)
    eval_loss = eval_results["eval_loss"]
    if args.model_save_path:
        model_checkpoint = {
            "args": args,
            "epoch": epoch,
            "eval_loss": eval_loss,
            "best_loss": min(eval_loss, best_loss),
            "step_in_epoch": step_in_epoch,
            "num_batches_in_epoch": num_batches_in_epoch,
            "global_step": global_step,
            "model_state_dict": model.state_dict()
        }
        if optimizer:
            model_checkpoint["optimizer_state_dict"] = optimizer.state_dict()
        if eval_loss < best_loss:
            best_save_path = args.model_save_path + "_best.ckpt"
            print("  current model has best eval loss, saving to %s" % best_save_path)
            torch.save(model_checkpoint, best_save_path)
        if save_latest_state:
            latest_save_path = args.model_save_path + "_latest.ckpt"
            print("  saving latest version of model to %s" % latest_save_path)
            torch.save(model_checkpoint, latest_save_path)
    best_loss = min(eval_loss, best_loss)
    return best_loss

def evaluate(args, model, eval_dataset, global_step):
    device = torch.device("cuda" if args.gpu else "cpu")
    eval_sampler = SequentialSampler(eval_dataset)
    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler,
                                 batch_size=DEFAULT_TRAIN_BATCH_SIZE,
                                 collate_fn=collate)

    print("***** Running evaluation at step %d ****" % global_step)
    print("  Num examples = %d" % len(eval_dataset))
    print("  Batch size = %d" % args.eval_batch_size)

    eval_loss = 0.0
    num_toks = 0
    model.eval()

    for step, batch in enumerate(eval_dataloader):
        with torch.no_grad():
            batch = batch_to_device(batch, device)
            res_dict = model(batch)
            if step + 1 % args.log_steps == 0:
                print("  evaluated %d batches" % step + 1)
            eval_loss += res_dict["loss"].item() * res_dict["num_toks"].item()
            num_toks += res_dict["num_toks"].item()
            del res_dict

    eval_loss = eval_loss / num_toks
    perplexity = torch.exp(torch.tensor(eval_loss))

    print("***** Eval results at step %d  *****" % global_step)
    print("  eval_loss = %.6f" % eval_loss)
    print("  perplexity = %.6f" % perplexity)
    return {
        "eval_loss": eval_loss,
        "perplexity": perplexity
    }


"""forms for home app"""

from django import forms
from django.utils.translation import ugettext_lazy as _
from django.template.defaultfilters import filesizeformat
from django.conf import settings
from .models import ProjectTable,ProjectAGS,ContactTable,Projectprofile,Projectdefaultprofilecategory, Projectdefaultprofile, ProjectExcel

variable1Choices = (
        ('TCR','TCR'),
        ('SCR','SCR'),
        ('RQD','RQD'),
        ('N SPT','N SPT'),
        ('CL','CL'),
        ('PH','PH'),
        ('SC','SC'),
        ('LL','LL'),
        ('PL','PL'),
        ('PI','PI'),
        ('Moisture Content','Moisture Content'),
        ('Specific Gravity','Specific Gravity'),
        ('Bulk Density','Bulk Density'),
        ('Max Dry Density','Max Dry Density'),
        ('Min Dry Density','Min Dry Density'),
        ('Water Content','Water Content'),
        ('UCS','UCS'),
        ('Peak Angle of Friction','Peak Angle of Friction'),
        ('Residual Angle of Friction','Residual Angle of Friction'),
        ('Peak Cohesion','Peak Cohesion'),
        ('Residual Cohesion','Residual Cohesion'),
        ('Particle Size','Particle Size'),
        ('% passing 0.425mm','% passing 0.425mm'),
        ('% of Gravel','% of Gravel'),
        ('Water Level','Water Level')
    )

variable2Choices = (
        ('Elevation','Elevation'),
        ('Depth','Depth'),
        # ('Percentage Passing (%)','Percentage Passing (%)'),
    )

variable3Choices = (
        ('borehole','By Bore Hole'),
        ('machine','By Machine Type'),
        ('boreholeandmachine','By Bore Hole and Machine Type'),
        # ('boreholenumber','By Bore Number'),
    )

chartindexChoices = (
        ('First','First'),
        ('Second','Second'),
        ('Third','Third'),
        ('Fourth','Fourth'),
        ('Five','Five'),
        ('Six','Six'),
        ('Seven','Seven'),
        ('Eight','Eight'),
        ('Nine','Nine'),
        ('Ten','Ten'),
        ('Eleven','Eleven'),
        ('Twelveve','Twelveve'),
    )

class ProjectForm(forms.ModelForm):
    """form for project"""

    def clean_project_image(self):
        """image upload size"""
        photo = self.cleaned_data["thumnail"]
        if photo and photo.size > settings.MAX_UPLOAD_SIZE:
            raise forms.ValidationError(
                _("Please keep photo filesize under %s. filesize is slightly over %s") %
                                        (filesizeformat(settings.MAX_UPLOAD_SIZE),
                                         filesizeformat(photo.size)))
        return photo

    class Meta:
        """Meta"""
        model = ProjectTable
        fields = ("name", "summery", "location",
                  "number_of_bore_home","max_depth","thumnail")
        labels = {"name": "Project Name",
                  "summery": "Project Descriptions",
                  "thumnail": "Project Photo",
                  "location": "Project Location",
                  "number_of_bore_home": "Number of Bore Hole",
                  "max_depth": "Max Depth"
                  }

class ProjectAGSForm(forms.ModelForm):
    """form for project"""

    class Meta:
        """Meta"""
        model = ProjectAGS
        fields = ("ags_file",)
        labels = {
                  "ags_file": "Project AGS files"
                  }

class ProjectEXCELForm(forms.ModelForm):
    """form for project"""

    class Meta:
        """Meta"""
        model = ProjectExcel
        fields = ("excel_file",)

class ContactForm(forms.ModelForm):
    """form for contact"""

    class Meta:
        """Meta"""
        model = ContactTable
        fields = ("name","email","phone","message")
        labels = {
                  "name": "Name",
                  "email": "Email",
                  "phone": "Phone",
                  "message": "Message",
                  }
        widgets = {
            'name': forms.TextInput(attrs={'class': 'contactform'}),
            'email': forms.TextInput(attrs={'class': 'contactform'}),
            'phone': forms.TextInput(attrs={'class': 'contactform'}),
            'message': forms.Textarea(attrs={'class': 'contactformmessage'}),
        }

class ProfileForm(forms.ModelForm):
    """form for contact"""

    class Meta:
        """Meta"""
        model = Projectprofile
        fields = ("name","group","chart")
        labels = {
                  "name": "Name",
                  "group": "Group",
                  "chart": "Charts"
                  }

class ProfileDefaultCategoryForm(forms.ModelForm):
    """form for Default category"""

    class Meta:
        """Meta"""
        model = Projectdefaultprofilecategory
        fields = ("name",)
        labels = {
                  "name": "Name"
                  }

class ProfileDefaultForm(forms.ModelForm):
    """form for Default Profile"""
    variable1 = forms.ChoiceField(choices = variable1Choices)
    variable2 = forms.ChoiceField(choices = variable2Choices)
    variable3 = forms.ChoiceField(choices = variable3Choices)

    class Meta:
        """Meta"""
        model = Projectdefaultprofile
        fields = ("category","variable1","variable2","variable3",)
        labels = {
                  "category": "Category",
                  "variable1": "Variable One",
                  "variable2": "Variable Two",
                  "variable3": "Variable Three",
                  }


import threading
from queue import Queue
import time

import youtube_dl
from bs4 import BeautifulSoup
import requests

def download(video_key):
	print('http://www.youtube.com/watch?v=%s' % video_key)
	ydl_opts = {
	    'format': 'bestaudio/best',
	    'postprocessors': [{
	        'key': 'FFmpegExtractAudio',
	        'preferredcodec': 'mp3',
	        'preferredquality': '140',
	    }],
	}
	with youtube_dl.YoutubeDL(ydl_opts) as ydl:
	    ydl.download(['http://www.youtube.com/watch?v=%s' % video_key])

def remove_punctuation(sentance):
	# Will replace all puncutation marks with a single space
	# Will Also replace all double/tripple spaces with a single space

	punctuation_marks = """. , : ; ! [ ] ( ) # { } - _ | ` ´ " ' / & < > ~ ^ ¨ * % $ @"""

	for mark in punctuation_marks.split(" "):
		sentance = sentance.replace(mark, " ")

		sentance = sentance.replace("  ", " ")
		sentance = sentance.replace("   ", " ")

	return sentance

def accuracy_test(spotify_title, youtube_title, accuracy_level):

	# Remove puncuation and capital letters
	spotify_title = remove_punctuation(spotify_title).lower()
	youtube_title = remove_punctuation(youtube_title).lower()

	# Split titles for itteration
	spotify_title_list = spotify_title.split(" ")
	youtube_title_list = youtube_title.split(" ")

	# If accuracy level is higher then amount of chars in title, lower it
	# This need to be fixed in order to make searches more accurate
	if len(spotify_title_list) < accuracy_level:
		accuracy_level = len(spotify_title) - 1

	# Compare the mount of matching strings between the two titles
	similar_word_count = 0
	for x in spotify_title_list:
		for y in youtube_title_list:
			if x == y:
				similar_word_count += 1

	# Will only return True if two or more strings are similar
	if similar_word_count < accuracy_level:
		return False
	else:
		return True

def get_spotify_playlist(uri):
	temp_tracks = []

	embed_format = "https://embed.spotify.com/?uri="
	r = requests.get(embed_format+uri)
	soup = BeautifulSoup(r.text)

	track_artists = soup.findAll("li", { "class" : "artist" })
	track_title = soup.findAll("li", { "class" : "track-title" })

	for n in range(0, len(track_title)):
		track = "%s:%s" % (track_artists[n].text, track_title[n].text)
		temp_tracks.append(track)

	return temp_tracks

def get_youtube_link(tracks):
	yt_search = "https://www.youtube.com/results?search_query="
	video_keys = []

	for track in tracks:
		track = track.replace(":"," ")

		r = requests.get(yt_search + track)
		soup = BeautifulSoup(r.text)
		item_section = soup.find("ol", {"class": "item-section"})
		items = item_section.findAll("div", {"class": "yt-lockup-tile"})

		view_count = 0
		video_key = ""

		videos = []

		for item in items:
			# SOMEWHERE IN HERE IS WHERE THE ACCUARCY CHECKER NEEDS TO GO
			# FORMAT FOR accuracy_test IS:
			# Spotify_title, Youtube_title, accuracy_level
			# I DONT WANT TO BREAK, I WANT TO PICK NEXT SONG

			item_title = item.find("a", {"class": "yt-uix-tile-link"})

			if accuracy_test(track, item_title.text, 4) == False:
				continue
			else:
				pass

			item_meta_info = item.find("ul", {"class": "yt-lockup-meta-info"})
			item_key = item.get('data-context-item-id')

			# Some items are not videos but playlists with no meta info which would crash script
			try:
				item_views = str(item_meta_info).split("<li>")[2].replace(u"\xa0", "").split(" ")[0]
				print("Video not accurate enough")
			except:
				pass

			item_views = int(item_views)

			if item_views > view_count:
				view_count = item_views
				video_key = item_key

		print(track, video_key)
		video_keys.append(video_key)

	# Will print all video keys collected
	for x in video_keys:
		print(x)

	return video_keys

# lock to serialize console output
lock = threading.Lock()

def do_work(item):
    time.sleep(1) # pretend to do some lengthy work.
    # Make sure the whole print completes or threads can mix up output in one line.
    with lock:
        print(threading.current_thread().name,item)

# The worker thread pulls an item from the queue and processes it
def worker():
    while True:
        item = q.get()
        download(item)
        q.task_done()

if __name__ == '__main__':
	uri = "spotify:user:tehdawg:playlist:7yblzr1UMec0P7xixy3GOY"
	tracks = get_spotify_playlist(uri)
	video_keys = get_youtube_link(tracks)

	# Create the queue and thread pool.
	q = Queue()
	for i in range(4):
	     t = threading.Thread(target=worker)
	     t.daemon = True  # thread dies when main thread (only non-daemon thread) exits.
	     t.start()

	# stuff work items on the queue (in this case, just a number).
	start = time.perf_counter()
	for video_key in video_keys:
	    q.put(video_key)

	q.join()


import numpy as np
import time
import matplotlib.pyplot as plt; plt.ion()
from mpl_toolkits.mplot3d import Axes3D
from mpl_toolkits.mplot3d.art3d import Poly3DCollection
import Planner
from collsion_check import *
import heapq
from utils import *
from parameters import *
from main import *

class wAStar:
    def __init__(self, boundary, blocks, cell_density = 5, weight = 1):
        """
            cell_density: cells per unit length
            weight: the epsilon value for heuristic function
        """
        # we assume that boundary will be integer
        boundary = boundary[0][0:6]
        x = int(boundary[3]-boundary[0])
        y = int(boundary[4]-boundary[1])
        z = int(boundary[5]-boundary[2])
        self.boundary = np.array(boundary)
        self.mxi = x * cell_density - 1
        self.myi = y * cell_density - 1
        self.mzi = z * cell_density - 1
        self.space = np.zeros((x * cell_density, y * cell_density, z * cell_density), dtype=np.int)
        self.dist = 1 / cell_density
        self.blocks = blocks
        self.weight = weight
        self.insert_blocks()

    def insert_blocks(self):
        """
            fill the voxel where there is block exits
        """
        for block in self.blocks:
            # print(np.array([block[0], block[3]]))
            p1 = self.c2g(block[0:3])
            p2 = self.c2g(block[3:6])
            # print("block space:" + str(p1) + str(p2) )
            self.space[p1[0]:p2[0]+1, p1[1]:p2[1]+1, p1[2]:p2[2]+1] = 1

    def c2g(self, point):
        """
            continuous world coordinate to grid voxel index
        """
        point = point - self.boundary[0:3]
        res = np.ceil(point/self.dist)-1
        res[res<0] = 0
        # res.astype(np.int)
        res = np.array([int(res[0]), int(res[1]), int(res[2])])
        return res

    def g2c(self, index):
        """
            grid voxel index to continuous world coordinate
        """
        return (index)*self.dist + self.boundary[0:3] + 0.5*self.dist

    def h(self,point, goal):
        """
            h is admissible
        """
        delta = np.abs(point-goal)
        return max(delta) + 0.7*min(delta)

    def solve(self, start, goal, show_path = False):
        """
            the main operation of weighted A* algorithm
        """
        si = self.c2g(start)
        # print(si)
        ti = self.c2g(goal)
        closed = np.zeros(self.space.shape)
        open = [[0, si[0], si[1], si[2]]]
        open_buff = {hashindex(si):0}
        g_matrix = np.ones(self.space.shape)*np.inf
        g_matrix[si[0], si[1], si[2]] = 0

        p_matrix = [[[[0,0,0] for i in range(self.mzi+1)] for i in range(self.myi+1)] for i in range(self.mxi+1)]
        p_matrix[si[0]][si[1]][si[2]] = 0

        c = 0
        while closed[ti[0],ti[1],ti[2]] == 0:
            # c += 1
            # if c % 500 == 0:
            #     print("iteration times:" + str(c))
            # g, px,py,pz = open.pop(0)

            g, px, py, pz = heapq.heappop(open)
            if closed[px][py][pz] == 1:
                continue

            point = [px,py,pz]
            # open_buff.pop(hashindex(point))

            closed[point[0],point[1],point[2]] = 1
            if g_matrix[point[0],point[1],point[2]] == np.inf:
                break
            child_points, costs = self.findChild(point, closed)
            # print(child_points)
            for i, npoint in enumerate(child_points):
                nval = g_matrix[point[0],point[1],point[2]] + costs[i]
                if g_matrix[npoint[0],npoint[1],npoint[2]] > nval:
                    g_matrix[npoint[0], npoint[1], npoint[2]] = nval
                    # print(npoint[0])
                    pointer = p_matrix[npoint[0]][npoint[1]][npoint[2]]
                    pointer[0] = point[0]
                    pointer[1] = point[1]
                    pointer[2] = point[2]

                    # hasflag = False
                    # for i in range(len(open)):
                    #     if open[i][1] == npoint[0] and open[i][2] == npoint[1] and open[i][3] == npoint[2]:
                    #         open[i][0] = nval + self.weight*self.h(npoint,ti)
                    #         hasflag = True
                    #         break
                    # if not hasflag:
                    #     open.append([nval + self.weight*self.h(npoint,ti), npoint[0], npoint[1], npoint[2]])
                    # open.sort()

                    heapq.heappush(open,[nval + self.weight*self.h(npoint,ti), npoint[0], npoint[1], npoint[2]])

        path = [ti]
        pointer = p_matrix[ti[0]][ti[1]][ti[2]]
        while pointer != 0:
            path.append(pointer)
            pointer = p_matrix[pointer[0]][pointer[1]][pointer[2]]
        # path = path[::-1]
        # path.appe
        path = np.array(path)
        path = self.g2c(path)
        path = path[::-1]
        n = np.sum(closed)
        print("The number of considered nodes is: "+str(n))
        if(show_path):
            print("the path will be:")
            print(path)
        return path,g_matrix[ti[0],ti[1],ti[2]]*self.dist

    def findChild(self,point, closed):
        res = []
        costs = []
        for i,dir in enumerate(DIRS):
            npoint = point.copy()
            npoint[0] += dir[0]
            npoint[1] += dir[1]
            npoint[2] += dir[2]
            if npoint[0] >= 0 and npoint[0] <= self.mxi and npoint[1] >= 0 and npoint[1] <= self.myi and npoint[2] >= 0 and npoint[2] <= self.mzi:
                if closed[npoint[0],npoint[1],npoint[2]] == 0 and self.space[npoint[0],npoint[1],npoint[2]] == 0:
                    res.append(npoint)
                    costs.append(COSTS[i])
        return res, costs

if __name__ == "__main__":
    # boundary = [[ -5.,  -5.,  -5. , 10.,  10.,  10. ,120., 120., 120.]]
    # blocks = [[4.5, 4.5, 2.5, 5.5, 5.5, 3.5],[0,0,0,1,1,1]]
    # planner = wAStar(boundary, blocks)
    # # print(planner.g2c(np.ones(3)*0))
    # # print(planner.c2g(np.ones(3)*-4.8))
    # planner.insert_blocks()
    # # print(planner.c2g(np.ones(3) * 10))
    # print(planner.space[52,52,42])
    # print(max(np.ones(3)))

    print('Running single cube test...\n')
    start = np.array([2.3, 2.3, 1.3])
    goal = np.array([7.0, 7.0, 5.5])
    success, pathlength = runtest('./maps/single_cube.txt', start, goal, True)
    print('Success: %r' % success)
    print('Path length: %d' % pathlength)
    print('\n')


import numpy as np
from sklearn.neighbors import BallTree, KDTree
import os

import gc

# each action -> a lru_knn buffer
# there are two "key"s, a learned one and a fixed one.
# the fixed one ensures that our method should be better than baseline, i.e. model free episodic control
class LRU_KNN_SingleAction(object):
    def __init__(self, capacity, hash_dim, env_name,action_number=0):
        self.env_name = env_name
        self.capacity = capacity
        # self.states = np.empty((capacity, z_dim), dtype=np.float32)  # learned keys
        self.prev_action = [[] for _ in range(capacity)]
        self.next_action = [[] for _ in range(capacity)]
        self.next_id = [[] for _ in range(capacity)]
        self.prev_id = [[] for _ in range(capacity)]
        self.hashes = np.empty((capacity, hash_dim), dtype=np.float32)  # fixed keys
        # self.hashes = dict()  # fixed keys
        self.q_values_decay = np.zeros(capacity)
        self.lru = np.zeros(capacity)
        self.curr_capacity = 0
        self.tm = 0.0
        # self.tree = None
        self.hash_tree = None
        # self.hash_tree = None
        self.addnum = 0
        self.buildnum = 256
        self.buildnum_max = 256
        self.bufpath = './buffer/%s' % self.env_name
        self.build_tree_times = 0
        self.build_tree = False
        self.action_number = action_number

    def peek(self, h, value_decay, modify, prev_id=-1, prev_action=-1):
        if self.curr_capacity == 0 or self.build_tree is False:
            return None, None

        dist, ind = self.hash_tree.query([h], k=1)
        ind = ind[0][0]
        # if self.states[ind] == key:
        # if np.allclose(self.states[ind], key):
        # if np.allclose(self.hashes[ind], h, atol=1e-08):
        if dist[0][0] < 1e-9:
            self.lru[ind] = self.tm
            self.tm += 0.01
            if modify:
                if value_decay > self.q_values_decay[ind]:
                    self.q_values_decay[ind] = value_decay
                    if prev_id >= 0:
                        self.prev_id[ind].append(prev_id)
                        self.prev_action[ind].append(prev_action)
            return self.q_values_decay[ind], ind
        # print self.states[ind], key
        else:
            return None, None

    @staticmethod
    def switch_first_half(obs, obs_next, batch_size):
        half_size = int(batch_size / 2)
        tmp = obs[:half_size, ...]
        obs[:half_size, ...] = obs_next[:half_size, ...]
        obs_next[:half_size, ...] = tmp
        return obs, obs_next

    def update(self, idxes, values):
        assert len(idxes) == len(values)
        for id, value in zip(idxes, values):
            self.hashes[id] = value

    def sample_neg_keys(self, avoids, batch_size):
        # sample negative keys
        assert batch_size + len(
            avoids) <= self.curr_capacity, "can't sample that much neg samples from episodic memory!"
        idxes = []
        while len(idxes) < batch_size:
            id = np.random.randint(0, self.curr_capacity)

            if (id not in idxes) and not (np.array([np.array_equal(self.hashes[id], x) for x in avoids]).any()):
                idxes.append(id)
        return idxes

    def act_value(self, key, h, knn=4):
        value, _ = self.peek(None, h, modify=False)
        if value is not None:
            return value, True
        else:
            # print(self.curr_capacity,knn)
            return self.knn_value(key, knn=knn), False

    def knn_value(self, key, knn):
        knn = min(self.curr_capacity, knn)
        if self.curr_capacity == 0 or self.build_tree == False:
            return 0.0
        key = np.squeeze(key)
        dist, ind = self.hash_tree.query([key], k=knn)

        coeff = np.exp(dist[0])
        coeff = coeff / np.sum(coeff)
        # value = 0.0
        value_decay = 0.0
        # count = 0
        for j, index in enumerate(ind[0]):
            value_decay += self.q_values_decay[index] * coeff[j]
            # count += self.count[index] * coeff[j]
            self.lru[index] = self.tm
            self.tm += 0.01

        q_decay = value_decay

        return q_decay

    # def add(self, hash, value_decay, prev_id=-1, prev_action=-1):
    #     if self.curr_capacity >= self.capacity:
    #         # find the LRU entry
    #         old_index = int(np.argmin(self.lru))
    #         self.hashes[old_index] = hash
    #         # self.hashes[tuple(hash)] = old_index
    #         # self.obses[old_index] = obs
    #         self.q_values_decay[old_index] = value_decay
    #         self.lru[old_index] = self.tm
    #         if prev_id >= 0:
    #             self.next_id[prev_id] = [old_index]
    #             self.next_action[prev_id] = [self.action_number]
    #         self.prev_id[old_index] = [prev_id]
    #         self.prev_action[old_index] = [prev_action]
    #         # self.next_id[old_index] = next_id
    #         self.tm += 0.01
    #         return old_index
    #     else:
    #         # self.states[self.curr_capacity] = key
    #         self.hashes[self.curr_capacity] = hash
    #         # self.hashes[tuple(hash)] = self.curr_capacity
    #         # self.obses[self.curr_capacity] = obs
    #         self.q_values_decay[self.curr_capacity] = value_decay
    #         self.lru[self.curr_capacity] = self.tm
    #         if prev_id >= 0:
    #             self.next_id[prev_id] = [self.curr_capacity]
    #             self.next_action[prev_id] = [self.action_number]
    #         self.prev_id[self.curr_capacity] = [prev_id]
    #         self.prev_action[self.curr_capacity] = [prev_action]
    #         # self.next_id[self.curr_capacity] = next_id
    #         self.curr_capacity += 1
    #         self.tm += 0.01
    #         print("adding here", self.curr_capacity)
    #         return self.curr_capacity - 1

    def update_kdtree(self):
        if self.build_tree:
            del self.hash_tree
            # del self.hash_tree
        if self.curr_capacity <= 0:
            return
        print("build tree", self.curr_capacity)
        # self.tree = KDTree(self.states[:self.curr_capacity])
        self.hash_tree = KDTree(self.hashes[:self.curr_capacity])
        # self.hash_tree = KDTree(self.hashes[:self.curr_capacity])
        self.build_tree = True
        self.build_tree_times += 1
        if self.build_tree_times == 50:
            self.build_tree_times = 0
            gc.collect()


# Import the necessary libraries and load the data:
import dash
import dash_bootstrap_components as dbc
from dash import html, dcc
from dash.dependencies import Input, Output
import plotly.express as px
import pandas as pd
from model import load_model    # , predict_diagnosis
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
from pathlib import Path

current_file_path = Path(__file__).resolve()
root_path = Path(current_file_path / "../../../..").resolve()

# Load the data
pre_pca_data = pd.read_csv(root_path / ("src/visualization/cancer/"
                                        "data/cancer_data.csv"))
pre_all_data = pd.read_csv(root_path / ("src/visualization/cancer/"
                                        "data/cancer_data_all_features.csv"))
original_data = pd.read_csv(root_path / ("data/raw/cancer_data_original.csv"))

path_to_logistic_regression_model = root_path / ("src/visualization/cancer/model/logistic_regression_model.pkl")
path_to_logistic_regression_model_all_features = root_path / ("src/visualization/cancer/model"
                                                              "/logistic_regression_model_all_features.pkl")

# Prepare the data for visualization:

# Separate features and target
X_pca = pre_pca_data.drop(columns=['diagnosis'])
y_pca = pre_pca_data['diagnosis']

X_all = pre_all_data.drop(columns=['diagnosis'])
y_all = pre_all_data['diagnosis']

X_original = original_data.drop(columns=["diagnosis", "Unnamed: 32", "id"])
y_original = original_data['diagnosis']

# Split the data into train and test sets
X_train_pca, X_test_pca, y_train_pca, y_test_pca = train_test_split(X_pca, y_pca, test_size=0.2, random_state=42)
X_train_all, X_test_all, y_train_all, y_test_all = train_test_split(X_all, y_all, test_size=0.2, random_state=42)
X_train_original, X_test_original, y_train_original, y_test_original = train_test_split(
    X_original, y_original, test_size=0.2, random_state=42)

# Load the model and make predictions
model_pca = load_model(path_to_logistic_regression_model)
y_pred_pca = model_pca.predict(X_test_pca)

model_all = load_model(path_to_logistic_regression_model_all_features)
y_pred_all = model_all.predict(X_test_all)

# Calculate model performance metrics
accuracy_pca = accuracy_score(y_test_pca, y_pred_pca)
precision_pca = precision_score(y_test_pca, y_pred_pca)
recall_pca = recall_score(y_test_pca, y_pred_pca)
f1_pca = f1_score(y_test_pca, y_pred_pca)
roc_auc_pca = roc_auc_score(y_test_pca, y_pred_pca)

# Calculate model performance metrics
accuracy_all = accuracy_score(y_test_all, y_pred_all)
precision_all = precision_score(y_test_all, y_pred_all)
recall_all = recall_score(y_test_all, y_pred_all)
f1_all = f1_score(y_test_all, y_pred_all)
roc_auc_all = roc_auc_score(y_test_all, y_pred_all)

# Create the app layout with the visualizations:

app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])

app.layout = dbc.Container([
    dbc.Row([
        dbc.Col(html.H1("Cancer Diagnosis Prediction Dashboard"), className="text-center")
    ], className="mt-4"),

    # Feature Importance Visualization
    dbc.Row([
        dbc.Col([
            dcc.Graph(id='feature-importance-pca', figure=px.bar(x=X_pca.columns,
                      y=model_pca.coef_[0], labels={'x': 'Features', 'y': 'Importance'},
                      title="PCA Pre Processed Feature Importance"))
        ])
    ], className="mt-4"),

    # Feature Importance Visualization
    dbc.Row([
        dbc.Col([
            dcc.Graph(id='feature-importance-all', figure=px.bar(x=X_all.columns,
                      y=model_all.coef_[0], labels={'x': 'Features', 'y': 'Importance'},
                      title="Feature Importance (all)"))
        ])
    ], className="mt-4"),

    # Feature Relationships Visualization
    dbc.Row([
        dbc.Col([
            dcc.Dropdown(id='x-axis-pca', options=[{'label': i, 'value': i}
                         for i in X_pca.columns], value='radius_mean', placeholder="Select Feature 1"),
            dcc.Dropdown(id='y-axis-pca', options=[{'label': i, 'value': i}
                         for i in X_pca.columns], value='texture_mean', placeholder="Select Feature 2"),
            dcc.Graph(id='scatter-plot-pca')
        ])
    ], className="mt-4"),
    # Feature Relationships Visualization
    dbc.Row([
        dbc.Col([
            dcc.Dropdown(id='x-axis-all', options=[{'label': i, 'value': i}
                         for i in X_all.columns], value='radius_mean', placeholder="Select Feature 1"),
            dcc.Dropdown(id='y-axis-all', options=[{'label': i, 'value': i}
                         for i in X_all.columns], value='texture_mean', placeholder="Select Feature 2"),
            dcc.Graph(id='scatter-plot-all')
        ])
    ], className="mt-4"),
    # Original Feature Relationships Visualization
    dbc.Row([
        dbc.Col([
            dcc.Dropdown(id='x-axis-original', options=[{'label': i, 'value': i}
                         for i in X_original.columns], value='radius_mean', placeholder="Select Feature 1"),
            dcc.Dropdown(id='y-axis-original', options=[{'label': i, 'value': i}
                         for i in X_original.columns], value='texture_mean', placeholder="Select Feature 2"),
            dcc.Graph(id='scatter-plot-original')
        ])
    ], className="mt-4"),
    dbc.Row([
        dbc.Col([
            dcc.Graph(id='performance-metrics-pca', figure=px.bar(
                x=[accuracy_pca, precision_pca, recall_pca, f1_pca, roc_auc_pca],
                y=['Accuracy', 'Precision', 'Recall', 'F1', 'ROC AUC'],
                orientation='h',
                labels={'x': 'Metric Value', 'y': 'Metrics'},
                title="PCA preprocessed Model Performance Metrics",
                text=[round(accuracy_pca, 2), round(precision_pca, 2), round(
                    recall_pca, 2), round(f1_pca, 2), round(roc_auc_pca, 2)]
            ))
        ])
    ], className="mt-4"),
    dbc.Row([
        dbc.Col([
            dcc.Graph(id='performance-metrics-all', figure=px.bar(
                x=[accuracy_all, precision_all, recall_all, f1_all, roc_auc_all],
                y=['Accuracy', 'Precision', 'Recall', 'F1', 'ROC AUC'],
                orientation='h',
                labels={'x': 'Metric Value', 'y': 'Metrics'},
                title="All features kept Model Performance Metrics",
                text=[round(accuracy_all, 2), round(precision_all, 2), round(
                    recall_all, 2), round(f1_all, 2), round(roc_auc_all, 2)]
            ))
        ])
    ], className="mt-4"),

])

# Update the scatter plot based on the selected features

@app.callback(
    Output('scatter-plot-pca', 'figure'),
    [Input('x-axis-pca', 'value'), Input('y-axis-pca', 'value')]
)
def update_scatter_pca(x_axis, y_axis):
    return px.scatter(data_frame=pre_pca_data, x=x_axis, y=y_axis, color='diagnosis',
                      title="PCA Preprocessed Feature Relationships", labels={'diagnosis': 'Diagnosis'},
                      hover_data=X_pca.columns, height=500)

@app.callback(
    Output('scatter-plot-all', 'figure'),
    [Input('x-axis-all', 'value'), Input('y-axis-all', 'value')]
)
def update_scatter_all(x_axis, y_axis):
    return px.scatter(data_frame=pre_all_data, x=x_axis, y=y_axis, color='diagnosis',
                      title="Preprocessed Feature Relationships", labels={'diagnosis': 'Diagnosis'},
                      hover_data=X_all.columns, height=500)

@app.callback(
    Output('scatter-plot-original', 'figure'),
    [Input('x-axis-original', 'value'), Input('y-axis-original', 'value')]
)
def update_scatter_original(x_axis, y_axis):
    return px.scatter(data_frame=original_data, x=x_axis, y=y_axis, color='diagnosis',
                      title="Original Feature Relationships", labels={'diagnosis': 'Diagnosis'},
                      hover_data=X_all.columns, height=500)

if __name__ == '__main__':
    app.run_server(debug=True)


from copy import deepcopy

import pandas as pd
from matplotlib import pyplot as plt

from statsmodels.tsa.seasonal import seasonal_decompose
import ruptures as rpt
from luminol.anomaly_detector import AnomalyDetector
from fbprophet import Prophet
from fbprophet.plot import add_changepoints_to_plot

# from pandas.plotting import autocorrelation_plot

@pd.api.extensions.register_dataframe_accessor("explorer")
class Explorer(object):
    def __init__(self, pandas_obj):
        self._obj = pandas_obj

        if not type(self._obj.index) == pd.core.indexes.datetimes.DatetimeIndex:
            raise TypeError("YOU NEED TO SET INDEX AS DATETIME FORMAT")

        self.weekday_obj = self.group_by_weekday()  # weekday
        self._figsize = (8, 8)

        self.outliers = None

    # 1.set figsize
    @property
    def figsize(self):
        return self._figsize

    @figsize.setter
    def figsize(self, value):
        self._figsize = value

    @property
    def obj(self):
        return self._obj

    @property
    def outliers(self):
        return self._outliers

    @outliers.setter
    def outliers(self, outliers):
        self._outliers = outliers

    # 2. group by
    def group_by_weekday(self):
        return self._obj.groupby(self._obj.index.weekday)

    # 3. statistics
    def stats_by_feature(self):
        obj = self._obj.describe().T
        sum_obj = pd.DataFrame(self._obj.sum())
        sum_obj.columns = ['sum']
        return pd.concat([obj, sum_obj], axis=1)

    def stats_by_freq(self, freq='Y'):
        obj = self._obj.groupby(pd.Grouper(freq=freq)).describe()
        obj.columns = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max']

        sum_obj = pd.DataFrame(self._obj.groupby(pd.Grouper(freq=freq)).sum())
        sum_obj.columns = ['sum']
        return pd.concat([obj, sum_obj], axis=1)

    def stats_by_weekday(self):
        obj = self.weekday_obj.describe()
        obj.columns = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max']

        sum_obj = pd.DataFrame(self.weekday_obj.sum())
        sum_obj.columns = ['sum']
        return pd.concat([obj, sum_obj], axis=1)

    # 4. plot
    # 4-1. line and bar plot
    def plot_by_feature(self, kind='bar', how='sum', style=None, stacked=None):
        if how == 'sum':
            self._obj.sum().plot(kind=kind, style=style, stacked=stacked,
                                 figsize=self._figsize,
                                 ylim=[0, self._obj.sum().max() * 1.2])
        elif how == 'mean':
            self._obj.mean().plot(kind=kind, style=style, stacked=stacked,
                                  figsize=self._figsize,
                                  ylim=[0, self._obj.mean().max() * 1.2])

    def plot_by_freq(self, freq='Y', kind='line', how='sum', style=None, stacked=None):
        freq_obj = self._obj.groupby(pd.Grouper(freq=freq))
        if how == 'sum':
            freq_obj.sum().plot(
                kind=kind, style=style, stacked=stacked,
                figsize=self._figsize, ylim=[0, freq_obj.sum().max().max() * 1.2])
        elif how == 'mean':
            freq_obj.mean().plot(
                kind=kind, style=style, stacked=stacked,
                figsize=self._figsize, ylim=[0, freq_obj.mean().max().max() * 1.2])

    def plot_by_weekday(self, kind='line', how='sum', style=None, stacked=None):
        if how == 'sum':
            self.weekday_obj.sum().plot(kind=kind, style=style, stacked=stacked,
                                        figsize=self._figsize,
                                        ylim=[0, self.weekday_obj.sum().max().max() * 1.2])
        elif how == 'mean':
            self.weekday_obj.mean().plot(kind=kind, style=style, stacked=stacked,
                                         figsize=self._figsize,
                                         ylim=[0, self.weekday_obj.mean().max().max() * 1.2])

    # 4-2. histogram and density
    def hist(self, bins=15):
        self._obj.hist(figsize=self._figsize, bins=bins)

    def density(self):
        self._obj.plot(kind='kde', figsize=self._figsize)

    # 4-3. boxplot
    def boxplot(self, freq='Y'):
        groups = pd.DataFrame()
        code = self._obj.columns[0]

        for name, group in self._obj.groupby(pd.Grouper(freq=freq)):
            if freq == 'Y':
                groups[name.year] = pd.Series(group[code].values)
            elif freq == 'Q':
                groups[name.year + (name.quarter - 1) * 1 / 4] = pd.Series(group[code].values)
            elif freq == 'M':
                groups[name.year + (name.month - 1) * 1 / 12] = pd.Series(group[code].values)
        groups.boxplot(figsize=self._figsize)

    # 4-4. scatter
    def scatter(self, color=None, label=None):
        plt.scatter(self._obj.index,
                    self._obj[self._obj.columns[0]],
                    c=color, label=label, alpha=0.8)

    # 4-5. trend, seasonal
    def decompose_yearly(self):
        return_obj = seasonal_decompose(self._obj, freq=365)
        return return_obj

    def decompose_monthly(self):
        return_obj = seasonal_decompose(self._obj, freq=30)
        return return_obj

    def decompose_weekly(self):
        return_obj = seasonal_decompose(self._obj, freq=7)
        return return_obj

    # 4-6. trend changepoint
    def prophet_change_point(self):
        obj = self._obj.reset_index()
        obj.columns = ['ds', 'y']

        model = Prophet(weekly_seasonality=True, yearly_seasonality=True)
        model.fit(obj)

        # 1. plot1
        plt.figure(figsize=self.figsize)
        plt.plot(self._obj)
        for i, changepoint in enumerate(model.changepoints):
            print(i, changepoint)
            plt.axvline(changepoint, ls='--', lw=1, c='r')

        # 2. plot2
        deltas = model.params['delta'].mean(0)
        fig = plt.figure(facecolor='w', figsize=self.figsize)
        ax = fig.add_subplot(111)
        ax.bar(range(len(deltas)), deltas)
        ax.grid(True, which='major', c='gray', ls='-', lw=1, alpha=0.2)
        ax.set_ylabel('Rate change')
        ax.set_xlabel('Potential changepoint')
        fig.tight_layout()

    # 4-7. Outlier
    # get_outlier
    def detect_outlier(self):
        ts_dict = {}
        values = self._obj.values

        for i in range(len(self._obj)):
            ts_dict[i] = int(values[i])

        # 2. detect anomaly
        detector = AnomalyDetector(ts_dict)
        score = detector.get_all_scores()

        # 3. outlier dictionary
        outlier_dict = {}
        for timestamp, value in score.iteritems():
            outlier_dict[timestamp] = value

        self.outliers = sorted(outlier_dict.items(), key=lambda kv: kv[1], reverse=True)

    def show_outlier(self, count=15):
        outlier_timestamp = []

        # 1. print
        for i in range(count):
            outlier = self._obj.iloc[self.outliers[i][0]]
            outlier_timestamp.append(outlier.name)

            print("time: ", outlier.name, "\tvalue: ", outlier[0], "\tscore: ", self.outliers[i][1])

        # 2. detect
        obj = deepcopy(self._obj)
        obj['outlier'] = False
        obj.loc[outlier_timestamp, 'outlier'] = True

        plt.figure(figsize=self.figsize)
        plt.scatter(obj.index, obj[obj.columns[0]], c=obj.outlier, label=obj.outlier)

    def delete_outlier(self, count=15):
        outlier_index_list = [self.outliers[:count][i][0] for i in range(count)]

        obj = deepcopy(self._obj)
        obj.loc[self._obj.index[outlier_index_list]] = None
        return obj

    # 4-8. holiday
    def plot_holiday(self, holiday=None):
        try:
            obj = pd.merge(self.obj.reset_index(), holiday, on='ds', how='left')
        except:
            raise TypeError("holiday should be a dataframe with 'ds', and 'holiday'")

        obj = obj.fillna(0)
        obj['factor'], _ = pd.factorize(obj['holiday'])

        plt.figure(figsize=self.figsize)
        plt.scatter(obj.ds.values, obj[obj.columns[1]].values,
                    c=obj['factor'].values)

    def split_holiday(self, holiday=None):
        try:
            obj = pd.merge(self.obj.reset_index(), holiday, on='ds', how='left')
        except:
            raise TypeError("holiday should be a dataframe with 'ds', and 'holiday'")

        normal_day = obj.loc[obj['holiday'].isnull() == True]
        holiday = obj.loc[obj['holiday'].isnull() == False]

        return normal_day[normal_day.columns[:2]].set_index('ds'), holiday[holiday.columns[:2]].set_index('ds')

    """
    def ruptures_change_point(self, model='l2', algorithm='BottomUp', breakpoint=2):
        # check trend change point
        if algorithm == 'BottomUp':
            algo = rpt.BottomUp(model=model).fit(self._obj.values)
        result_list = algo.predict(n_bkps=breakpoint)

        # 1. print
        for i, result in enumerate(result_list):
            if i == len(result_list) - 1:
                break

            print(self._obj.iloc[result-1])
            print(self._obj.iloc[result])

        # 2. plot
        rpt.display(self._obj.values, result_list, figsize=self.figsize)
    """

    """
    # 4-6. autocorreation
    def plot_autocorrelation(self):
        plt.figure(figsize=self._figsize)
        autocorrelation_plot(self._obj[self._obj.columns[0]])

    # 4-7. comparison plot
    def plot_comparison(self, code, time_grouper, how):

    # 5-1. trend change point

    # 5-2. outlier
    def plot_outlier(self):
        pass
    """


from sys import exit
def my_counter(name_file,column): # считает сколько какие значения встретились в файле
    count={}
    count['all']
    with open(name_file) as mfile:
        zag=mfile.readline()
        for st in mfile:
            stm=st.split('\t')
            count['all']+=1
            if not(stm[column] in list(count.keys())):
                count[stm[column]]=0
            else:
                count[stm[column]]+=1
    print('Results \n')
    for el in list(count.keys()):
        print('{}: {} | {} %'.format(el,count[el],str(count[el]/count['all']*100)))

def zagprint(name_file):
    with open(name_file) as mfile:
        st1 = mfile.readline().split('\t')
        st2 = mfile.readline().split('\t')
        if len(st1) == len(st2):
            print('Number of columns is correct \n')
        else:
            print('DANGER not all data with columns \n')
            print('1 ' + str(len(st1)))
            print('2 ' + str(len(st2)))

    with open(name_file) as mfile:
        zag = mfile.readline()
        first_str=mfile.readline()
        first_str=first_str.split('\t')
        zag=zag.split('\t')
        for j in range(len(zag)):
            print('{} [{}] | {}'.format(zag[j],j,first_str[j]))

def checkfile(name_file):
    try:
        m=open(name_file,'r')
        m.close()
    except(FileExistsError):
        print("File not found")
        exit()

# проверяет существование файла
def writting_file(name_file,name_file_out):
    print()
    print('Из файла '+name_file)
    print('Результаты записаны в файл '+name_file_out)
# Сообщение назавния выходного файла

def few_columns(name_file,columns,numberC):

    with open(name_file) as mfile:
        st1 = mfile.readline().split('\t')
        st2 = mfile.readline().split('\t')
        if len(st1) == len(st2):
            print('Number of columns is correct \n')
        else:
            print('DANGER not all data with columns \n')

    for num in columns:
        with open(name_file) as mfile:
            zag = mfile.readline().split('\t')
            j = 0
            print(str(zag[num] + ' | '),end='')
            while j < numberC:
                j+=1
                st=mfile.readline()
                stm=st.split('\t')
                print(stm[num]+'\t',end='')
        print()
# напечатает numberC строчек для колонок с номерами из списка columns из name_file


# Remove All Before

# Не все элементы важны. Вам нужно удалить из список все элементы до указаного.
#
# example
#
# На примере мы имеем список [1, 2, 3, 4, 5] где нужно было удалить все элементы до 3 - 1 и 2 соответственно.
#
# Есть два ньюанса: (1) если в списке нет элемента до которого нужно удалить остальные элементы,
# то список не должен измениться. (2) если list пустой, то он должен остаться пустым.
#
# Входные данные: Список и элемент до которого нужно удалить другие элементы.
#
# Выходные данные: Набор значений (кортеж, список, итератор ...).
#
# Пример:
#
# remove_all_before([1, 2, 3, 4, 5], 3) == [3, 4, 5]
# remove_all_before([1, 1, 2, 2, 3, 3], 2) == [2, 2, 3, 3]

def remove_all_before(items: list, border: int):
    if border not in items:
        return items
    else:
        cut = items.index(border)
    return items[cut:]

if __name__ == '__main__':
    print("Example:")
    print(list(remove_all_before([1, 2, 3, 4, 5], 3)))

    # These "asserts" are used for self-checking and not for an auto-testing
    assert list(remove_all_before([1, 2, 3, 4, 5], 3)) == [3, 4, 5]
    assert list(remove_all_before([1, 1, 2, 2, 3, 3], 2)) == [2, 2, 3, 3]
    assert list(remove_all_before([1, 1, 2, 4, 2, 3, 4], 2)) == [2, 4, 2, 3, 4]
    assert list(remove_all_before([1, 1, 5, 6, 7], 2)) == [1, 1, 5, 6, 7]
    assert list(remove_all_before([], 0)) == []
    assert list(remove_all_before([7, 7, 7, 7, 7, 7, 7, 7, 7], 7)) == [7, 7, 7, 7, 7, 7, 7, 7, 7]
    print("Coding complete? Click 'Check' to earn cool rewards!")

'''
Альтернативные решения:

№ 1
def remove_all_before(items, border):
    try:
        return items[items.index(border):]
    except ValueError:
        return items

№ 2
def remove_all_before(items: list, border: int) -> Iterable:
    return items[items.index(border):] if border in items else items

№ 3
remove_all_before=lambda i,b:b in i and i[i.index(b):] or i
'''


from typing import List

def canFinish(numCourses: int, prerequisites: List[List[int]]) -> bool:
    mapLink = {i: [] for i in range(numCourses)}

    for courses, coursesNeeded in prerequisites:
        mapLink[courses].append(coursesNeeded)

    thatsOkay = set()

    def dfs(i):
        if mapLink[i] == []:
            return True
        elif i in thatsOkay:
            return False

        thatsOkay.add(i)
        for courseNeeded in mapLink[i]:
            if not dfs(courseNeeded):
                return False
        thatsOkay.remove(i)
        # preMap[i] = []
        return True

    for i in range(numCourses):
        if not dfs(i):
            return False
    return True


import math

record = float(input())
distance = float(input())
time_1m = float(input())

ivan = distance * time_1m
delay = math.floor(distance / 15) * 12.5
total = ivan + delay
slower = abs(record - total)
if record > total:
    print(f'Yes, he succeeded! The new world record is {total:.2f} seconds.')
else:
    print(f'No, he failed! He was {slower:.2f} seconds slower.')


from datetime import datetime

import jwt
from django.conf import settings
from jwt import DecodeError
from slqe.model.models import User
import time

def is_verified(token):
    try:
        payload = jwt.decode(token, settings.SECRET_KEY, algorithms='HS256')
        verify_token = jwt.encode({
            'id': payload['id'],
            'exp': payload['exp']
        }, settings.SECRET_KEY, algorithm='HS256')
        User.objects.get(id=payload['id'], is_active=True)
        dt = datetime.now()
        expired = time.mktime(dt.timetuple())
        if token != verify_token or int(expired) > payload['exp']:
            return False
    except User.DoesNotExist:
        return False
    except ValueError:
        return False
    except TypeError:
        return False
    except DecodeError:
        return False
    return True

def is_permitted(token, roles):
    try:
        payload = jwt.decode(token, settings.SECRET_KEY, algorithms='HS256')
        user = User.objects.get(id=payload['id'], is_active=True)

        if user.role.name in roles:
            return True
    except User.DoesNotExist:
        return False
    except ValueError:
        return False
    except TypeError:
        return False
    except DecodeError:
        return False
    return False

class JWTAuthentication:
    pass


#!/bin/python3

import math
import os
import random
import re
import sys

def insertionSort(arr):
    shifts=0
    arrindex=[0]*(max(arr)+1)
    count=0
    currmin=arr[0]
    currmax=arr[0]
    for x,y in enumerate(arr):
        if y<currmin:
            currmin=y
            shifts+=count
            count+=1
            arrindex[y]+=1
            print('1here',shifts)
            continue
        if y>currmax:
            currmax=y
            count+=1
            arrindex[y]+=1
            continue

        if (y-currmin)<(currmax-y):
            smallercount=0
            print('currmin',currmin, y)
            for x in arrindex[currmin:y]:
                smallercount+=x
            shifts+=(count-smallercount)
            print('here c' ,shifts)
        else:
            for x in arrindex[y+1:currmax+1]:
                print('here test',x)
                shifts+=x
        print(shifts)
        arrindex[y]+=1
        count+=1

    return shifts

print(insertionSort([2,1,3,1,2]))


'''
@Author: 风满楼
@Date: 2020-04-23 17:11:14
@LastEditTime: 2020-04-26 16:45:56
@LastEditors: Please set LastEditors
@Description: 处理输入的类和功能函数
@FilePath: /frame_sort/models/input.py
'''

from tensorflow.keras.layers import Input, Embedding, Dense, RepeatVector

class SparseClass():
    '''
    @description: 处理离散数据的类
    @param {
        feat_name: 特征的名字
        vocablary_size: 特征中不重复的个数
        embedding_dim: embedding后向量的维度
    }
    @return:
    '''
    def __init__(self, feat_name, vocablary_size, embedding_dim=5):
        self.feat_name = feat_name
        self.vocablary_size = vocablary_size
        self.embedding_dim = 5

class DenseClass():
    '''
    @description: 处理连续数据的类
    @param {
        feat_name: 特征的名字
        embedding_dim:embedding后向量的维度
    }
    @return:
    '''
    def __init__(self, feat_name, embedding_dim=5):
        self.feat_name = feat_name
        self.embedding_dim = 5

def get_input_layer(all_input, embedding=False):
    sparse_layer_list = []
    dense_layer_list = []
    for item in all_input:
        if isinstance(item, SparseClass):
            sparse_layer_list.append(Input(shape=(1,), name=item.feat_name))
        elif isinstance(item, DenseClass):
            dense_layer_list.append(Input(shape=(1,), name=item.feat_name))
    return sparse_layer_list, dense_layer_list

def get_embedding_layer(all_input_layers, all_input_column):
    '''
    @description: 这个函数真是丑陋，我无法忍受
    @param {type}
    @return:
    '''
    sparse_embedding_layers = []
    dense_embedding_layers = []
    for item in all_input_layers:
        for item_class in all_input_column:
            if isinstance(item_class, SparseClass):
                if item.name.split(":")[0] == item_class.feat_name:
                    sparse_embedding_layers.append(Embedding(item_class.vocablary_size,item_class.embedding_dim)(item))
            if isinstance(item_class, DenseClass):
                if item.name.split(":")[0] == item_class.feat_name:
                    dense_embedding_layers.append(
                        RepeatVector(1)(Dense(item_class.embedding_dim)(item))
                    )
    return sparse_embedding_layers, dense_embedding_layers


# Python program to read JSON from a file.
import json

# Opening JSON file
with open('sample.json', 'r') as openfile:

	# Reading from json file
	json_object = json.load(openfile)

print(json_object)
print(type(json_object))


from . import _sfac
import numpy as np
from collections import namedtuple

StructureFactors = namedtuple("StructureFactors", "hkl q q_mag values normalization")
Reflections = namedtuple("Reflection", "q q_mag hkl")

LAMBDA_Cu = 1.54059

UNIQUE_REFLECTION_MULTIPLICITY = {
    ("triclinic", "-1"): 2,
    ("monoclinic", "2/m"): 4,
    ("orthorhombic", "mmm"): 8,
    ("tetragonal", "4/m"): 8,
    ("tetragonal", "4/mmm"): 16,
    ("cubic", "m-3"): 24,
    ("cubic", "m-3m"): 48,
}

UNIQUE_REFLECTION_TYPES = {
    ("-1", "*"): (
        ((0, 0, 0), ((1, 0, 0), (0, 1, 0), (0, 0, 1))),
        ((-1, 0, 1), ((-1, 0, 0), (0, 1, 0), (0, 0, 1))),
        ((-1, 1, 0), ((-1, 0, 0), (0, 1, 0), (0, 0, -1))),
        ((0, 1, -1), ((1, 0, 0), (0, 1, 0), (0, 0, -1))),
    ),
    ("2/m", "aface"): (
        ((0, 0, 0), ((0, 1, 0), (1, 0, 0), (0, 0, 1))),
        ((0, -1, 1), ((0, -1, 0), (1, 0, 0), (0, 0, 1))),
    ),
    ("2/m", "bface"): (
        ((0, 0, 0), ((1, 0, 0), (0, 1, 0), (0, 0, 1))),
        ((-1, 0, 1), ((-1, 0, 0), (0, 1, 0), (0, 0, 1))),
    ),
    ("2/m", "cface"): (
        ((0, 0, 0), ((1, 0, 0), (0, 0, 1), (0, 1, 0))),
        ((-1, 1, 0), ((-1, 0, 0), (0, 0, 1), (0, 1, 0))),
    ),
    ("mmm", "*"): (((0, 0, 0), ((1, 0, 0), (0, 1, 0), (0, 0, 1))),),
    ("4/mmm", "*"): (((0, 0, 0), ((1, 0, 0), (1, 1, 0), (0, 0, 1))),),
    ("4/m", "*"): (
        ((0, 0, 0), ((1, 0, 0), (1, 1, 0), (0, 0, 1))),
        ((1, 2, 0), ((1, 1, 0), (0, 1, 0), (0, 0, 1))),
    ),
    ("6/mmm", "*"): (((0, 0, 0), ((1, 0, 0), (1, 1, 0), (0, 0, 1))),),
    ("6/m", "*"): (
        ((0, 0, 0), ((1, 0, 0), (1, 1, 0), (0, 0, 1))),
        ((1, 2, 0), ((0, 1, 0), (1, 1, 0), (0, 0, 1))),
    ),
    ("-3m1", "*"): (
        ((0, 0, 0), ((1, 0, 0), (1, 1, 0), (0, 0, 1))),
        ((0, 1, 1), ((0, 1, 0), (1, 1, 0), (0, 0, 1))),
    ),
    ("-31m", "*"): (
        ((0, 0, 0), ((1, 0, 0), (1, 1, 0), (0, 0, 1))),
        ((1, 1, -1), ((1, 0, 0), (1, 1, 0), (0, 0, -1))),
    ),
    ("-3m", "*"): (
        ((0, 0, 0), ((1, 0, 0), (1, 0, -1), (1, 1, 1))),
        ((1, 1, 0), ((1, 0, -1), (0, 0, -1), (1, 1, 1))),
    ),
    ("-3", "*"): (
        ((0, 0, 0), ((1, 0, 0), (1, 0, -1), (1, 1, 1))),
        ((1, 1, 0), ((1, 0, -1), (0, 0, -1), (1, 1, 1))),
        ((0, -1, -2), ((1, 0, 0), (1, 0, -1), (-1, -1, -1))),
        ((1, 0, -2), ((1, 0, -1), (0, 0, -1), (-1, -1, -1))),
    ),
    ("m3m", "*"): (((0, 0, 0), ((1, 0, 0), (1, 1, 0), (1, 1, 1))),),
    ("m3", "*"): (
        ((0, 0, 0), ((1, 0, 0), (1, 1, 0), (1, 1, 1))),
        ((1, 2, 0), ((0, 1, 0), (1, 1, 0), (1, 1, 1))),
    ),
}

def hklmax(uc, dmin):
    return (np.array(uc.lengths) / dmin).astype(int)

def reflections(crystal, wavelength=LAMBDA_Cu, dmin=LAMBDA_Cu / 2, sort=True):
    """Calculate the unique reflections contributing to
    diffraction for the given crystal, along with the q
    vector at the provided wavelength"""
    h_max, k_max, l_max = hklmax(crystal.unit_cell, dmin)
    recip = crystal.unit_cell.reciprocal_lattice.copy()
    system = crystal.space_group.crystal_system
    centering = (
        "*" if system != "monoclinic" else crystal.space_group.choice[0] + "face"
    )
    laue_class = crystal.space_group.laue_class
    if (
        crystal.space_group.has_hexagonal_rhombohedral_choices()
        and crystal.unit_cell.is_rhombohedral
    ):
        raise NotImplementedError("Rhombohedral crystals not currently supported")

    #   apexes, bases = zip(*UNIQUE_REFLECTION_TYPES[(laue_class, centering)])
    #   h_range = np.r_[0: h_max + 1]
    #   k_range = np.r_[0: k_max + 1]
    #   l_range = np.r_[0: l_max + 1]
    #   sections = []
    #   for a, b in zip(apexes, bases):
    #       v1 = np.r_[b[0][0], b[0][1], b[0][2]]
    #       v2 = np.r_[b[1][0], b[1][1], b[1][2]]
    #       v3 = np.r_[b[2][0], b[2][1], b[2][2]]
    #       hkl = v1 * h_range[:, None]
    #       hkl = hkl + (v2 * k_range[:, None])[:, None]
    #       hkl = hkl + (v3 * l_range[:, None, None])[:, None, None]
    #       hkl = hkl.reshape(-1, 3)
    #       hkl += a
    #       sections.append(hkl)
    #   hkl = np.vstack(sections)
    h, k, l = np.mgrid[-h_max:h_max, -k_max:k_max, -l_max:l_max].astype(np.int32)
    hkl = np.c_[h.ravel(), k.ravel(), l.ravel()]
    G = hkl @ recip
    q = np.linalg.norm(G, axis=1)
    mask = q <= (2 / wavelength)
    if sort:
        G = G[mask]
        q = q[mask]
        hkl = hkl[mask]
        order = np.argsort(q)
        return Reflections(G[order], q[order], hkl[order])

    return Reflections(G[mask], q[mask], hkl[mask])

def powder_pattern(crystal, wavelength=LAMBDA_Cu, two_theta_range=(5, 50)):
    """Calculate the powder pattern the given crystal at a given wavelength
    over a given range of 2-theta"""
    sfac = structure_factors(crystal, wavelength=wavelength)
    hkl, G, q, sfac, norm = sfac
    f2 = np.abs(sfac)
    f2 = f2 * f2 / norm
    theta = np.arcsin(wavelength * q / 2)
    two_theta = 2 * theta
    l, u = np.radians(two_theta_range)
    l = max(l, 1e-3)
    mask = (two_theta <= u) & (two_theta >= l)
    t = theta[mask]
    tt = two_theta[mask]
    i = f2[mask]
    costt = np.cos(tt)
    sintt = np.sin(tt)
    scaled_intensities = i * (1 + costt * costt) / (8 * sintt * sintt * np.cos(t))
    return np.degrees(tt), scaled_intensities

def structure_factors(crystal, wavelength=LAMBDA_Cu):
    """Calculate the structure factors for a given crystal at the provided
    wavelength"""
    G, q, hkl = reflections(crystal, wavelength=wavelength)
    facs = _sfac.calculate_unique_plane_factors(hkl, q)
    keep = facs > 0
    facs = facs[keep]
    G = G[keep]
    q = q[keep]
    hkl = hkl[keep]
    uc = crystal.unit_cell_atoms()
    asym_sfac_idx = [
        _sfac.get_form_factor_index(el.symbol)
        for el in crystal.asymmetric_unit.elements
    ]
    indices = [asym_sfac_idx[i] for i in uc["asym_atom"]]
    N = len(indices)
    unique_indices = set(asym_sfac_idx)
    sintl = q / 2
    sintl2 = sintl * sintl
    scattering_factors = {
        i: _sfac.scattering_factors(i, sintl2) for i in unique_indices
    }
    frac_pos = uc["frac_pos"]
    exp_fac = np.zeros(hkl.shape[0], dtype=np.complex128)
    exp_fac.imag = 2 * np.pi
    sfac = np.zeros_like(exp_fac)
    normalization = np.zeros_like(q)
    for i, idx in enumerate(indices):
        fj = scattering_factors[idx]
        pos = frac_pos[i]
        hkl_dot_pos = hkl @ pos
        sfac += fj * np.exp(exp_fac * hkl_dot_pos)
        normalization += fj * fj
    f000 = sfac[0]
    sfac *= facs
    return StructureFactors(hkl, G, q, sfac, normalization)


import os   #Required to make os calls to open applications
from getFileFromUser import getFileFromUser #Imports method to get filename from user

[fileName, fileDesc, fileExt] = getFileFromUser()

#Declaring functions for each open method, tiered for .txt's
def openTxt():
    value = os.system("open -b com.microsoft.Word " + fileName)
    if value != 0:
        value = os.system("open -a TextEdit " + fileName)
    if value != 0:
        value = os.system("open -a Notes " + fileName)

def openMp3():
    os.system("open -a 'quicktime player' " + fileName)

def openJpeg():
    os.system("open -a Preview " + fileName)

def openCsv():
    os.system("open -a Atom " + fileName)

#Dispatch table associating file extensions with methods to open them
dispatchFileOpenByExt = {
  'txt': openTxt,
  'mp3': openMp3,
  'jpeg': openJpeg,
  'csv': openCsv,
}

#Pass fileExt into dispatch table to trigger corresponding application execution
try:
    dispatchFileOpenByExt[fileExt]()
except KeyError:
    print("\nUnsupported file type!!!")


import random
import gensim
from utils import add_noise
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
import pickle

class Guesser(object):
    """
    player (not spymaster) class.
    :param w2v_path: path for pretrained embeddings
    :param field: instances of Field
    :param logger:
    """

    def __init__(self, field, embeddings_dict, logger):

        self.field = field
        self.wv = embeddings_dict
        self.logger = logger

    def guess_from_clue(self, clue):
        """
        given a clue and number from spymaster, calculates all the similarity for each cards in the field.
        :param clue: string, clue of the spumaster
        :return: full ranked list of (card, score) pairs.
        """
        sorted_card_score_pairs = []
        for card in self.field:
            if card.taken_by == "None" and clue in self.wv:
                w_vec = self.wv[clue].reshape(-1, len(self.wv[clue]))
                c_vec = self.wv[card.name].reshape(-1, len(self.wv[card.name]))
                similarity = np.asscalar(cosine_similarity(w_vec, c_vec))
                card_score_pair = (card, similarity)
            elif card.taken_by == "None" and clue not in self.wv:
                self.logger.info("clue: {} is not included in guesser embedding. similarity is set to 0.0".format(clue))
                card_score_pair = (card, 0.0)
            else:
                continue
            sorted_card_score_pairs.append(card_score_pair)
        sorted_card_score_pairs = sorted(sorted_card_score_pairs, key=lambda x: x[1], reverse=True)

        self.logger.info("Guesser ranking for all not-taken cards:")
        for card in sorted_card_score_pairs:
            print_text = "{} {} {}".format(card[0].name, card[1], card[0].color)
            self.logger.info(print_text)

        return sorted_card_score_pairs


#Will continue tomorrow at '8.4 Skriv transaktioner till filen'
# Planning :
# Imports
from functions import *

# Functions
check_file_exists()
read_file()

# Variabler

# Main loop
# Input of text
# Write it in a file
# Read the file
# Complete the program

while True:
    meny = ("\n#######################"
            "\n# Lilla banken"
            "\n# Meny"
            "\n# Saldo: {} kr"
            "\n#######################"
            "\n1. Visa saldo"
            "\n2. Gör en insättning"
            "\n3. Gör ett uttag"
            "\n4. Nollställ kontot"
            "\n0. Avsluta programmet"
            "\nGör ditt val: ".format(balance()))

    val = validate_int(meny, "Felaktig inmatning !")

# Val 0
    if val == 0:
        break

#Val 1
    elif val == 1:
        print(print_transactions())

# Val 2
    elif val == 2:
        deposit = validate_int("Ange hur mycket du vill sätta in: ", "Felaktig inmatning !")
        if deposit > 0:
            add_transaction(deposit, True)
            print("Tack för din insättning på {}kr".format(deposit))

        else:
            print("En insättning måste vara större än 0.")

# Val 3
    elif val == 3:
        withdraw = validate_int("Ange hur mycket du vill ta ut: ", "Felaktig inmatning !")
        if withdraw <= balance() and withdraw >= 0:
            add_transaction(-withdraw, True)
            print("Uttag goodkänt! Nytt saldo {}".format(balance()))

        elif withdraw < 0:
            print("Uttaget måste vara större än 0.")

        else:
            print("Uttaget får inte vara större än saldot. Uttag medgs ej")

# Val 4
    elif val == 4:
        os.remove(filename)                      # Tar bort filen
        transactions.clear()                     # Töm listan
        read_file()                              # Skapa filen och läs in den

    else:
        print("Felaktigt val !")

print("Tack för ditt besök, välkommen åter!")


"""Integration Status

Revision ID: dac99ae0c6fd
Revises: 0c2fe32b5649
Create Date: 2023-04-12 06:58:21.560292+00:00

"""
import sqlalchemy as sa

from alembic import op

# revision identifiers, used by Alembic.
revision = "dac99ae0c6fd"
down_revision = "0c2fe32b5649"
branch_labels = None
depends_on = None

status_enum = sa.Enum("green", "red", name="external_integration_status")

def upgrade() -> None:
    # ### commands auto generated by Alembic ###
    op.create_table(
        "externalintegrationerrors",
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("time", sa.DateTime(), nullable=True),
        sa.Column("error", sa.Unicode(), nullable=True),
        sa.Column("external_integration_id", sa.Integer(), nullable=True),
        sa.ForeignKeyConstraint(
            ["external_integration_id"],
            ["externalintegrations.id"],
            name="fk_error_externalintegrations_id",
            ondelete="CASCADE",
        ),
        sa.PrimaryKeyConstraint("id"),
    )

    status_enum.create(op.get_bind())
    op.add_column(
        "externalintegrations",
        sa.Column(
            "status",
            status_enum,
            server_default="green",
            nullable=True,
        ),
    )
    op.add_column(
        "externalintegrations",
        sa.Column("last_status_update", sa.DateTime(), nullable=True),
    )
    # ### end Alembic commands ###

def downgrade() -> None:
    # ### commands auto generated by Alembic ###
    op.drop_table("externalintegrationerrors")
    op.drop_column("externalintegrations", "last_status_update")
    op.drop_column("externalintegrations", "status")
    status_enum.drop(op.get_bind())
    # ### end Alembic commands ###