import math
#import random
#from pyrr import Matrix44, Quaternion, Vector3, vector

import moderngl as mgl           #https://moderngl-window.readthedocs.io/en/latest/
import moderngl_window as mglw   #https://github.com/moderngl/moderngl/blob/master/examples/multiple_instance_rendering.py
from moderngl_window.conf import settings
from moderngl_window.timers.clock import Timer
from moderngl_window.utils.scheduler import Scheduler

from numpy.random import default_rng
#from sys import intern
import numpy as np
#import numba
from PIL import Image

SIZE_X = 800
SIZE_Y = 600

START = -6400
END= 6400
STEP = 32
ROW = END // STEP
SHRINK = 32

#@numba.jit(fastmath=True, parallel=True)
def tree_map():
    t_arr = np.array([])
    rng = default_rng()
    x_offset_spawn = y_offset_spawn = 0
    for y in range(START // STEP,END // STEP):
    #for y in numba.prange(START // STEP,END // STEP):
        rand_src = rng.standard_normal(END * 2 // STEP)
        for x in range(START // STEP,END // STEP):
        #for x in numba.prange(START // STEP,END // STEP):
            if rand_src[x + (END // STEP)] > 1.75 and (x_offset_spawn <= 0 and y_offset_spawn <= 0):
                t_arr = np.append((x * (STEP / SHRINK), (y * (STEP / SHRINK)), 2.0),t_arr).astype(dtype="f4")

    return t_arr
#@numba.jit(fastmath=True, parallel=True)
def get_map_arr():
    n_arr_x = np.divide(np.arange(START, END, STEP, dtype="f4"),SHRINK)
    n_zeros = np.ones((END*2)//STEP,dtype="f4")
    n_arr = np.column_stack((n_arr_x,n_zeros))
    n_arr_big = None

    #for x in numba.prange(-ROW,ROW):
    for x in range(-ROW,ROW):
        n_arr_big = np.append(n_arr_big,np.hstack((np.full((ROW*2,1),x * (STEP / SHRINK)),n_arr))).astype("f4")

    n_arr_big = np.append(n_arr_big,tree_map()).astype(dtype="f4")
    ###### player, de to første floats er bare dummies(could they be something else like direction/animation?)
    #n_arr_big = np.append(n_arr_big,(0.0,0.0,0.0)).astype(dtype="f4")

    return np.delete(n_arr_big,0)#np.append(np.array([0.0,0.0,0.0]),np.delete(n_arr_big,0))

MAP_COORDS = get_map_arr()
INSTANCE_AMOUNT = (len(MAP_COORDS) + 3) // 3
#conver til å dekke hele altså fra -1 til 1?
VERTECIES = np.array([
    #vertex,    texture-coord
    0.5, 0.5,   #1.0, 1.0,
    0.5, -0.5,  #1.0, -1.0,
    -0.5, -0.5, #-1.0, -1.0,
    -0.5, 0.5,  #-1.0, 1.0,
    ])
RENDER_INDICIES = np.array([
    0, 1, 3,
    1, 2, 3
    ])
#TREE_PIC = Image.open("C:\\Users\\yo\\Desktop\\devvings\\assets\\rpgsheet.png")
TREE_PIC = Image.open("./rpgsheet.bmp")
TREE_PIC_TEXTURE = TREE_PIC.crop((8*STEP,1*STEP,9*STEP,2*STEP)).transpose(Image.FLIP_TOP_BOTTOM).convert('RGBA')
PIC = Image.open("./brahladin.bmp")
PIC = PIC.transpose(Image.FLIP_TOP_BOTTOM).convert('RGBA')
OTHER_PIC = Image.open("./gr33n.bmp")
OTHER_PIC = OTHER_PIC.transpose(Image.FLIP_TOP_BOTTOM).convert('RGBA')

class CustomSetup:
    #@numba.jit()
    def __init__(self):

        # Configure to use pyglet window
        settings.WINDOW['class'] = 'moderngl_window.context.pyglet.Window'
        settings.WINDOW['gl_version'] = (4, 0)
        settings.WINDOW['size'] = (SIZE_X,SIZE_Y)
        settings.WINDOW['aspect_ratio'] = SIZE_X / SIZE_Y
        settings.WINDOW['title'] = "__<=<=<´-´>=>=>__/´/\,,,,,,...###__<\¤(=(¤<\__###...,,,,,,/´/\__<=<=<´-´>=>=>__"
        settings.WINDOW['clear_color'] = (0)
        settings.WINDOW['vsync'] = False
        settings.WINDOW['log_level'] = None

        self.X_POINTS = 0
        self.Y_POINTS = 0
        self.wnd = mglw.create_window_from_settings()
        self.wnd.key_event_func = self.key_event
        self.ctx = self.wnd.ctx
        self.ctx.wireframe = False
        self.ctx.enable(self.ctx.BLEND | self.ctx.CULL_FACE)
        self.ctx.cull_face = 'front'
        #self.ctx.multisample = True
        #self.ctx.blend_func = mgl.SRC_ALPHA, mgl.ONE_MINUS_SRC_ALPHA

        self.prog = self.ctx.program(
           vertex_shader="""
                #version 330

                in vec2 in_pos;
                in float in_texid;
                in vec2 in_vert;
                uniform vec2 pos;

                out vec3 out_texinfo;

                void main() {
                    gl_Position = vec4(pos + (in_vert + in_pos), 0.0, 4.0);
                    out_texinfo = vec3(in_vert,in_texid);
                }
            """
            ,
            fragment_shader="""
                #version 330

                uniform sampler2DArray Texture;

                in vec3 out_texinfo;
                out vec4 outColor;

                void main() {
                    outColor = texture(Texture, vec3(out_texinfo.xy +0.5, out_texinfo.z));
            }
            """,
        )

        """
            x og y til np-array
            bit-degree-dict for movement
            så gang degrees * frame_time og add det til x og y
            null X_POINTS

            time som self-var? i np arr for speed?
        """
        self.P_COORDS_X = 0.0
        self.P_COORDS_Y = 0.0
        self.prog['pos'] = (self.P_COORDS_X, self.P_COORDS_Y)

        """
        self.g_pos = self.prog['GameIn']
        self.g_pos.binding = 0

        self.toGpuBuffer = self.ctx.buffer(reserve=self.g_pos.size)
        self.toGpuBuffer.bind_to_uniform_block(0)
        self.toGpuBuffer.write(np.array([0,0]).tobytes())
        """

        images = [
            PIC,
            OTHER_PIC,
            TREE_PIC_TEXTURE,
        ]

        depth = len(images)
        dataList = []

        for image in images:
            if 32 != image.size[0] or 32 != image.size[1]:
                raise ValueError(f"image size mismatch: {image.size[0]}x{image.size[1]}")

            dataList.append(list(image.getdata()))

        imageArrayData = np.array(dataList, np.uint8).tobytes()
        components = 4

        self.tex = self.ctx.texture_array((32, 32, depth),components, imageArrayData, alignment=1)
        self.prog["Texture"] = 0
       # self.tex.build_mipmaps()
       # self.tex.filter=(mgl.NEAREST,mgl.NEAREST)
        self.tex.repeat_x = self.tex.repeat_y = False
        self.tex.use(location=0)

        self.vertex_buffer = self.ctx.buffer(VERTECIES.astype("f4").tobytes())
        self.pos_buffer = self.ctx.buffer(MAP_COORDS.astype("f4").tobytes())
        self.index_buffer = self.ctx.buffer(RENDER_INDICIES.astype("i4").tobytes())
        self.actor_vertex = self.ctx.buffer(
                    np.array([
                        -self.P_COORDS_X,
                        -self.P_COORDS_Y,
                        0.0
                        ]).astype("f4").tobytes(), dynamic=True)

        self.vao_content = [
            (self.vertex_buffer, '2f', 'in_vert'),
            (self.pos_buffer, '2f 1f/i', 'in_pos', "in_texid"),
        ]

        self.vao = self.ctx.vertex_array(self.prog, self.vao_content, self.index_buffer)

        self.t = self.ctx.vertex_array(
            self.prog,[
                (self.vertex_buffer, '2f', 'in_vert'),
                (self.actor_vertex, '2f 1f/i',"in_pos","in_texid"),
                ],
            self.index_buffer,)

    #self.toGpuBuffer.write(self.P_COORDS.astype(dtype=np.float32).tobytes())
    #

    def run(self):
        self.timer = Timer()
        self.timer.start()
        self.last_frame = 0
        self.update_delay = 1 / 60  # updates per second
        self.last_updated = 0
        #self.acc = 1.6
        #self.fric = 4.8

        while not self.wnd.is_closing:
            self.wnd.clear()
            self.time, self.last_frame = self.timer.next_frame()

            # First VAO, contains static map-data
            self.prog["pos"] = (self.P_COORDS_X, self.P_COORDS_Y)
            self.P_COORDS_X += self.X_POINTS
            self.P_COORDS_Y += self.Y_POINTS
            self.vao.render(instances=INSTANCE_AMOUNT)

            # second VAO contains moving data
            self.actor_vertex.write(
                np.array([
                    -self.P_COORDS_X,
                    -self.P_COORDS_Y,
                    0.0
                    ]).astype("f4").tobytes()
                )
            self.t.render()
            self.actor_vertex.orphan(self.actor_vertex.size)

            if self.time - self.last_updated > self.update_delay:
                self.last_updated = self.time

            self.wnd.swap_buffers()

        self.wnd.destroy()

    def key_event(self, key, action, modifiers):
        if action == self.wnd.keys.ACTION_RELEASE:
            if key == self.wnd.keys.W:
                self.Y_POINTS = 0
            if key == self.wnd.keys.A:
                self.X_POINTS = 0
            if key == self.wnd.keys.S:
                self.Y_POINTS = 0
            if key == self.wnd.keys.D:
                self.X_POINTS = 0

        elif action == self.wnd.keys.ACTION_PRESS:
            if key == self.wnd.keys.A:
                self.X_POINTS += self.last_frame
            if key == self.wnd.keys.S:
                self.Y_POINTS += self.last_frame
            if key == self.wnd.keys.D:
                self.X_POINTS -= self.last_frame
            if key == self.wnd.keys.W:
                self.Y_POINTS -= self.last_frame

if __name__ == '__main__':
    app = CustomSetup()
    app.run()


"""
Cross Tables and Scatter Plots Exercise
"""
from os.path import dirname, realpath
from numpy import arange
from pandas import DataFrame, read_excel
from matplotlib.pyplot import show, subplots

PATH = dirname(realpath(__file__))

def crosstable():
    """
    wrapper function
    """
    cross = DataFrame(
        read_excel(
            io=f"{PATH}/cross_table_scatter_plot.xlsx",
            sheet_name="Cross table",
        )
    )
    cross_data = DataFrame(cross.iloc[17:23, 1:4].values, columns=cross.iloc[16, 1:4])
    print(cross_data)
    sca = DataFrame(
        read_excel(
            io=f"{PATH}/cross_table_scatter_plot.xlsx",
            sheet_name="Scatter plot",
        )
    )
    sca_data = DataFrame(sca.iloc[11:115, 2:5].values, columns=sca.iloc[10, 2:5])
    print(sca_data)

    labels, employed, unemployed = (cross_data.iloc[:, i] for i in range(3))
    xaxis = arange(len(labels))
    width = 0.35
    _, axis = subplots()
    rects1 = axis.bar(xaxis - width / 2, employed, width, label=cross_data.columns[1])
    rects2 = axis.bar(xaxis + width / 2, unemployed, width, label=cross_data.columns[2])
    axis.set_ylabel("Percentage %")
    axis.set_title("Employment by age group")
    axis.set_xticks(xaxis)
    axis.set_xticklabels(labels)
    axis.legend()
    axis.bar_label(rects1, padding=3)
    axis.bar_label(rects2, padding=3)

    aapl, googl, bac = (sca_data.iloc[:, i] for i in range(3))
    _, axis1 = subplots()
    axis1.scatter(aapl, googl)
    axis1.set_title("aapl, googl")
    _, axis2 = subplots()
    axis2.scatter(bac, googl)
    axis2.set_title("bac, googl")
    _, axis3 = subplots()
    axis3.scatter(aapl, bac)
    axis3.set_title("aapl, bac")

if __name__ == "__main__":
    crosstable()
    show()


import json
import pickle
import time

from pathlib import Path

import torch
import torchvision
from functools import partial
import numpy as np
import matplotlib
from matplotlib import pyplot as plt
plt.style.use('seaborn-paper')
import seaborn as sns
sns.set_context("paper")
# import scienceplots
# plt.style.use(['science','no-latex'])
from util.util import *
from scipy.integrate import simps
import scipy.stats

from sklearn.metrics import confusion_matrix
import pandas as pd
try:
    from load_model import eval_trained_model, load_trained_model
    import datasets
    from util.dicts import imagenet_dict
except:
    from multi_task.load_model import eval_trained_model, load_trained_model
    from multi_task import datasets
    from multi_task.util.dicts import imagenet_dict

# import torch.backends.cudnn as cudnn
# cudnn.benchmark = True
# cudnn.enabled = True

imagenet_dict_reversed = dict(zip(imagenet_dict.values(), imagenet_dict.keys()))

if_pretrained_imagenet = True
# np.warnings.filterwarnings('error', category=np.VisibleDeprecationWarning)

def calc_l1_norms_of_weights(save_model_path):
    if not if_pretrained_imagenet:
        model_name_short = save_model_path[37:53] + '...' + save_model_path[-12:-10]
        state = torch.load(save_model_path)
        model_rep_state = state['model_rep']
    else:
        model_name_short = 'pretrained_imagenet'
        model = torchvision.models.__dict__['resnet18'](pretrained=True).cuda()
        model_rep_state = model.state_dict()
    layers_to_neurons_to_l1_norms = {}
    for k in model_rep_state.keys():
        if 'conv' not in k:
            continue
        cur_dict = {}
        for i in range(model_rep_state[k].shape[0]):
            cur_dict[i] = torch.norm(model_rep_state[k][i], 1).item()
        layers_to_neurons_to_l1_norms[k] = cur_dict
    np.save(f'l1_norms_{model_name_short}.npy', layers_to_neurons_to_l1_norms, allow_pickle=True)

def reload_model(save_model_path, param_file):
    if not if_pretrained_imagenet:
        try:
            trained_model = load_trained_model(param_file, save_model_path)
        except:
            print('assume problem where cifar networks ignored the enable_bias parameter')
            trained_model = load_trained_model(param_file, save_model_path,
                                               if_actively_disable_bias=True)
    else:
        trained_model = torchvision.models.__dict__['resnet18'](pretrained=True).cuda()
    return trained_model

def ablate_neurons(model, layer_ind, neurons_to_ablate):
    target_layer_name = layers_bn[layer_ind].replace('_', '.')

    # model['rep'].connectivities[layer_ind][:, neurons_to_ablate] = 0
    def save_activation(name, mod, inp, out):
        if name == target_layer_name:
            out[:, neurons_to_ablate, :, :] = 0
            return out

    hooks = []
    model_true = model
    if not if_pretrained_imagenet:
        model_true = model['rep']
    for name, m in model_true.named_modules():
        hooks.append(m.register_forward_hook(partial(save_activation, name)))
    return hooks

def calc_accuracies_after_ablation(param_file, model, layer_ind, neurons_sorted, step_size, n_steps, n_start):
    accs = []
    loader = None
    start_hooks = ablate_neurons(model, layer_ind, neurons_sorted[:n_start])
    for i in range(n_steps + 1):
        if n_start + i * step_size > len(neurons_sorted):
            print(i, n_start + i * step_size, len(neurons_sorted))
            break
        hooks = ablate_neurons(model, layer_ind, neurons_sorted[n_start:n_start + i * step_size])
        errror, loader = eval_trained_model(param_file, model, loader=loader, if_print=False, if_pretrained_imagenet=if_pretrained_imagenet)
        acc_cur = 1 - errror
        accs.append(acc_cur)
        for hook in hooks:
            hook.remove()
    print(accs)
    for hook in start_hooks:
        hook.remove()
    return np.array(accs)

def select_neurons_to_ablate_by_wasserstein_dist(data_per_neuron, selection_fun=lambda vals: np.sum(np.abs(vals))):
    # wasser_dists = np.load(wd_path, allow_pickle=True).item()
    # total_wd_per_neuron = [selection_fun(list(neuron_wds.values())) for neuron_wds in wasser_dists.values()]
    total_wd_per_neuron = [selection_fun(neuron_data) for neuron_data in data_per_neuron]
    idx = np.argsort(total_wd_per_neuron)[::-1]
    return np.array(idx)

def compare_ablation_strategies(save_model_path, param_file, step_size, n_steps, layer_ind, n_start,
                                selections_funs, folder_name):
    Path(f'ablations/{folder_name}').mkdir(exist_ok=True)
    model = reload_model(save_model_path, param_file)
    for (fun, fun_name, data_per_neuron) in selections_funs:
        print(fun_name)
        # model = reload_model(save_model_path, param_file)
        sorted_neurons = select_neurons_to_ablate_by_wasserstein_dist(data_per_neuron, selection_fun=fun)
        accs = calc_accuracies_after_ablation(param_file, model, layer_ind, sorted_neurons, step_size, n_steps, n_start)
        x_range = range(0, (n_steps + 1) * step_size, step_size)[:len(accs)]
        area = simps(accs, dx=step_size / x_range[-1])
        plt.plot(x_range, accs, label=fun_name + f'_{area:.2f}')
        np.save(f'ablations/{folder_name}/accs_{fun_name}_{layer_ind}.npy', (accs, area), allow_pickle=True)
    plt.legend()
    plt.savefig(f'ablations/{folder_name}/l_{layer_ind}.png', format='png', bbox_inches='tight', pad_inches=0)
    plt.show()
    plt.close()

def plot_ablation_strategy_comparisons(folder_name, fun_names, layer_inds):
    for layer_ind in layer_inds:
        step_size = 5
        n_steps = 51
        if layer_ind > 11:
            step_size = 10
        for fun_name in fun_names:
            accs, area = np.load(f'ablations/{folder_name}/accs_{fun_name}_{layer_ind}.npy', allow_pickle=True)
            x_range = range(0, (n_steps + 1) * step_size, step_size)[:len(accs)]
            plt.plot(x_range, accs, label=fun_name + f'_{area:.2f}')
        plt.legend()
        plt.title(layer_ind)
        plt.savefig(f'ablations/{folder_name}/l_{layer_ind}_absmax_{"_".join(fun_names)}.png', format='png', bbox_inches='tight', pad_inches=0)
        plt.close()

def create_areas_array(folder_name, fun_names, layer_inds):
    areas = np.zeros((len(layer_inds), len(fun_names)))
    for layer_ind in layer_inds:
        for i, fun_name in enumerate(fun_names):
            _, area = np.load(f'ablations/{folder_name}/accs_{fun_name}_{layer_ind}.npy', allow_pickle=True)
            areas[layer_ind, i] = area
    return areas

def make_predictions(model, loaded_activations, layeri, if_disable, disable_ind=None):
    with torch.no_grad():
        acts = loaded_activations.clone()#.cuda()
        if if_disable:
            acts[:, disable_ind] = 0  # 10 * acts_avg[i]#

        y_pred = []
        n_batches = 40
        n_samples = acts.shape[0]
        batch_size = int(np.ceil(n_samples / n_batches))
        for i in range(n_batches):
            x = acts[i * batch_size:(i+1) * batch_size, :].cuda()
            if layeri == 6:
                x = model.layer3(x) # for layer 6
            if layeri <= 10:
                x = model.layer4(x)
            x = model.avgpool(x)
            x = torch.flatten(x, 1)

            x = model.fc(x)
            y_pred_cur = torch.argmax(x, dim=1).detach().cpu().numpy()
            y_pred.append(y_pred_cur)
        y_pred = np.hstack(y_pred)
        del acts
    return y_pred

def sort_neurons_for_class(wasser_dists_values, target_class):
    w_dists = np.array([w_dict[target_class] for w_dict in wasser_dists_values])
    idx = np.argsort(w_dists)
    return idx, w_dists

def plot_prec_recall_change(save_model_path, param_file,
                          # saved_activations_path='activations_on_validation_preserved_spatial_10_pretrained_imagenet.npy',
                          saved_activations_path='activations_on_validation_14_pretrained_imagenet.npy',
                            saved_labels_path='labels_pretrained_imagenet.npy', wd_path=None,
                            l1_norms=None):

    model = reload_model(save_model_path, param_file)
    print(saved_activations_path)
    # works for layers 10 & 14:
    # with open(saved_activations_path, 'rb') as f:
    #     loaded_activations = torch.tensor(pickle.load(f))
    # works for layer 6:
    loaded_activations = torch.tensor(np.load(saved_activations_path))

    labels = np.load(saved_labels_path)
    wasser_dists_values = list(np.load(wd_path, allow_pickle=True).item().values())
    # print('Attention! What I do to get last linear weights only works for the pretrained imagenet model')

    if 'pretrained_imagenet' in saved_activations_path:
        fc_weights = model.fc.weight.detach().cpu().numpy()

    layeri = int(saved_activations_path.split('_')[-3])

    y_pred = make_predictions(model, loaded_activations, layeri, if_disable=False)
    conf_matr = confusion_matrix(labels, y_pred)
    precision_before, recall_before = conf_matr.diagonal() / conf_matr.sum(axis=0), conf_matr.diagonal() / conf_matr.sum(axis=1)

    n_neurons_dict = {
        6: 128,
        10: 256,
        14: 512
    }

    testing = False

    if not testing:
        ablate_neurons_config = {
            256: [8, 16, 32, 64, 128, 192],
            512: [16, 32, 64, 128, 256, 384],
            128: [4, 8, 16, 32, 64, 96],
        }
    else:
        ablate_neurons_config = {
            256: [8, 16, 32],
            512: [16, 32, 64],
            128: [4, 8, 16],
        }

    n_neurons = n_neurons_dict[layeri]
    assert n_neurons == loaded_activations.size()[1]

    n_classes = 1000 if not testing else 5

    n_neurons_to_disable = ablate_neurons_config[n_neurons]

    save_postfix = f'layer{layeri}_disabled_{"_".join(str(x) for x in n_neurons_to_disable)}'
    if n_classes == 1000:
        save_postfix += '_imagenet'
    elif n_classes == 10:
        save_postfix += '_cifar10'
    elif n_classes == 5:
        save_postfix += '_testing'
    else:
        raise Exception(f"Unknown n_classes: {n_classes}")

    # def change_2d_array_into_dict(arr, neuron_sorting_fun_names):
    #         means_dict = {}
    #         for i, n in enumerate(neuron_sorting_fun_names):
    #             means_dict[n] = arr[i, :]
    #         return means_dict

    permuted_neurons = np.random.permutation(n_neurons)

    sorting_funs_list = [
        ('random', lambda a, b: permuted_neurons),
        ('sort-max', lambda wasser_dists_values, idx: np.array(sort_neurons_for_class(wasser_dists_values, idx)[0][::-1])),
        ('sort-min', lambda wasser_dists_values, idx: sort_neurons_for_class(wasser_dists_values, idx)[0])
    ]

    if layeri == 14:
        if 'pretrained_imagenet' in saved_activations_path:
            sorting_funs_list += [
                ('sort-readout-max', lambda _, idx: np.argsort(fc_weights[idx])[::-1]),
                ('sort-readout-min', lambda _, idx: np.argsort(fc_weights[idx])),
            ]
    else:
        sorting_funs_list += [
            ('sort-l1', lambda _, idx: np.argsort(l1_norms))# this is sort-l1 for non-last layers
        ]

    neuron_sorting_fun_names, neuron_sorting_funs = zip(*sorting_funs_list)

    try:
        precision_means_dict = np.load(f'prec_means_dict{save_postfix}.npy', allow_pickle=True).item()
        recall_means_dict = np.load(f'recall_means_dict{save_postfix}.npy', allow_pickle=True).item()
        precision_stds_dict = np.load(f'prec_stds_dict{save_postfix}.npy', allow_pickle=True).item()
        recall_stds_dict = np.load(f'recall_stds_dict{save_postfix}.npy', allow_pickle=True).item()
    except:
        precision_means_dict = {}
        recall_means_dict = {}
        precision_stds_dict = {}
        recall_stds_dict = {}

        # precision_means = np.zeros((len(neuron_sorting_funs), len(n_neurons_to_disable)))
        # recall_means = np.zeros((len(neuron_sorting_funs), len(n_neurons_to_disable)))
        # precision_stds = np.zeros((len(neuron_sorting_funs), len(n_neurons_to_disable)))
        # recall_stds = np.zeros((len(neuron_sorting_funs), len(n_neurons_to_disable)))

    for (sorting_fun_name, sorting_fun) in zip(neuron_sorting_fun_names, neuron_sorting_funs):
        if sorting_fun_name in precision_means_dict:
            continue

        precision_means_dict[sorting_fun_name] = np.zeros(len(n_neurons_to_disable))
        recall_means_dict[sorting_fun_name] = np.zeros(len(n_neurons_to_disable))
        precision_stds_dict[sorting_fun_name] = np.zeros(len(n_neurons_to_disable))
        recall_stds_dict[sorting_fun_name] = np.zeros(len(n_neurons_to_disable))

        for j, n_to_disable in enumerate(n_neurons_to_disable):
            print(f'{n_to_disable} disabled')
            p_diffs = []
            r_diffs = []
            old_cur_idx = None
            for class_idx in range(n_classes):
                print(class_idx)
                st = time.time()
                cur_idx = list(sorting_fun(wasser_dists_values, class_idx)[:n_to_disable])
                # x = loaded_activations.clone().cuda()
                # x[:, cur_idx] = 0
                # x = model.fc(x)
                # conf_matr = confusion_matrix(labels, torch.argmax(x, dim=1).detach().cpu().numpy())

                if (old_cur_idx != cur_idx):
                    # don't recompute if ablations doesn't change
                    y_pred = make_predictions(model, loaded_activations, layeri, if_disable=True, disable_ind=cur_idx)
                    conf_matr = confusion_matrix(labels, y_pred)
                    p, r = conf_matr.diagonal() / conf_matr.sum(axis=0), conf_matr.diagonal() / conf_matr.sum(axis=1)
                    p = np.nan_to_num(p)

                old_cur_idx = cur_idx
                p_diff = p[class_idx] - precision_before[class_idx]
                r_diff = r[class_idx] - recall_before[class_idx]
                p_diffs.append(p_diff)
                r_diffs.append(r_diff)
                ed = time.time()
                print(f'time={ed - st}')

            precision_means_dict[sorting_fun_name][j] = np.mean(p_diffs)
            recall_means_dict[sorting_fun_name][j] = np.mean(r_diffs)
            precision_stds_dict[sorting_fun_name][j] = np.std(p_diffs)
            recall_stds_dict[sorting_fun_name][j] = np.std(r_diffs)

            # plt.rcParams.update({'font.size': 17})
            # proper_hist(np.array(p_diffs), bin_size=0.02)
            # plt.savefig('prec_change32_hist.png', format='png')
            # plt.show()

    np.save(f'prec_means_dict{save_postfix}.npy', precision_means_dict)
    np.save(f'recall_means_dict{save_postfix}.npy', recall_means_dict)
    np.save(f'prec_stds_dict{save_postfix}.npy', precision_stds_dict)
    np.save(f'recall_stds_dict{save_postfix}.npy', recall_stds_dict)

    info = {'neuron_sorting_fun_names':neuron_sorting_fun_names, 'n_neurons_to_disable':n_neurons_to_disable}
    np.save(f'info_{save_postfix}.npy', info)

    plt.title('Precision change')
    plt.xlabel('Units ablated')
    # plt.xscale('log')
    plt.xticks(n_neurons_to_disable)
    # plt.gca().get_xaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())
    for n in neuron_sorting_fun_names:
        plt.errorbar(n_neurons_to_disable, precision_means_dict[n], yerr=precision_stds_dict[n],
                    label=n, capsize=5)
    plt.legend()
    plt.tight_layout()
    plt.savefig(f'prec_change_{save_postfix}.png', format='png')
    plt.savefig(f'prec_change_{save_postfix}.svg', format='svg')
    plt.show()
    plt.close()

    plt.title('Recall change')
    plt.xlabel('Units ablated')
    # plt.xscale('log')
    plt.xticks(n_neurons_to_disable)
    # plt.gca().get_xaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())
    for n in neuron_sorting_fun_names:
        plt.errorbar(n_neurons_to_disable, recall_means_dict[n], yerr=recall_stds_dict[n],
                     label=n, capsize=5)
    plt.legend()
    plt.tight_layout()
    plt.savefig(f'rec_change_{save_postfix}.png', format='png')
    plt.savefig(f'rec_change_{save_postfix}.svg', format='svg')
    plt.show()
    plt.close()

def load_and_plot_prec_and_recall_change(dataset_str):
    assert dataset_str in ["imagenet", "cifar10"], f"Unknown dataset_str: {dataset_str}"

    # save_postfixes = [f'layer10_disabled_{i}' for i in [8, 16, 32, 64, 128, 192]]
    save_postfixes = [f'layer6_disabled_{i}' for i in [4, 8, 16, 32, 64, 96]]
    prec_means_all, prec_stds_all, recall_means_all, recall_stds_all = [], [], [], []
    n_neurons_to_disable = []
    for save_postfix in save_postfixes:
        pm = np.load(f'prec_means_{save_postfix}.npy')
        ps = np.load(f'prec_stds_{save_postfix}.npy')
        rm = np.load(f'recall_means_{save_postfix}.npy')
        rs = np.load(f'recall_stds_{save_postfix}.npy')
        info = np.load(f'info_{save_postfix}.npy', allow_pickle=True).item()
        neuron_sorting_fun_names = info['neuron_sorting_fun_names']
        n_neurons_to_disable += info['n_neurons_to_disable']

        prec_means_all.append(pm)
        prec_stds_all.append(ps)
        recall_means_all.append(rm)
        recall_stds_all.append(rs)
    def f(x):
        return np.concatenate(x, axis=1)
    prec_means_all, prec_stds_all, recall_means_all, recall_stds_all = f(prec_means_all), f(prec_stds_all), f(recall_means_all), f(recall_stds_all)
    plt.title('Precision change')
    plt.xlabel('Units ablated')
    plt.xticks(n_neurons_to_disable)
    for i in range(prec_means_all.shape[0]):
        plt.errorbar(n_neurons_to_disable, prec_means_all[i], yerr=prec_stds_all[i],
                     label=neuron_sorting_fun_names[i], capsize=5)
    plt.legend()
    plt.savefig(f'prec_change_combined.png', format='png')
    plt.show()

    plt.title('Recall change')
    plt.xlabel('Units ablated')
    plt.xticks(n_neurons_to_disable)
    for i in range(recall_means_all.shape[0]):
        plt.errorbar(n_neurons_to_disable, recall_means_all[i], yerr=recall_stds_all[i],
                     label=neuron_sorting_fun_names[i], capsize=5)
    plt.legend()
    plt.savefig(f'recall_change_combined.png', format='png')
    plt.show()

def ablate_neurons_based_on_saved_activations(save_model_path, param_file,
                                              # saved_activations_path='activations_on_validation_preserved_spatial_10_pretrained_imagenet.npy',
                                              saved_activations_path='activations_on_validation_14_pretrained_imagenet.npy',
                                              saved_labels_path='labels_pretrained_imagenet.npy', wd_path=None):
    model = reload_model(save_model_path, param_file)
    with open(saved_activations_path, 'rb') as f:
        loaded_activations = torch.tensor(np.load(f))
    acts_avg = loaded_activations.mean(dim=0)

    labels = np.load(saved_labels_path)

    layeri = int(saved_activations_path.split('_')[-3])

    n_neurons_dict = {
        6: 128,
        10: 256,
        14: 512
    }

    # ablate_neurons_config = {
    #     256: [8, 16, 32, 64, 128, 192],
    #     512: [16, 32, 64, 128, 256, 384],
    #     128: [4, 8, 16, 32, 64, 96],
    # }

    # this is for the statistics in the appendix
    ablate_neurons_config = {
        256: np.arange(256),
        512: np.arange(512),
        128: np.arange(128),
    }

    n_neurons = n_neurons_dict[layeri]
    assert n_neurons == loaded_activations.size()[1], f"n_neurons {n_neurons} doesn't match size: {loaded_activations.size()[1]}"

    ablate_neurons = ablate_neurons_config[n_neurons]

    n_classes = 1000
    precisions_after_disabling = np.zeros((len(ablate_neurons), n_classes))
    recalls_after_disabling = np.zeros((len(ablate_neurons), n_classes))
    acc_diffs = np.zeros(len(ablate_neurons))

    with torch.no_grad():
        y_pred = make_predictions(model, loaded_activations, layeri, if_disable=False)
        conf_matr = confusion_matrix(labels, y_pred)
        precision_before, recall_before = conf_matr.diagonal() / conf_matr.sum(axis=0), conf_matr.diagonal() / conf_matr.sum(axis=1)
        acc_before = conf_matr.diagonal().sum() / conf_matr.sum()

        perm = np.random.RandomState(1).permutation(n_neurons)

        for i, n in enumerate(ablate_neurons):
            print(i)
            # dind = perm[:n]
            dind = n
            y_pred = make_predictions(model, loaded_activations, layeri, if_disable=True, disable_ind=dind)
            conf_matr = confusion_matrix(labels, y_pred)
            precision, recall = conf_matr.diagonal() / conf_matr.sum(axis=0), conf_matr.diagonal() / conf_matr.sum(axis=1)
            precisions_after_disabling[i] = precision
            recalls_after_disabling[i] = recall
            acc_diffs[i] = conf_matr.diagonal().sum() / conf_matr.sum() - acc_before

    # plt.plot(acc_diffs, 'o')
    # plt.ylim(-0.0015, 0.0015)
    # plt.show()

    # plt.plot((recalls_after_disabling - recall_before)[444], 'o')
    # plt.show()

    wasser_dists_values = list(np.load(wd_path, allow_pickle=True).item().values())
    wds = pd.DataFrame(wasser_dists_values).to_numpy().T

    w = model.fc.weight.detach().cpu().numpy()

    rng = np.arange(n_neurons) ; np.random.shuffle(rng)
    maxs = np.array([np.max(list(wd_neuron.values())) for wd_neuron in wasser_dists_values])
    idx = np.argsort(maxs)
    mins = np.array([np.abs(np.min(list(wd_neuron.values()))) for wd_neuron in wasser_dists_values])
    max_recall_drop = (recalls_after_disabling - recall_before).min(axis=-1)
    max_prec_drop = (precisions_after_disabling - precision_before).min(axis=-1)
    print(scipy.stats.pearsonr(maxs, max_recall_drop))
    print(scipy.stats.pearsonr(maxs, max_prec_drop))
    print(scipy.stats.pearsonr(mins, max_recall_drop))
    print(scipy.stats.pearsonr(mins, max_prec_drop))
    # l1_norms_dict = np.load(f'l1_norms_{model_name_short}.npy', allow_pickle=True).item()
    # l1s = l1_norms_dict['layer4.1.conv2.weight']
    # l1s = l1s.values()
    # l1s = np.array(list(l1s))
    # idx = np.argsort(l1s)

    # np.save('prec_rec_10.npy',
    #         (precision_before, recall_before, precisions_after_disabling, recalls_after_disabling, acc_diffs),
    #         allow_pickle=True)
    exit()
    accs_cumulative_ablation = np.zeros(n_neurons)
    with torch.no_grad():
        for i in range(0, n_neurons):
            print(i)
            x = loaded_activations.clone()
            x[:, idx[:i]] = 0#acts_avg[:i]#
            x = model.fc(x)
            y_pred = torch.argmax(x, dim=1).detach().cpu().numpy()
            conf_matr = confusion_matrix(labels, y_pred)
            accs_cumulative_ablation[i] = conf_matr.diagonal().sum() / conf_matr.sum()
    plt.plot(accs_cumulative_ablation)
    accs_cumulative_ablation = np.zeros(n_neurons)
    with torch.no_grad():
        for i in range(0, n_neurons):
            print(i)
            x = loaded_activations.clone()
            x[:, idx[:i]] = acts_avg[idx[:i]]#
            x = model.fc(x)
            y_pred = torch.argmax(x, dim=1).detach().cpu().numpy()
            conf_matr = confusion_matrix(labels, y_pred)
            accs_cumulative_ablation[i] = conf_matr.diagonal().sum() / conf_matr.sum()
    plt.plot(accs_cumulative_ablation)
    plt.show()

    imagenet_dict_reversed = dict(zip(imagenet_dict.values(), imagenet_dict.keys()))

    idx, wdists = sort_neurons_for_class(wasser_dists_values, 689)

    class_idx1 = 11
    idx, wdists = sort_neurons_for_class(wasser_dists_values, class_idx1)
    x = loaded_activations.clone()
    cur_idx = list(idx[:8])  # + list(idx[-5:])#idx[0]
    x[:, cur_idx] = 0  # acts_avg[cur_idx]
    x = model.fc(x)
    conf_matr = confusion_matrix(labels, torch.argmax(x, dim=1).detach().cpu().numpy())
    p, r = conf_matr.diagonal() / conf_matr.sum(axis=0), conf_matr.diagonal() / conf_matr.sum(axis=1)
    class_idx2 = class_idx1
    print(f'{precision_before[class_idx2]:.3f} {p[class_idx2]:.3f} \n{recall_before[class_idx2]:.3f} {r[class_idx2]:.3f}')
    print(w[class_idx2, cur_idx])

    y = 1

if __name__ == '__main__':
    # load_and_plot_prec_and_recall_change()
    # exit()

    # #   single-head cifar
    # save_model_path = r'/mnt/raid/data/chebykin/saved_models/14_33_on_September_16/optimizer=SGD|batch_size=128|lr=0.1|connectivities_lr=0.0|chunks=[64|_64|_64|_128|_128|_128|_128|_256|_256|_256|_256|_512|_512|_512|_512]|architecture=binmatr2_resnet18|width_mul=1|weight_decay=0.000_240_model.pkl'
    # param_file = 'params/binmatr2_cifar_sgd1bias_fc_batch128_weightdecay3e-4_singletask.json'
    # model_name_short = save_model_path[37:53] + '...' + save_model_path[-12:-10]

    if if_pretrained_imagenet:
        model_name_short = 'pretrained_imagenet'
        save_model_path = 'dummy'
        param_file = 'dummy'

    try:
        l1_norms_dict = np.load(f'l1_norms_{model_name_short}.npy', allow_pickle=True).item()
    except:
        calc_l1_norms_of_weights(save_model_path)
        l1_norms_dict = np.load(f'l1_norms_{model_name_short}.npy', allow_pickle=True).item()

    usual_layer_idx_to_l1_norms_idx = { 0:'layer1.0.conv2.weight',
                                        1:'layer1.1.conv1.ordinary_conv.weight',
                                        2:'layer1.1.conv2.ordinary_conv.weight',
                                        3:'layer2.0.conv1.ordinary_conv.weight',
                                        4:'layer2.0.conv2.ordinary_conv.weight',
                                        5:'layer2.1.conv1.ordinary_conv.weight',
                                        6:'layer2.1.conv2.ordinary_conv.weight',
                                        7:'layer3.0.conv1.ordinary_conv.weight',
                                        8:'layer3.0.conv2.ordinary_conv.weight',
                                        9:'layer3.1.conv1.ordinary_conv.weight',
                                        10:'layer3.1.conv2.ordinary_conv.weight',
                                        11:'layer4.0.conv1.ordinary_conv.weight',
                                        12:'layer4.0.conv2.ordinary_conv.weight',
                                        13:'layer4.1.conv1.ordinary_conv.weight',
                                        14:'layer4.1.conv2.ordinary_conv.weight'}

    path_prefix = 'stored_activations/'
    for layeri in [14]: #[6, 10, 14]:
        # based_on = "_shift"
        based_on = "_dist"
        # ablate_neurons_based_on_saved_activations(save_model_path, param_file,
        #                 saved_activations_path=f'{path_prefix}activations_on_validation_preserved_spatial_{layeri}_pretrained_imagenet.npy',
        #                 wd_path=f'wasser_dists/wasser{based_on}_attr_hist_pretrained_imagenet_afterrelu_test_{layeri}.npy')
        # exit()

        plot_prec_recall_change(save_model_path, param_file,
                                # saved_activations_path=f'activations_on_validation_{layeri}_pretrained_imagenet.npy',
                                saved_activations_path=f'{path_prefix}activations_on_validation_preserved_spatial_{layeri}_pretrained_imagenet.npy',
                                wd_path=f'wasser_dists/wasser{based_on}_attr_hist_pretrained_imagenet_afterrelu_test_{layeri}.npy',
                                l1_norms=list(l1_norms_dict[usual_layer_idx_to_l1_norms_idx[layeri].replace('ordinary_conv.', '')].values()))
    exit()
    # plot_ablation_strategy_comparisons(model_name_short, ['sum', 'max', 'max+min', 'min'], range(15))
    # exit()
    # areas = create_areas_array(model_name_short, ['sum', 'max', 'max+min', 'min'], range(15))
    # exit()
    n_start = 0
    # layer_ind = 10
    for layer_ind in [14, 13, 12]:#range(12):#[7, 8, 9, 10]:#
        n_steps = 51
        step_size = 5
        if layer_ind <= 2:
            step_size = 3
            n_steps = 22
        if layer_ind >= 11:
            step_size = 10
        # wd_path = 'wasser_dists/wasser_dist_attr_hist_bettercifar10single_' + str(layer_ind) + '.npy'
        wd_path = 'wasser_dists/wasser_dist_attr_hist_pretrained_imagenet_afterrelu_' + str(layer_ind) + '.npy'
        plt.title(f'{layer_ind}')
        wasser_dists_values = np.load(wd_path, allow_pickle=True).item().values()
        compare_ablation_strategies(save_model_path, param_file, step_size, n_steps, layer_ind, n_start,
                [
                    # (lambda wd_neuron: np.sum(np.abs(list(wd_neuron.values()))), 'sum', wasser_dists_values),
                    # (lambda wd_neuron: np.max(list(wd_neuron.values())), 'max', wasser_dists_values),
                    # # lambda vals: np.max(vals) + (np.abs(np.min(vals)) if np.min(vals) < -0.1 else 0),
                    # (lambda wd_neuron: np.max(list(wd_neuron.values())) + np.abs(np.min(list(wd_neuron.values()))), 'max+min', wasser_dists_values),
                    # (lambda wd_neuron: np.abs(np.min(list(wd_neuron.values()))), 'min', wasser_dists_values),
                    # (lambda wd_neuron: np.random.rand(), 'rand', wasser_dists_values),
                    # (lambda l1_norm: l1_norm, 'l1', l1_norms_dict[usual_layer_idx_to_l1_norms_idx[layer_ind].replace('ordinary_conv.', '')].values()),
                    # (lambda wd_neuron: np.random.rand(), 'rand2', wasser_dists_values),
                    # (lambda wd_neuron: np.random.rand(), 'rand3', wasser_dists_values),
                    # (lambda wd_neuron: np.random.rand(), 'rand4', wasser_dists_values),
                    (lambda wd_neuron: np.max(np.abs(list(wd_neuron.values()))), 'absmax', wasser_dists_values),
                ], model_name_short)

    # if True:
    #     sorted_neurons = select_neurons_to_ablate_by_wasserstein_dist(wd_path, selection_fun=lambda vals:np.max(vals))
    # else:
    #     sorted_neurons = list(range(512))
    #     np.random.shuffle(sorted_neurons)
    # accs = calc_accuracies_after_ablation(param_file, trained_model, layer_ind, sorted_neurons, step_size, n_steps)
    # plt.plot(range(0, n_steps * step_size, step_size), accs)
    # plt.show()

'''
w = model.fc.weight.detach().cpu().numpy()

imagenet_dict_reversed = dict(zip(imagenet_dict.values(), imagenet_dict.keys()))

wasser_dists_values = np.load(wd_path, allow_pickle=True).item().values()
wasser_dists_values = list(np.load(wd_path, allow_pickle=True).item().values())

def sort_neurons_for_class(wasser_dists_values, target_class):
    w_dists = np.array([w_dict[target_class] for w_dict in wasser_dists_values])
    idx = np.argsort(w_dists)
    return idx, w_dists

idx, wdists = sort_neurons_for_class(wasser_dists_values, 689)
print(wdists[idx[:5]])

x = loaded_activations.clone()
x[:, idx[:16]] = 0
x = model.fc(x)
y_pred = torch.argmax(x, dim=1).detach().cpu().numpy()
conf_matr = confusion_matrix(labels, y_pred)
p, r = conf_matr.diagonal() / conf_matr.sum(axis=0), conf_matr.diagonal() / conf_matr.sum(axis=1)
class_idx = 689
print(precision_before[class_idx], p[class_idx])
print(recall_before[class_idx], r[class_idx])
'''


"""
SubtractOffset - subtract an offset from an array

configuration arguments:
  offset: string or number. If string, then it's an input field name.
  in_field: string, name of field to extract from alert data
  out_field: string. Can be the same as in_field (which is the default)

payload input:
  [in_field]: the input data.
  [offset]: number, if configuration 'offset' was a string
"""
import logging
import numpy as np

from snewpdag.dag import Node

class SubtractOffset(Node):
  def __init__(self, offset, in_field, **kwargs):
    self.in_field = in_field
    self.out_field = kwargs.pop('out_field', in_field)
    self.use_offset_field = isinstance(offset, str)
    if self.use_offset_field:
      self.offset_field = offset
    else:
      self.offset = offset
    super().__init__(**kwargs)

  def alert(self, data):
    if self.use_offset_field:
      offset = data[self.offset_field]
    else:
      offset = self.offset
    nfield = np.array(data[self.in_field]) # will make copy
    data[self.out_field] = nfield - offset
    return True


from typing import *

def printNumbers(self, n: int) -> List[int]:
    def dfs(x):
        if x == n:
            s = ''.join(num[self.start:])
            if s != '0': res.append(int(s))
            if n - self.start == self.nine: self.start -= 1
            return
        for i in range(10):
            if i == 9: self.nine += 1
            num[x] = str(i)
            dfs(x + 1)
        self.nine -= 1

    num, res = ['0'] * n, []
    self.nine = 0
    self.start = n - 1
    dfs(0)
    return res


from google.appengine.dist import use_library
use_library('django', '1.2')

# pylint: disable=E0611
from google.appengine.api import users
# pylint: disable=E0611
from google.appengine.ext import db, webapp
# pylint: disable=E0611
from google.appengine.ext.webapp import template
# pylint: disable=E0611
from google.appengine.ext.webapp.util import run_wsgi_app

import config
import queue

from datetime import datetime, timedelta
try:
  from django.utils import simplejson as json
except ImportError:
  import json
import logging
import random
import sys
import time
import traceback

from common import getProject, getTemplatePath
from datamodel import LoggedError, LoggedErrorInstance, AggregatedStats

####### Parse the configuration. #######

NAME = config.get('name')

SECRET_KEY = config.get('secretKey')

REQUIRE_AUTH = config.get('requireAuth', True)

####### Utility methods. #######

INSTANCE_FILTERS = ('environment', 'server', 'affectedUser')

INTEGER_FILTERS = ('affectedUser',)

def getFilters(request):
  """Gets the filters applied to the given request."""
  filters = {}
  for key, value in request.params.items():
    if key in INSTANCE_FILTERS or key in ('project', 'errorLevel', 'maxAgeHours'):
      filters[key] = value
  return filters

def filterInstances(dataSet, key, value):
  """Filters a data set."""
  if key in INTEGER_FILTERS:
    return dataSet.filter(key + ' =', int(value))
  elif key == 'maxAgeHours':
    return dataSet
  else:
    return dataSet.filter(key + ' =', value)

def getErrors(filters, limit, offset):
  """Gets a list of errors, filtered by the given filters."""
  for key in filters:
    if key in INSTANCE_FILTERS:
      return None, getInstances(filters, limit=limit, offset=offset)

  errors = LoggedError.all().filter('active =', True)
  for key, value in filters.items():
    if key == 'maxAgeHours':
      errors = errors.filter('firstOccurrence >', datetime.now() - timedelta(hours = int(value)))
    elif key == 'project':
      errors = errors.filter('project =', getProject(value))
    else:
      errors = errors.filter(key, value)
  if 'maxAgeHours' in filters:
    errors = errors.order('-firstOccurrence')
  else:
    errors = errors.order('-lastOccurrence')

  return errors.fetch(limit, offset), None

def getInstances(filters, parent = None, limit = None, offset = None):
  """Gets a list of instances of the given parent error, filtered by the given filters."""

  query = LoggedErrorInstance.all()
  if parent:
    query = query.filter('error =', parent)

  if filters:
    for key, value in filters.items():
      if key in INSTANCE_FILTERS:
        query = filterInstances(query, key, value)
      elif key == 'project' and not parent:
        query = query.filter('project =', getProject(value))

  return query.order('-date').fetch(limit or 51, offset or 0)

####### Pages #######

class AuthPage(webapp.RequestHandler):
  """Base class for pages that require authentication."""

  def __getUser(self):
    """Gets a user."""
    return users.get_current_user()

  def get(self, *args):
    """Handles a get, ensuring the user is authenticated."""
    user = self.__getUser()
    if user or not REQUIRE_AUTH:
      self.doAuthenticatedGet(user, *args)
    else:
      self.redirect(users.create_login_url(self.request.uri))

  def doAuthenticatedGet(self, _, *__):
    """Performs a get with an authenticated user."""
    self.error(500)

  def post(self, *args):
    """Handles a post, ensuring the user is authenticated."""
    user = self.__getUser()
    if user or not REQUIRE_AUTH:
      self.doAuthenticatedPost(user, *args)
    else:
      self.redirect(users.create_login_url(self.request.uri))

  def doAuthenticatedPost(self, _, *__):
    """Performs a post with an authenticated user."""
    self.error(500)

class ReportPage(webapp.RequestHandler):
  """Page handler for reporting a new exception."""

  def post(self):
    """Handles a new error report via POST."""
    key = self.request.get('key')

    if key != SECRET_KEY:
      self.error(403)
      return

    # Add the task to the instances queue.
    queue.queueException(self.request.body)

class StatPage(webapp.RequestHandler):
  """Page handler for collecting error instance stats."""

  def get(self):
    """Handles a new error report via POST."""
    key = self.request.get('key')

    if key != SECRET_KEY:
      self.error(403)
      return

    counts = []
    project = self.request.get('project')
    if project:
      project = getProject(project)
      if not project:
        self.response.out.write(' '.join(['0' for _ in counts]))
    for minutes in self.request.get('minutes').split():
      query = LoggedErrorInstance.all()
      if project:
        query = query.ancestor(project)
      counts.append(query.filter('date >=', datetime.now() - timedelta(minutes = int(minutes))).count())

    self.response.out.write(' '.join((str(count) for count in counts)))

class AggregateViewPage(webapp.RequestHandler):
  """Page handler for collecting error instance stats."""

  def get(self, viewLength):
    """Handles a new error report via POST."""
    if viewLength != 'day':
      # TODO(robbyw): For viewLength == week or viewLength == month, aggregate the aggregates.
      viewLength = 'day'

    data = AggregatedStats.all().order('-date').get()
    data = json.loads(data.json)[:25]

    for _, row in data:
      logging.info(row)
      row['servers'] = sorted(row['servers'].items(), key = lambda x: x[1], reverse=True)
      row['environments'] = sorted(row['environments'].items(), key = lambda x: x[1], reverse=True)

    keys, values = zip(*data)
    errors = LoggedError.get([db.Key(key) for key in keys])

    context = {
      'title': 'Top 25 exceptions over the last %s' % viewLength,
      'errors': zip(errors, values),
      'total': len(data)
    }
    self.response.out.write(template.render(getTemplatePath('aggregation.html'), context))

class ListPage(AuthPage):
  """Page displaying a list of exceptions."""

  def doAuthenticatedGet(self, user):
    self.response.headers['Content-Type'] = 'text/html'

    filters = getFilters(self.request)

    page = int(self.request.get('page', 0))
    errors, instances = getErrors(filters, limit = 51, offset = page * 50)

    if errors is not None:
      hasMore = len(errors) == 51
      errors = errors[:50]
    else:
      hasMore = len(instances) == 51
      instances = instances[:50]

    context = {
      'title': NAME,
      'extraScripts': ['list'],
      'user': user,
      'filters': filters.items(),
      'errors': errors,
      'instances': instances,
      'hasMore': hasMore,
      'nextPage': page + 1
    }
    self.response.out.write(template.render(getTemplatePath('list.html'), context))

class ViewPage(AuthPage):
  """Page displaying a single exception."""

  def doAuthenticatedGet(self, user, *args):
    key, = args
    self.response.headers['Content-Type'] = 'text/html'
    error = LoggedError.get(key)
    filters = getFilters(self.request)
    context = {
      'title': '%s - %s' % (error.lastMessage, NAME),
      'extraScripts': ['view'],
      'user': user,
      'error': error,
      'filters': filters.items(),
      'instances': getInstances(filters, parent=error)[:100]
    }
    self.response.out.write(template.render(getTemplatePath('view.html'), context))

class ResolvePage(AuthPage):
  """Page that resolves an exception."""

  def doAuthenticatedGet(self, _, *args):
    key, = args
    self.response.headers['Content-Type'] = 'text/plain'
    error = LoggedError.get(key)
    error.active = False
    error.put()

    self.response.out.write('ok')

class ClearDatabasePage(AuthPage):
  """Page for clearing the database."""

  def doAuthenticatedGet(self, _):
    if users.is_current_user_admin():
      for error in LoggedError.all():
        error.delete()
      for instance in LoggedErrorInstance.all():
        instance.delete()
      self.response.out.write('Done')
    else:
      self.redirect(users.create_login_url(self.request.uri))

class ErrorPage(webapp.RequestHandler):
  """Page that generates demonstration errors."""

  def get(self):
    """Handles page get for the error page."""
    for _ in range(10):
      error = random.choice(range(4))
      errorLevel = 'error'
      project = 'frontend'
      try:
        if error == 0:
          project = 'backend'
          x = 10 / 0
        elif error == 1:
          errorLevel = 'warning'
          json.loads('{"abc", [1, 2')
        elif error == 2:
          x = {}
          x = x['y']
        elif error == 3:
          x = {}
          x = x['z']

      except (KeyError, ZeroDivisionError, ValueError):
        excInfo = sys.exc_info()
        stack = traceback.format_exc()
        env = random.choice(['dev', 'prod'])
        exception = {
          'timestamp': time.time(),
          'project': project,
          'serverName':'%s %s %d' % (env, project, random.choice(range(3))),
          'type': excInfo[0].__module__ + '.' + excInfo[0].__name__,
          'environment': env,
          'errorLevel': errorLevel,
          'message': str(excInfo[1]),
          'logMessage': 'Log message goes here',
          'backtrace': stack,
          'context':{'userId':random.choice(range(20))}
        }
        queue.queueException(json.dumps(exception))

    self.response.out.write('Done!')

####### Application. #######

def main():
  """Runs the server."""
  endpoints = [
    ('/', ListPage),

    ('/clear', ClearDatabasePage),

    ('/report', ReportPage),

    ('/view/(.*)', ViewPage),
    ('/resolve/(.*)', ResolvePage),

    ('/stats', StatPage),
    ('/review/(.*)', AggregateViewPage),
  ] + queue.getEndpoints()
  if config.get('demo'):
    endpoints.append(('/error', ErrorPage))
  application = webapp.WSGIApplication(endpoints, debug=True)

  run_wsgi_app(application)

if __name__ == "__main__":
  main()


# Creates a class named Song
class Song(object):
    # Instantiate or create an object from the class; creating multiple individual modules
    # self refers to empty object python creates and you can assign variables to it
    # Here we have self and lyrics and assign self.lyrics to lyrics.
    def __init__(self, lyrics):
        self.lyrics = lyrics
    # Function that has self (which we set to lyrics) passed to it
    # The function has a for loop that reads the lines assigned to self.lyrics and
    # for each line prints to stdout.
    def sing_me_a_song(self):
        for line in self.lyrics:
            print(line)

# Creates a variable called happy_bday with a list of strings assigned to it, and then sees we are calling
# a class we defined called Song and are passing the list of strings to the variable called lyrics which
# will then assign to self.lyrics.
happy_bday = Song(["Happy birthday to you",
                    "I don't want to get sued,"
                    "So I'll stop right there"])
# Since classes give us a blueprint for creating many individual copies of an object, we can perform the same
# process on a different variable without interfering with the previous object.
bulls_on_parade = Song(["They rally around tha family",
                        "With pockets full of shells"])
# Calls the sing_me_a_song function from the Song class using the previously defined lyrics for each object.
happy_bday.sing_me_a_song()
bulls_on_parade.sing_me_a_song()


input = __import__('sys').stdin.readline

n = int(input())
a = [list(map(int, input().split())) for _ in range(n)]

a.sort(key = lambda x: (x[1], x[0]))
count = 1
end = a[0][1]

for i in a[1:]:
    if i[0] >= end:
        count += 1
        end = i[1]

print(count)


#!/usr/bin/python3
import sys

# Rates
usd_rates = {
    "EUR": 0.84,
    "JPY": 109.19,
    "GBP": 0.72,
    "AUD": 1.29,
    "CAD": 1.25,
    "CHF": 0.92,
    "CNY": 6.50
}

eur_rates = {
    "USD": 1.19,
    "JPY": 128.92,
    "GBP": 0.86,
    "AUD": 1.54,
    "CAD": 1.49,
    "CHF": 1.10,
    "CNY": 7.80
}

jpy_rates = {
    "USD": 0.0091,
    "EUR": 0.0078,
    "GBP": 0.0067,
    "AUD": 0.012,
    "CAD": 0.011,
    "CHF": 0.0083,
    "CNY": 0.059
}

gbp_rates = {
    "USD": 1.39,
    "EUR": 1.16,
    "JPY": 148.85,
    "AUD": 1.80,
    "CAD": 1.74,
    "CHF": 1.28,
    "CNY": 9.06
}

aud_rates = {
    "USD": 0.77,
    "EUR": 0.65,
    "JPY": 83.65,
    "GBP": 0.56,
    "CAD": 0.97,
    "CHF": 0.72,
    "CNY": 5.09
}

cad_rates = {
    "USD": 0.80,
    "EUR": 0.67,
    "JPY": 86.85,
    "GBP": 0.57,
    "AUD": 1.03,
    "CHF": 0.74,
    "CNY": 5.27
}

chf_rates = {
    "USD": 1.09,
    "EUR": 0.91,
    "JPY": 117.34,
    "GBP": 0.78,
    "AUD": 1.39,
    "CAD": 1.35,
    "CNY": 7.67
}

cny_rates = {
    "USD": 0.15,
    "EUR": 0.13,
    "JPY": 16.85,
    "GBP": 0.11,
    "AUD": 0.20,
    "CAD": 0.19,
    "CHF": 0.13
}

# Accepted Currencies
currencies = {
    'USD': usd_rates,
    'EUR': eur_rates,
    'JPY': jpy_rates,
    'GBP': gbp_rates,
    'AUD': aud_rates,
    'CAD': cad_rates,
    'CHF': chf_rates,
    'CNY': cny_rates
}

# Error codes
def error_code(type):
    if type == 1:
        return "Invalid input. Please enter a valid amount."
    elif type == 2:
        return "Invalid currency code. Please enter a valid currency code"
    else:
        return "Error"

# Getting correct input
while True:
    try:
        amount = float(input("Enter amount: "))
        if amount >= 0:
            break
        else:
            print("Invalid input. Please enter a valid amount.")
    except ValueError:
        print(error_code(1))

while True:
    try:
        original_cur = str(input("Enter original currency code: "))
        if original_cur in currencies:
            break
        else:
            print(error_code(2))
    except ValueError:
        print(error_code(2))

while True:
    try:
        desired_cur = str(input("Enter desired currency code: "))
        if desired_cur in currencies:
            if desired_cur == original_cur:
                result = amount
                print("Result: {:0.2f} {}".format(result, desired_cur))
                sys.exit(0)
            break
        else:
            print(error_code(2))
    except ValueError:
        print(error_code(2))

rate = currencies[original_cur][desired_cur]
result = amount * rate

print("Result: {:0.2f} {}".format(result, desired_cur))


# -*- coding: utf-8 -*-
import socket
import numpy as np
import cv2

host, port = 'localhost', 10488

# 利用 socket 接受二进制数据
def receive_data():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.bind((host, port))
    data_all = []
    while True:
        # 2048 --- buffer_size 参数为接收的最大数据量
        data, address = s.recvfrom(2048)
        if data == b'bye':
            print("收到客户端发送的 bye, 断开连接")
            break
        data_all.append(data)
    s.close()
    return data_all

# 解码并组成图像
def decode_and_compos_image(data):
    image = np.array(data, np.uint8).tobytes()
    print("image=", image)
    try:
        decode_image = cv2.imdecode(image, cv2.IMREAD_COLOR)
        cv2.imshow('image_show', decode_image)
        print("showed the image to screen")
        cv2.imwrite("../image/receive_image/im-8.png", decode_image)
    except socket.error as e:
        print("非图像数据", e)
        exit(-1)
    cv2.waitKey(0)
    if 0xFF == ord('q'):
        cv2.destroyAllWindows()

if __name__ == '__main__':
    print("server 已启动")
    # 接收数据
    data = receive_data()
    # print("data =", data)
    # 解包组成图像
    decode_and_compos_image(data)


from odoo import _, api, models
from odoo.exceptions import ValidationError

class HrExpense(models.Model):
    _inherit = "hr.expense"

    @api.model
    def _get_under_validation_exceptions(self):
        """Extend for more field exceptions."""
        return ["message_follower_ids", "access_token"]

    def _check_allow_write_under_validation(self, vals):
        """Allow to add exceptions for fields that are allowed to be written
        even when the record is under validation."""
        exceptions = self._get_under_validation_exceptions()
        if any(val not in exceptions for val in vals):
            return False
        return True

    def write(self, vals):
        for rec in self:
            if (
                rec.sheet_id.state == "submit"
                and rec.sheet_id.review_ids
                and not rec.sheet_id.validated
                and not rec.sheet_id.rejected
                and not rec._check_allow_write_under_validation(vals)
            ):
                raise ValidationError(_("The expense report is under validation."))
        return super().write(vals)


from rest_framework.views import APIView
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework import status
from drf_yasg.utils import swagger_auto_schema
from ..models import (
    CompanyQuestion,
)
from ..serializers import (
    CompanyQuestionSerializer,
)

class CompanyQuestionCreateView(APIView):
    @swagger_auto_schema(
        request_body=CompanyQuestionSerializer,
        operation_description="Create new companyquestion",
        operation_summary="Create new companyquestion",
        responses={
            201: CompanyQuestionSerializer,
            400: "Bad request"
        }
    )
    def post(self, request:Request):
        data = request.data
        serializer = CompanyQuestionSerializer(data=data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED,)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST,)

class CompanyQuestionUpdateView(APIView):
    @swagger_auto_schema(
        request_body=CompanyQuestionSerializer,
        operation_description="Update companyquestion",
        operation_summary="Update companyquestion",
        responses={
            200: CompanyQuestionSerializer,
            400: "Bad request"
        }
    )
    def put(self, request:Request, pk:int):
        companyquestion = CompanyQuestion.objects.get(pk=pk)
        companyquestion.question = request.data['question']
        companyquestion.answer = request.data['answer']
        companyquestion.save()
        serializer = CompanyQuestionSerializer(companyquestion)
        return Response(serializer.data, status=status.HTTP_200_OK,)

class CompanyQuestionGetView(APIView):
    @swagger_auto_schema(
        operation_description="Get companyquestion",
        operation_summary="Get companyquestion",
        responses={
            200: CompanyQuestionSerializer,
            400: "Bad request"
        }
    )
    def get(self, request:Request):
        companyquestion = CompanyQuestion.objects.all()
        serializer = CompanyQuestionSerializer(companyquestion, many=True)
        return Response(serializer.data, status=status.HTTP_200_OK,)

class CompanyQuestionGetIdView(APIView):
    @swagger_auto_schema(
        operation_description="Get companyquestion by id",
        operation_summary="Get companyquestion by id",
        responses={
            200: CompanyQuestionSerializer,
            400: "Bad request"
        }
    )
    def get(self, request:Request, pk:int):
        companyquestion = CompanyQuestion.objects.get(pk=pk)
        serializer = CompanyQuestionSerializer(companyquestion)
        return Response(serializer.data, status=status.HTTP_200_OK,)

class CompanyQuestionDeleteView(APIView):
    @swagger_auto_schema(
        operation_description="Delete companyquestion",
        operation_summary="Delete companyquestion",
        responses={
            204: "No content",
            400: "Bad request"
        }
    )
    def delete(self, request:Request, pk:int):
        companyquestion = CompanyQuestion.objects.get(pk=pk)
        companyquestion.delete()
        return Response(status=status.HTTP_204_NO_CONTENT,)


import re
import os
import sys
import zipfile
import numpy as np
import tensorflow as tf
from typing import Tuple
from typing import Any
from typing import AnyStr
from typing import TextIO
from typing import Dict

class StandardScaler():
    def __init__(self, mean, std):
        self.mean = mean
        self.std = std

    def transform(self, data):
        return (data - self.mean) / self.std

    def inverse_transform(self, data):
        return (data * self.std) + self.mean

def combine_mask(seq: tf.Tensor) -> Tuple:
    """对input中的不能见单位进行mask

    :param seq: 输入序列
    :param d_type: 运算精度
    :return: mask
    """
    look_ahead_mask = _create_look_ahead_mask(seq)
    padding_mask = create_padding_mask(seq)
    return tf.maximum(look_ahead_mask, padding_mask)

def create_padding_mask(seq: tf.Tensor) -> Tuple:
    """ 用于创建输入序列的扩充部分的mask

    :param seq: 输入序列
    :return: mask
    """
    seq = tf.cast(x=tf.math.equal(seq, 0), dtype=tf.float32)
    return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)

def _create_look_ahead_mask(seq: tf.Tensor) -> Tuple:
    """ 用于创建当前点以后位置部分的mask

    :param seq: 输入序列
    :return: mask
    """
    seq_len = tf.shape(seq)[1]
    look_ahead_mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
    return look_ahead_mask

def load_checkpoint(checkpoint_dir: AnyStr, checkpoint_save_size: Any,
                    model: tf.keras.Model = None) -> tf.train.CheckpointManager:
    """ 加载检查点

    :param checkpoint_dir: 检查点保存目录
    :param checkpoint_save_size: 检查点最大保存数量
    :param model: 传入的模型
    :return: 检查点管理器
    """
    if not model:
        raise ValueError("加载检查点时所传入模型有误，请检查后重试！")

    checkpoint = tf.train.Checkpoint(model=model)
    checkpoint_manager = tf.train.CheckpointManager(checkpoint=checkpoint, directory=checkpoint_dir,
                                                    max_to_keep=checkpoint_save_size)

    checkpoint.restore(checkpoint_manager.latest_checkpoint).expect_partial()

    return checkpoint_manager

class ProgressBar(object):
    """ 进度条工具 """

    EXECUTE = "%(current)d/%(total)d %(bar)s (%(percent)3d%%) %(metrics)s"
    DONE = "%(current)d/%(total)d %(bar)s - %(time).4fs/step %(metrics)s"

    def __init__(self, total: Any = 100, num: Any = 1, width: Any = 30, fmt: AnyStr = EXECUTE,
                 symbol: AnyStr = "=", remain: AnyStr = ".", output: TextIO = sys.stderr):
        """
        :param total: 执行总的次数
        :param num: 每执行一次任务数量级
        :param width: 进度条符号数量
        :param fmt: 进度条格式
        :param symbol: 进度条完成符号
        :param remain: 进度条未完成符号
        :param output: 错误输出
        """
        assert len(symbol) == 1
        self.args = {}
        self.metrics = ""
        self.total = total
        self.num = num
        self.width = width
        self.symbol = symbol
        self.remain = remain
        self.output = output
        self.fmt = re.sub(r"(?P<name>%\(.+?\))d", r"\g<name>%dd" % len(str(total)), fmt)

    def __call__(self, current: Any, metrics: AnyStr):
        """
        :param current: 已执行次数
        :param metrics: 附加在进度条后的指标字符串
        """
        self.metrics = metrics
        percent = current / float(self.total)
        size = int(self.width * percent)
        bar = "[" + self.symbol * size + ">" + self.remain * (self.width - size - 1) + "]"

        self.args = {
            "total": self.total * self.num,
            "bar": bar,
            "current": current * self.num,
            "percent": percent * 100,
            "metrics": metrics
        }
        print("\r" + self.fmt % self.args, file=self.output, end="")

    def reset(self, total: Any, num: Any, width: Any = 30, fmt: AnyStr = EXECUTE,
              symbol: AnyStr = "=", remain: AnyStr = ".", output: TextIO = sys.stderr):
        """重置内部属性

        :param total: 执行总的次数
        :param num: 每执行一次任务数量级
        :param width: 进度条符号数量
        :param fmt: 进度条格式
        :param symbol: 进度条完成符号
        :param remain: 进度条未完成符号
        :param output: 错误输出
        """
        self.__init__(total=total, num=num, width=width, fmt=fmt,
                      symbol=symbol, remain=remain, output=output)

    def done(self, step_time: Any, fmt: AnyStr = DONE):
        """
        :param step_time: 该时间步执行完所用时间
        :param fmt: 执行完成之后进度条格式
        """
        self.args["bar"] = "[" + self.symbol * self.width + "]"
        self.args["time"] = step_time
        print("\r" + fmt % self.args + "\n", file=self.output, end="")

def get_dict_string(data: Dict, prefix: AnyStr = "- ", precision: AnyStr = ": {:.4f} "):
    """将字典数据转换成key——value字符串

    :param data: 字典数据
    :param prefix: 组合前缀
    :param precision: key——value打印精度
    :return: 字符串
    """
    result = ""
    for key, value in data.items():
        result += (prefix + key + precision).format(value)

    return result

def read_npy_file(filename):
    """
    专门用于匹配dataset的map读取文件的方法
    :param filename: 传入的文件名张量
    :return: 返回读取的数据
    """
    data = np.load(filename.numpy().decode())
    return data.astype(np.float32)

def process_train_pairs(train_enc: Any, train_dec: Any, month_enc: Any, month_dec: Any, labels: Any):
    [train_e, ] = tf.py_function(read_npy_file, [train_enc], [tf.float32, ])
    [train_d, ] = tf.py_function(read_npy_file, [train_dec], [tf.float32, ])
    [month_e, ] = tf.py_function(read_npy_file, [month_enc], [tf.float32, ])
    [month_d, ] = tf.py_function(read_npy_file, [month_dec], [tf.float32, ])
    [label_s, ] = tf.py_function(read_npy_file, [labels], [tf.float32, ])

    return train_e, train_d, month_e, month_d, label_s

def make_zip(source_dir, output_filename):
    zip_file = zipfile.ZipFile(output_filename, "w")
    pre_len = len(os.path.dirname(source_dir))
    for parent, dir_names, filenames in os.walk(source_dir):
        for filename in filenames:
            path_file = os.path.join(parent, filename)
            arc_name = path_file[pre_len:].strip(os.path.sep)  # 相对路径
            zip_file.write(path_file, arc_name)
    zip_file.close()

def coreff(x, y):
    x_mean = np.mean(x)
    y_mean = np.mean(y)
    c1 = sum((x - x_mean) * (y - y_mean))
    c2 = sum((x - x_mean) ** 2) * sum((y - y_mean) ** 2)
    return c1 / np.sqrt(c2)

def rmse(preds, y):
    return np.sqrt(sum((preds - y) ** 2) / preds.shape[0])

def eval_score(preds, label):
    acskill = 0
    RMSE = 0
    a = 0
    a = [1.5] * 4 + [2] * 7 + [3] * 7 + [4] * 6
    for i in range(24):
        RMSE += rmse(label[:, i], preds[:, i])
        cor = coreff(label[:, i], preds[:, i])

        acskill += a[i] * np.log(i + 1) * cor
    return 2 / 3 * acskill - RMSE


"""
R&D
https://www.kaggle.com/andreipopovici/forecasting-stocks/notebook
"""

import sys
import os
sys.path.insert(0, os.path.abspath('..'))
# import tensorflow as tf
import configs.alphaconf
import libs.stockslib as sl
import libs.technical_indicators as ti
from pprint import pprint
import pandas as pd
import numpy as np
from fbprophet import Prophet
from fbprophet.diagnostics import cross_validation
import matplotlib.pyplot as plt

sybmol = 'ABBV'

res = sl.RESOURCE(symbol=sybmol)
res.get_prices_from_alpha(key=configs.alphaconf.key, cacheage=3600*24*1, cachedir='..\cache')
res.get_history_from_alpha(key=configs.alphaconf.key, cacheage=3600*24, cachedir='..\history')
res.fix_alpha_columns()
res.fix_alpha_history_columns()

# Prepare data for the Prophet

dfraw = res.history
dfraw = dfraw.reset_index()

# split into train/test 90/10
df_size = len(dfraw)
train_size = round(df_size*95/100)
test_size = round(df_size*5/100)

df_train = dfraw[:train_size]
df_test = dfraw[-test_size:]

# init Prophet dataframe
train = pd.DataFrame()
# convert to Prophet required form
train[['ds', 'y']] = df_train[['date', 'Adjusted close']]

# Init & fit model
model = Prophet()
model.fit(train)

forecast_frame = model.make_future_dataframe(periods=test_size)

forecast = model.predict(forecast_frame)
pprint(forecast.tail())

validation = cross_validation(model, horizon='30 days', period='30 days', initial='1800 days')
# last 5 entries
pprint(validation.tail())

mean_error = np.mean(np.abs((validation['y'] - validation['yhat']) / validation['y'])) * 100
print(mean_error)

model.plot(forecast)
plt.show()


import cv2
import time

# Camera 0 is the integrated web cam on my netbook
camera_port = -1

# Now we can initialize the camera capture object with the cv2.VideoCapture class.
# All it needs is the index to a camera port.
camera = cv2.VideoCapture(camera_port)

# Captures a single image from the camera and returns it in PIL format
def get_image():
	# read is the easiest way to get a full image out of a VideoCapture object.
	retval, im = camera.read()
	print(retval)
	return im

print("Taking image...")
# Take the actual image we want to keep
camera_capture = get_image()

file = "test_image.jpg"
# A nice feature of the imwrite method is that it will automatically choose the
# correct format based on the file extension you provide. Convenient!
cv2.imwrite(file, camera_capture)

time.sleep(5)

camera_capture = get_image()

file = "test_image1.jpg"
# A nice feature of the imwrite method is that it will automatically choose the
# correct format based on the file extension you provide. Convenient!
cv2.imwrite(file, camera_capture)

# You'll want to release the camera, otherwise you won't be able to create a new
# capture object until your script exits
del(camera)


from enum import EnumType
import json
import types
from typing import Any, Type, TypeVar, get_args, get_origin, get_type_hints

from .util import (
    SIMPLE_TYPES,
    SerializationError,
    get_type_name,
    verify_type,
)

T = TypeVar("T")

def deserialize_item(data_type: Any, data: Any) -> Any:
    if data_type in SIMPLE_TYPES:
        verify_type(data, data_type)
        return data
    if isinstance(data_type, types.GenericAlias):
        return deserialize_generic(data_type, data)
    if isinstance(data_type, types.UnionType):
        return deserialize_union(data_type, data)
    if isinstance(data_type, EnumType):
        return deserialize_enum(data_type, data)
    return deserialize_object(data_type, data)

def deserialize_enum(type_info: EnumType, data: Any) -> Any:
    try:
        return type_info(data)
    except ValueError:
        raise SerializationError(
            f"Can't deserialize enum {type_info}: Invalid value {data}"
        )

def deserialize_union(type_info: types.UnionType, data: Any) -> Any:
    if type(data) != list or len(data) != 2:
        raise SerializationError(
            f"Can't deserialize Union type {type_info}: Invalid format.\nData:\n{data}"
        )

    for t in get_args(type_info):
        if data[0] == get_type_name(t):
            item = deserialize_item(t, data[1])
            verify_type(item, t)
            return item
    raise SerializationError(
        f"Union type {type_info} has failed to deserialize as any of its types. "
        f"Input type: {data[0]}"
    )

def deserialize_generic(type_info: types.GenericAlias, data: Any) -> Any:
    expected_type: Any = get_origin(type_info)
    verify_type(data, list)  # We use lists to represent lists, sets, dicts and tuples
    if expected_type in (list, set):
        item_type = get_args(type_info)[0]
        values = expected_type(deserialize_item(item_type, i) for i in data)
        return values
    if expected_type == dict:
        key_type, value_type = get_args(type_info)
        values = {
            deserialize_item(key_type, k): deserialize_item(value_type, v)
            for k, v in data
        }
        return values
    if expected_type == tuple:
        item_types = get_args(type_info)
        values = tuple(deserialize_item(t, d) for t, d in zip(item_types, data))
        return values
    raise SerializationError(f"Generic type {expected_type} is not supported")

def deserialize_object(data_type: Type[T], data: Any) -> T:
    type_hints = get_type_hints(data_type)
    out = data_type.__new__(data_type)
    for i, (name, item_type) in enumerate(type_hints.items()):
        item = deserialize_item(item_type, data[i])
        verify_type(item, item_type)
        object.__setattr__(out, name, item)
    return out

def deserialize(data_type: Any, data: str) -> Any:
    return deserialize_item(data_type, json.loads(data))


##The grundtvig parser##
#This software has been created to parse grundtvig center
#data into a format that is machine readible for text analysis
#Software outputs <FILENAME>.metadata for metadata
#                 <FILENAME>.parsed.txt for parsed text
#Created by Oliver Jarvis
#Organization: CHCAA
#Website: CHCAA.ai

#get list of relevant elements
#recursively extract the text until there is not left
#Start at the inner element return text at that location
#insert text into general string and then run lxml again on that text

#create a tag validator. What tags are allowed, which are dissallowed,
#pass it version number and return allowed tags and allowed children.
#

#find tags
#get tags
#create system with json files describing the structure of meta data and content

from lxml import etree, objectify
import lxml.html, lxml.html.clean
from argparse import ArgumentParser
import re
import os
import json
from pathlib import Path
#from GrundtvigTagValidator import metadata, body

class TEIParser:
    """
    Class for parsing TEI files

    Attributes
    ------------
    root_node
        the base node for the file
    filepath
        the filepath of the TEI file
    validator
        current validator for file

    Methods
    ------------
    extract_text
        gets all text within all tags starting at the first element
    find_tag
        boolean value indicating wether the element with tag exists
    is_element
        boolean value indicating whether element has tag, attribute and attribute value
    get_Elements
        returns all elements that match description
    write_json
        writes files to json
    write_raw
        writes raw document to file
    __get_inner
        scans elements
    __is_tag
        returns wether tag is actual tag
    """
    root_node = None
    def __init__(self, filename):
        parser = etree.XMLParser(remove_comments=True)
        documentTree = objectify.parse(filename, parser=parser)
        self.root_node = documentTree.getroot()
        self.filepath = filename
        self.validator = None
        self.document = dict()

    def write_raw(self, document_nodes, file_postfix, directory=""):
        filename = Path(self.filepath).stem
        filename += file_postfix + ".txt"
        file_loc = os.path.join(directory, filename)
        with open(file_loc, "w+") as f:
            for node in document_nodes:
                for c in self.document[node]:
                    f.write(c['content'] + '\n')

    def write_json(self, document, file_postfix="_metadata", directory=""):
        filename = Path(self.filepath).stem
        filename += file_postfix + ".json"
        file_loc = os.path.join(directory, filename)

        j = json.dumps(document, indent=4, ensure_ascii=False)
        f = open(file_loc, 'w')
        f.write(j)

    def extract_text(self, key, from_element):
        text = ""

        #check if there is text, if so add to text
        if from_element.text:
            text += self.clean_text(from_element.text)

        #get exclusions
        #exclusions are tags that should be skipped
        excludes = self.validator.getExcludes(key, etree.QName(from_element.tag).localname)
        replaces = self.validator.getReplaces(key, etree.QName(from_element.tag).localname)

        #get inner text from inner elements
        text += self.clean_text(self.__get_inner(from_element, excludes, replaces))

        #check if there is tail, if so add to text
        #if from_element.tail:
        #    text += self.clean_text(from_element.tail)
        return text

    def find_tag(self, root, tag_name) -> bool:
        found = False
        if not self.__is_tag(root, tag_name):
            for c in list(root):
                found = self.find_tag(c, tag_name)
                if found:
                    return True
        else:
            return True
        return False

    def is_element(self, elem, tag_name=None, attribute=None, attributeValue=None) -> bool:
        found = True
        #check through both tag_name, attribute and attributeValue for each element at the same timee
        #Adjusts output based on which elements aren't null
        for tg, at, av in zip(tag_name, attribute, attributeValue):
            if(tg and at and av):
                found =  etree.QName(elem.tag).localname == tg and (at in elem.attrib) and elem.attrib[at] == av
            elif tg and at:
                found = etree.QName(elem.tag).localname == tg and (at in elem.attrib)
            elif  tg and av:
                found = etree.QName(elem.tag).localname == tg and elem.attrib.values[0] == av
            elif at and av:
                found = elem.attrib[at] and elem.attrib.values[0] == av
            elif tg:
                found = etree.QName(elem.tag).localname == tg
            elif at:
                found = (at in elem.attrib)
            elif av:
                found = elem.attrib.values[0] == av
            if found:
                return found
        return found

##We now look through a hierarchy of selectors, until we get the right values. This let's us say e.g. sourceDesc > Date
    def get_elements(self, selector_path, root):
        nodes = root
        output_nodes = []
        for selector in selector_path:
            selector_array = [selector] #must be in array format to be parsed to find_Elements. Might not be best programming practice
            nodes = self.find_Elements(
                root = nodes,
                attribute = self.validator.getAttribute(selector_array),
                tag_name = self.validator.getTagName(selector_array),
                attributeValue = self.validator.getAttributeValue(selector_array),
                recursive = self.validator.getIsRecursive(selector_array)
                )
            if 'content_tag' in selector:
                #for ct in selector['content_tag']:
                nodes_found = self.find_Elements(
                    root = nodes,
                    tag_name = self.validator.getTagName(selector['content_tag']),
                    attribute = self.validator.getAttribute(selector['content_tag']),
                    attributeValue = self.validator.getAttributeValue(selector['content_tag']),
                    recursive = self.validator.getIsRecursive(selector['content_tag'])
                )
                output_nodes += nodes_found

        return output_nodes

    def find_Elements(self, root, tag_name=None, attribute=None, attributeValue=None, recursive=False) -> [lxml.etree.Element]:
        found_elements = []
        #go through each element in root. There may be several main tags of interest
        #e.g. get all the paragraphs and all the line groups.
        for e in root:
            if self.is_element(elem=e, tag_name=tag_name, attribute=attribute, attributeValue=attributeValue):
                found_elements += [e]
                if not recursive: #stop search if we just want the first element
                    return found_elements

            #Go through all elements recursively until they have all been found.
            for c in list(e):
                found_elements += self.find_Elements(
                    [c],
                    tag_name=tag_name,
                    attribute=attribute,
                    attributeValue=attributeValue,
                    recursive=recursive
                    )
                if(found_elements and not recursive):
                    return found_elements
        return found_elements

    def __is_tag(self, elem, tag_name):
        return etree.QName(elem.tag).localname == tag_name

    def setValidator(self, validator):
        self.validator = validator

    def clean_text(self, text):

        #if text.isspace():
        #    return " "
        #text = text.replace('\n', ' ')
        #text = text.replace('\t', ' ')

        #Only add spaces if the text had spaces to begin with
        #if not (text == text.strip()):
        #    if text.startswith(' ') or text.startswith('\t') and text.endswith(' ') or text.endswith('\t'):
        #        text = " " + text.strip() + " "
        #    elif text.startswith(' ') or text.startswith('\t'):
        #        text = " " + text.strip()
        #    elif text.endswith(' ') or text.endswith('\t'):
        #        text = text.strip() + " "
        re.sub("\ {2,}", " ", text)

        return text

    def __get_inner(self, from_element, excludes, replaces):
        text = ""

        #Go through all the child elements of from_element
        for c in list(from_element):
            #If elements has the attribute "facs" add a space to text.
            #Elements with a facs attribute tend to forego spaces between tags
            if self.is_element(c, tag_name=[None], attribute=["facs"], attributeValue=[None]):
                text += " "

            #check whether element is in the list to exclude
            #if so, start from top
            found = False
            if excludes:
                for ex in excludes:
                    found = False
                    if self.is_element(elem=c, tag_name=[ex['tag_name']], attribute=[None], attributeValue=[None]):
                        found = True
                    if found:
                        break
            if found:
                continue

            if c.text:
                text += self.clean_text(c.text)
            #recursively explore for more embedded tags
            if list(c):
                text += self.__get_inner(c, excludes, replaces)
            if c.tail:
                text += self.clean_text(c.tail)

        return text

    def parse(self, parse_text = True):
        """
        Takes our root_node and our validator and extracts all of our data into a dictionary document

        Arguments
        ---------

        clean_text
            If True will apply text cleaning function. This may remove potential
            metadata hidden in the formatting, but will make it neater to read.
        """

        if not self.validator:
            raise Exception("Missing validator")

        querySource = self.validator.getQueryTags()

        for key in querySource:
            selector_path = self.validator.getSelectorPath(key)
            elements = self.get_elements(selector_path, self.root_node)
            elements_text = []
            for e in elements:
                text = self.extract_text(key, e).strip()
                elements_text.append(
                    {
                        "tag":e,
                        "content":text
                    })

            self.document[key] = elements_text


N, Q, K = map(int, input().split(" "))
arr = list([int(x) for x in input().split(" ")])

sorts = []
for _ in range(Q):
    low, up = map(int, input().split(" "))

    if low != up:
        sorts.append([low, up])

min_k = K
max_k = K

useful_sorts = []
for segment in reversed(sorts):

    if min_k <= segment[0] <= max_k or min_k <= segment[1] <= max_k:
        min_k = min(min_k, segment[0])
        max_k = max(max_k, segment[1])
        useful_sorts.insert(0, segment)

for segment in sorts:
    low, up = segment
    arr = arr[:low] + sorted(arr[low:up + 1]) + arr[up+1 : ]

print (arr[K])


# 1. Elaborar un agente que resuelva un rompecabezas de 3X5
#   (utilice un algoritmo que permita resolverlo en menos de 1 minuto).
# Metodo de desarrollo A* con 2 heuristicas

import math
import os
import sys
import time
import copy
from queue import PriorityQueue

#UNICA CLASE YA QUE CON LA OTRA CLASE ME HICE BOLAS
class Romp:
    def __init__(self, estados, est_padre):
        self.tamano_x = 3
        self.tamano = 5
        self.estados = estados
        self.est_padre = est_padre

    def estado_act(self):
        return  self.estados

    def ejecutar_accion(self, accion):
        estado_padre = self.estados[:]
        nuevos_estados = self.estados[:]
        indice_vacio = nuevos_estados.index('0')
        if accion == 'Izquierda':
            if indice_vacio % self.tamano > 0:
                nuevos_estados[indice_vacio - 1], nuevos_estados[indice_vacio] = nuevos_estados[indice_vacio], nuevos_estados[indice_vacio - 1]
        if accion == 'Derecha':
            if indice_vacio % self.tamano < (self.tamano - 1):
                nuevos_estados[indice_vacio + 1], nuevos_estados[indice_vacio] = nuevos_estados[indice_vacio], nuevos_estados[indice_vacio + 1]
        if accion == 'Arriba':
            if indice_vacio - self.tamano >= 0:
                nuevos_estados[indice_vacio - self.tamano], nuevos_estados[indice_vacio] = nuevos_estados[indice_vacio], nuevos_estados[
                    indice_vacio - self.tamano]
        if accion == 'Abajo':
            if indice_vacio + self.tamano < self.tamano_x * self.tamano:
                nuevos_estados[indice_vacio + self.tamano], nuevos_estados[indice_vacio] = nuevos_estados[indice_vacio], nuevos_estados[
                    indice_vacio + self.tamano]
        return Romp(nuevos_estados, estado_padre)
#HEURISTICAS
def validar_1_fila(estado):
    fila = ['1', '2', '3', '4', '5']
    for i in fila:
        if fila.index(i) - estado.index(i) != 0:
            return False
    return True
def heurisitica_escalada_pos(estado):
    cont = 0
    fila1 = ['1', '2', '3', '4', '5']
    fila2 = ['6', '7', '8', '9', '10', '11', '12', '13', '14']
    #fila3 = []
    for i in fila1:
        if fila1.index(i) - estado.index(i) != 0:
            index = estado.index(i)
            #               0   /   3  - 7  /   3               0 % 5   -   7 % 5
            cont += (abs((int(i) / 5) - (index / 5)) + abs((int(i) % 5) - (index % 5)))
            cont *= 2
    for i in fila2:
        if fila2.index(i)+5 - estado.index(i) != 0:
            index = estado.index(i)
            cont += (abs((int(i) / 5) - (index / 5)) + abs((int(i) % 5) - (index % 5)))
    return cont

def euristica_escalada_distancia(estado):
    cont = 0
    fila = ['1', '2', '3', '4', '5']
    for i in fila:
        if fila.index(i) - estado.index(i) != 0:
            index = estado.index(i)
            cont += (abs((int(i) / 3) - (index / 3)) + abs((int(i) % 5) - (index % 5)))
    return cont

def distancia_cero_num(estado):
    cont = 0
    fila = ['1', '2', '3', '4', '5']
    for i in fila:
        if fila.index(i) - estado.index(i) != 0:
            index = estado.index(i)
            cont += (abs((int(i) / 3) - (index / 3)) + abs((int(i) % 5) - (index % 5)))
            index = estado.index('0')
            cont += (abs((int(i) / 3) - (index / 3)) + abs((int(i) % 5) - (index % 5)))
            return cont
    return cont

def heuristica1(estado):
    #PRIMERA LINEA DE ROMPECABEZAS PRIMERA PRIORIDAD Y MAYOR PUNTUACION
    cont = 0
    fila = ['1', '2', '3', '4', '5']
    for i in fila:
        if fila.index(i) - estado.index(i) != 0:
            cont += 3
    return cont

def heuristica2(estado):
    #CUADRANTE 1
    cont = 0
    linea1 = ['6', '7']
    linea2 = ['11', '12']
    for i in linea1:
        if linea1.index(i)+5 - estado.index(i) != 0:
            cont += 2
    for i in linea2:
        if linea2.index(i)+10 - estado.index(i) != 0:
            cont += 2
    return cont

def heurisitca_escuadra(estado):
    cont = 0
    esc_val = ['1', '2', '3', '6', '11']
    ind_val = [0, 1, 2, 5, 10]
    j = 0
    for i in esc_val:
        if (ind_val[j] - estado.index(i)) != 0:
            cont += 1
            j += 1
    return cont

#HEURISTICA ROBADA
def manhattan_calculate(estados):
    '''heuristicaa Manhattan: cuenta el numero de cuadros a partir de una ubicacion en relacion a su posicion final'''
    contador = 0
    for i in range(0, 14):
        index = estados.index(str(i + 1))  # because range starts at 0
        contador += (abs((i / 3) - (index / 3)) + abs((i % 5) - (index % 5)))  # %4 is the column and /4 is the row
    return contador

def hamming(estados):
    ''' heuristicaa Hamming: cuenta el numero de posiciones erroneas en diferentes estados'''
    distancia = 0
    objetivo_estados = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14']
    for i in objetivo_estados:
        if objetivo_estados.index(i) - estados.index(i) != 0 and i != 0:
            distancia += 1
    return distancia

def heuristica_Combinada(estado):
    c = 0
    c += heurisitca_escuadra(estado)
    #c += heuristica1(estado)
    #c += heurisitica_escalada_pos(estado)
    #c += hamming(estado)
    c += manhattan_calculate(estado)
    return c

#AQUI SE GENERARN LOS BEBES
def generar_hijos(padre):
    hijos = []
    acciones = ['Arriba', 'Abajo', 'Derecha', 'Izquierda']
    for accion in acciones:
        hijo_estado = padre.ejecutar_accion(accion)
        hijo_Nodo = Romp(hijo_estado.estados, padre)
        hijos.append(hijo_Nodo)
    return hijos

#UTILIZAMOS EL MISMO QUE EL DEL EJERCICIO
def gcalc(Nodo):
    ''' calcula g(n): encuentra el costo del estado actual a partir del estado origen o inicial'''
    contador = 0
    while Nodo.est_padre is not None:
        Nodo = Nodo.est_padre
        contador += 1
    return contador

def objetivo():
    return ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '0']

def num_str1(x):
    if len(x)<2:
        return (f" {x}")
    return x

def mostrar_pasos(paso):
    '''Retorna el Proceso'''
    pasos = []
    while (paso.est_padre is not None):
        pasos.append(paso.estados)
        paso = paso.est_padre
    pasos.reverse()
    for i in pasos:
        j = 0
        while(j <14):
            print(f"| {num_str1(i[j])} {num_str1(i[j+1])} {num_str1(i[j+2])}"
                  f" {num_str1(i[j+3])} {num_str1(i[j+4])} |")
            j += 5
        print('--*--*--*--*--*--*')

def a_estrella(estado_inicial, estado_objetivo):
    '''A* Search Algorithm'''
    start_time = time.time()
    frontera = list()
    contador = 0
    obj_visitado = list()
    est_visitado = list()
    frontera.append(estado_inicial)
    obj_visitado.append(estado_inicial)
    est_visitado.append(estado_inicial.estados)
    while frontera:
        valor_Mov = []
        movimientos_pos = []
        for x in frontera:
            #CAMBIAR A UNA SOLA H
            valor_Mov.append(heuristica_Combinada(x.estados) + gcalc(x))
            #HASTA ACA
            movimientos_pos.append(x)

        m = min(valor_Mov)  # finds minimum F value
        estado_inicial = movimientos_pos[valor_Mov.index(m)]

        #print(estado_inicial.estados)
        if estado_inicial.estados == estado_objetivo:  # solution found!
            end_time = time.time()

            print("Solucion:\n\n")
            mostrar_pasos(estado_inicial)

            #print("Movimientos: " + str(' '.join(find_path(estado_inicial))))
            print("Numero de nodos expandidos: " + str(contador))
            print("Tiempo empleado: " + str(round((end_time - start_time), 3)))
            # print("Memory Used: " + str(sys.gettamanoof(visitado) + sys.gettamanoof(frontera)) + " kb")
            break

        frontera.pop(frontera.index(estado_inicial))
        for hijo in generar_hijos(estado_inicial):
            contador += 1
            s = hijo.estados
            #print(s)

            if s not in est_visitado or gcalc(hijo) <= gcalc(obj_visitado[est_visitado.index(s)]):
                est_visitado.append(s)
                obj_visitado.append(hijo)
                frontera.append(hijo)

                # if len(est_visitado) > 30:
                #     est_visitado.pop(0)
                #     obj_visitado.pop(0)
        if time.time() > start_time + 60:
            mostrar_pasos(estado_inicial)
            break

def main():
    # NO REALIZABLES
    #ei = ['2', '14', '3', '5', '1', '6', '8', '12', '4', '0', '13', '7', '11', '9', '10']
    #ei = ['1', '3', '4', '8', '5', '2', '0', '6', '9', '10', '7', '11', '13', '14', '12']
    #ei = ['0', '1', '2', '3', '5', '6', '7', '4', '8', '9', '10', '11', '12', '13', '14']
    # SI REALIZABLES
    #ei = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '0', '11', '12', '13', '14']
    #ei = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '11', '12', '13', '14', '10']

    #ei = ['0', '1', '2', '3', '5', '6', '7', '8', '4', '10', '11', '12', '13', '9', '14']
    ei = ['0', '1', '2', '3', '4', '7', '8', '9', '10', '5', '6', '11', '12', '13', '14']
    a_estrella(Romp(ei, None), objetivo())

    print('----------------FIN----------------')

if __name__ == '__main__':
    main()