import argparse
import json
import random
import os.path as osp

import torch
import torch.nn.functional as F

from utils import ensure_path, set_gpu, l2_loss
from models.gcn_dense_att import GCN_Dense_Att

def save_checkpoint(name):
    torch.save(gcn.state_dict(), osp.join(save_path, name + '.pth'))
    torch.save(pred_obj, osp.join(save_path, name + '.pred'))

def mask_l2_loss(a, b, mask):
    return l2_loss(a[mask], b[mask])

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    #创建对象
    parser.add_argument('--max-epoch', type=int, default=3000)#添加参数，这些调用指定 ArgumentParser 如何获取命令行字符串并将其转换为对象。这些信息在 parse_args() 调用时被存储和使用
    parser.add_argument('--trainval', default='10,0')#训练集和测试集占得比例
    parser.add_argument('--lr', type=float, default=0.001)
    parser.add_argument('--weight-decay', type=float, default=0.0005)
    #weight-decay权值衰减，在损失函数中，他是放在正则项前面的系数，为了调节模型复杂度对损失函数的影响
    parser.add_argument('--save-epoch', type=int, default=300)
    parser.add_argument('--save-path', default='save/gcn-dense-att')

    parser.add_argument('--gpu', default='0')

    parser.add_argument('--no-pred', action='store_true')#action命令行遇到参数时默认store
    args = parser.parse_args()
    #解析添加的参数，把每个参数转换为适当的类型然后调用相应的操作
    set_gpu(args.gpu)

    save_path = args.save_path
    ensure_path(save_path)

    graph = json.load(open('materials/imagenet-dense-grouped-graph.json', 'r'))#json是用来存储简单的数据结构和对象的文件
    wnids = graph['wnids']
    n = len(wnids)

    edges_set = graph['edges_set']
    print('edges_set', [len(l) for l in edges_set])

    lim = 4#每个阶段k值的数量设置为4
    for i in range(lim + 1, len(edges_set)):
        edges_set[lim].extend(edges_set[i])
    edges_set = edges_set[:lim + 1]
    print('edges_set', [len(l) for l in edges_set])#这里将14层减少到了4层+上自环，将后面的层全部加到第4层。后面计算权重的时候将他们算作一层

    word_vectors = torch.tensor(graph['vectors']).cuda()
    word_vectors = F.normalize(word_vectors)#对输入数据进行标准化使得输入数据满足正态分布，使得网络更容易收敛
    #fc为全连接层
    fcfile = json.load(open('materials/fc-weights.json', 'r'))
    train_wnids = [x[0] for x in fcfile]
    fc_vectors = [x[1] for x in fcfile]
    assert train_wnids == wnids[:len(train_wnids)]
    fc_vectors = torch.tensor(fc_vectors).cuda()#cnn训练的权重
    fc_vectors = F.normalize(fc_vectors)#F.normalize将某一个维度除以那个维度对应的范数(默认是2范数)。

    hidden_layers = 'd2048,d'
    gcn = GCN_Dense_Att(n, edges_set,
                        word_vectors.shape[1], fc_vectors.shape[1], hidden_layers).cuda()#训练dgp来预测最后一层的cnn权值，最终输出维数对应于resnrt-50架构的最后一层
    #对几个邻接矩阵进行处理

    print('word vectors:', word_vectors.shape)
    print('fc vectors:', fc_vectors.shape)
    print('hidden layers:', hidden_layers)

    optimizer = torch.optim.Adam(gcn.parameters(), lr=args.lr, weight_decay=args.weight_decay)

    v_train, v_val = map(float, args.trainval.split(','))
    n_trainval = len(fc_vectors)
    n_train = round(n_trainval * (v_train / (v_train + v_val)))#round四舍五入函数
    print('num train: {}, num val: {}'.format(n_train, n_trainval - n_train))
    tlist = list(range(len(fc_vectors)))
    random.shuffle(tlist)#将原列表打乱顺序,这里是将1000打乱顺序
    min_loss = 1e18

    trlog = {}
    trlog['train_loss'] = []
    trlog['val_loss'] = []
    trlog['min_loss'] = 0

    for epoch in range(1, args.max_epoch + 1):
        gcn.train()
        output_vectors = gcn(word_vectors)
        # z=output_vectors[tlist[0]]
        # o=fc_vectors[tlist[0]]
        loss = mask_l2_loss(output_vectors, fc_vectors, tlist[:n_train])#对输出进行l2正则化
        #训练DGP来训练每个节点的权值，训练类的权重从一个预先训练好的resnet网络优化提取，训练模型，通过损失来预测所见类的分类器权重
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        #上面的loss是为了进行反向传播进行参数的更新，而下面的train_loss是为了观察训练时损失的下降情况

        gcn.eval()
        output_vectors = gcn(word_vectors)#这里进行了测试是为了记录训练时候的每一步的损失么？？
        train_loss = mask_l2_loss(output_vectors, fc_vectors, tlist[:n_train]).item()
        if v_val > 0:
            val_loss = mask_l2_loss(output_vectors, fc_vectors, tlist[n_train:]).item()#这里是为了干什么的，与上面的train_loss不是一样的么
            loss = val_loss
        else:
            val_loss = 0
            loss = train_loss
        print('epoch {}, train_loss={:.4f}, val_loss={:.4f}'
              .format(epoch, train_loss, val_loss))

        trlog['train_loss'].append(train_loss)
        trlog['val_loss'].append(val_loss)
        trlog['min_loss'] = min_loss
        torch.save(trlog, osp.join(save_path, 'trlog'))

        if (epoch % args.save_epoch == 0):
            if args.no_pred:
                pred_obj = None
            else:
                pred_obj = {
                    'wnids': wnids,
                    'pred': output_vectors#个人理解：通过gcn传出来的一个分类器
                }

        if epoch % args.save_epoch == 0:
            save_checkpoint('epoch-{}'.format(epoch))

        pred_obj = None
    #一共运行3000次，也就是进行3000次参数优化，每次进行一次loss,
    #这里有一个疑问，就是我每次输出的output_vectors跟fc_vector进行损失计算都是取前1000个,那么后面的就不会参与更新
    #那我最后取的pred就必须也取前1000个参数才可以


'''
1) Imprimir en la terminal
    dos enteros en una misma linea
    dos flotantes en una misma linea
    dos strings en una misma linea
    dos booleanos en una misma linea
    dos listas en una misma linea
    dos tuplas en una misma linea
    un diccionario en una misma linea
    un conjunto en una misma linea
'''
print(4, 8)
print(0.8, 0.6)
print("Corozo", "3.1416")
print(2>5, "fifi" == "fifi")
a = [1, 3, 5, 7]
b = [2, 4, 6, 8]
print(a, b)
c = (1, 4)
d = (2, 3)
print(c, d)
e = {"Lunes": "Monday"}
print(e)
f = {"A", "E", "I", "O", "U"}
print(f)

"2) Resolver con los operadores que considere conveniente"

""" Dadas las coordenadas P1(5,4,5) y P2(0,10,9).
Realice un codigo que determine la distancia entre ambos puntos """

import math

def distancia_entre_puntos(x1, y1, z1, x2, y2, z2):
    distancia = math.sqrt((x2-x1)**2 + (y2-y1)**2 + (z2-z1)**2)
    return distancia

distancia = distancia_entre_puntos(5, 4, 5, 0, 10, 9)
print(distancia)

""" La calificación de informatica se encuentra en el intervalo [0,5] y se calcula tomando 4 notas,
con porcentajes de 15%, 25%, 20% y 40%. Si un estudiante tiene las primeras 3 calificaciones definidas.
Realice un algoritmo que calcule la nota necesaria de la última nota para pasar la materia. """

nota1 = float(input("Ingrese su primera nota: "))
nota2 = float(input("Ingrese su segunda nota: "))
nota3 = float(input("Ingrese su tercera nota: "))

promedio = (nota1 * 0.15) + (nota2 * 0.25) + (nota3 * 0.20)

if promedio >= 3:
    print("La materia ya está aprobada, con un promedio de ", promedio)
else:
    nota4 = (3 - promedio)/0.40
    print("Para aprobar la materia, necesitas sacar en el 4to parcial una nota de ", nota4)

""" Dos automoviles realizan una carrera. El primero de ellos acelera a razón constante de 3m/s², el segundo
a razón de 5m/s². Si el segundo de ellos empieza su recorrido 10 segundos después que el primero ha empezado.
¿Cuanto tiempo habrá transcurrido cuando ambos se encuentran? """

v1 = 0
a1 = 3
v2 = 0
a2 = 5
t = 10
d1 = v1*t + 0.5*a1*t**2
d2 = v2*(t-10)+0.5*a2*(t-10)**2
dtotal = d1-d2
tencuentro = (2*dtotal/(a1 + a2))**0.5
ttotal = t+tencuentro
print("Los 2 automóviles se encontraran después de ", ttotal, " s")

""" Cuatro compañeros, contratan un taxi con el objeto de movilizarse juntos a la universidad.
El contrato es de lunes a viernes, y deben pagar al taxista $15000 por cada trayecto. El servicio se
prestará solo de ida.
Sin embargo, los cuatro no se movilizan juntos todos los dias. Por tanto, han planteado que la tarifa
debe dividirse entre el numero de compañeros que se movilicen en cada trayecto.En caso, de que ninguno
utilice el servicio. Deben pagar al taxista una tarifa de $10000, repartidos equitativamente entre los cuatro.
A continueación veamos el uso del servicio por parte de los compañeros en la última semana de clases:
            LUNES   MARTES  MIERCOLES   JUEVES  VIERNES
JUAN          Si        Si        No        Si    No
CAMILA        Si        Si        No        No    No
JOSE          Si        No        Si        No    No
MARIA         Si        Si        Si        No    No
¿Cuanto debe pagar cada estudiante? """

uso_servicio = [[True, True, False, True, False],
                [True, True, False, False, False],
                [True, False, True, False, False],
                [True, True, True, False, False]]

tarifa_sin_uso = 10000
tarifa_con_uso = 15000

costo_juan = 0
costo_camila = 0
costo_jose = 0
costo_maria = 0

for dia in range(5):
    num_usuarios = 0
    if uso_servicio[0][dia]:
        num_usuarios += 1
    if uso_servicio[1][dia]:
        num_usuarios += 1
    if uso_servicio[2][dia]:
        num_usuarios += 1
    if uso_servicio[3][dia]:
        num_usuarios += 1

    if num_usuarios == 0:
        costo_juan += tarifa_sin_uso / 4
        costo_camila += tarifa_sin_uso / 4
        costo_jose += tarifa_sin_uso / 4
        costo_maria += tarifa_sin_uso / 4
    else:
        costo_por_usuario = tarifa_con_uso / num_usuarios
        if uso_servicio[0][dia]:
            costo_juan += costo_por_usuario
        if uso_servicio[1][dia]:
            costo_camila += costo_por_usuario
        if uso_servicio[2][dia]:
            costo_jose += costo_por_usuario
        if uso_servicio[3][dia]:
            costo_maria += costo_por_usuario

print("Costo total para Juan: $", costo_juan)
print("Costo total para Camila: $", costo_camila)
print("Costo total para Jose: $", costo_jose)
print("Costo total para Maria: $", costo_maria)


# import itertools
# import math

# def is_prime(n):
#     for i in range(2, int(math.sqrt(n))+1): # 제곱근을 정수화 시켜준 후 +1
#         if n % i == 0:
#             return False
#     return True

# def solution(numbers):
#     count = 0

#     set_pmt = list(set(list(map(''.join, itertools.permutations(numbers)))))
#     print(set_pmt)

#     # 1. 순열이 소수라면 count+1
#     for x in set_pmt:
#         if is_prime(int(x)) == True:
#             count +=1

#     # 2. 투 포인터로 확인
#     n_list = list(map(''.join, numbers))
#     print(n_list)

#     result = 0
#     summary = False
#     end = 1

#     for start in range(len(n_list)):
#         tmp = n_list[start] + n_list[end]
#         print(tmp)
#         # summary = is_prime(int())
#         while (summary != True) and end < len(n_list):
#             end += 1
#         # 부분 합이 m일 때 카운트 증가
#         if summary == True:
#             result += 1
#         summary = False

#     print(result)

#     return count

# 투포인터는 합을 구할 때 쓰이는데
# 이건 조합을 구하는 문제라서 쓰면 안됐던 것 같다
# 이렇게 접근하는 것 보다 슬라이싱을 이용하고 비교를 하는게 맞는 것 같다\

import itertools
import math

def is_prime(n):
    for i in range(2, int(math.sqrt(n))+1): # 제곱근을 정수화 시켜준 후 +1
        if n % i == 0:
            return False
    return True

def solution(numbers):
    count = 0
    # 1. 각

    return count


n = int(input())
tri = [None]*n
for i in range(n):
    tri[i] = list(map(int, input().split()))

# dp - tab
tab = [None for _ in range(n)]
for i in range(n):
    tab[i] = [0]*(i+1)

def dp_tab():
    tab[0][0] = tri[0][0] # 맨위 삽입
    for x in range(1, n):
        for y in range(len(tri[x])):
            km = []
            if y-1 >= 0:
                km.append(tab[x-1][y-1])
            if y < len(tri[x])-1:
                km.append(tab[x-1][y])
            tab[x][y] = tri[x][y] + max(km)

dp_tab()

print(max(tab[-1]))


import turicreate as tc
import os

m=tc.load_model('model.model')
data=tc.image_analysis.load_images('test')

p=m.predict(data['image'], confidence_threshold=0.01)

res=tc.one_shot_object_detector.util.draw_bounding_boxes(data['image'],p)
res.explore()

if not os.path.isdir('out'):
    os.makedirs('out')
for i,img in enumerate(res):
    img.save("out/out_%s.png" % i)


import pytest

import clique.descriptor

class Mock(object):
    '''Mock class for test.'''

    x = clique.descriptor.Unsettable('x')

def test_unsettable():
    '''Unsettable descriptor prevent standard setting of attribute.'''
    instance = Mock()
    assert instance.x is None

    instance.__dict__['x'] = True
    assert instance.x is True

    with pytest.raises(AttributeError):
        instance.x = False


from __future__ import print_function
import boto3
import json
import os
from time import sleep
from pprint import pprint

RISKY_PORTS = [20, 21, 22, 1433, 1434, 3306, 3389, 4333, 5432, 5500]

template = """
Security Group Alert !
Alert Level : {0}
Account: {1}
Region: {2}
Group Id: {3}
Detail: {4}

"""

def aws_get_regions(aws_key, aws_secret):
    aws_client = boto3.client(
        "ec2",
        aws_access_key_id=aws_key,
        aws_secret_access_key=aws_secret,
        region_name='eu-west-1'
    )
    response = aws_client.describe_regions()
    result = []
    for region in response['Regions']:
        result.append(region['RegionName'])
    return result

def aws_get_security_groups(aws_key, aws_secret, test_sg=""):
    # TODO: Requires Unit Testing
    global message
    for region in aws_get_regions(aws_key, aws_secret):
        aws_client = boto3.client(
            "ec2",
            aws_access_key_id=aws_key,
            aws_secret_access_key=aws_secret,
            region_name=region
        )
        message = ""

        response = aws_client.describe_security_groups()
        account = response['SecurityGroups'][0]['OwnerId']
        for security_group in response['SecurityGroups']:
            # message = ""
            for ip_permissions in security_group['IpPermissions']:
                pprint(security_group)
                affected_ports = []
                if ip_permissions['IpProtocol'] == '-1' and '0.0.0.0/0' in str(ip_permissions['IpRanges']) and \
                        (test_sg == '' or security_group['GroupId'] == test_sg):
                    message += template.format('Critical', account, region, security_group['GroupId'],
                                               'All ports open to the world !')
                if ip_permissions['IpProtocol'] == 'tcp' and '0.0.0.0/0' in str(ip_permissions['IpRanges']) and \
                        (test_sg == '' or security_group['GroupId'] == test_sg):
                    from_port = ip_permissions['FromPort']
                    to_port = ip_permissions['ToPort']
                    for port in RISKY_PORTS:
                        if from_port <= port <= to_port:
                            affected_ports.append(port)
                    if len(affected_ports) > 0:
                        message += template.format('Important', account, region, security_group['GroupId'],
                                                   'TCP port(s) ' + ",".join(
                                                       str(x) for x in affected_ports) + ' open to the world !')
                    else:
                        message += template.format('Warning', account, region, security_group['GroupId'],
                                                   'TCP port' + (
                                                       ' ' + str(to_port) if to_port == from_port else 's from ' + str(
                                                           from_port) + ' to ' + str(to_port)) + ' open to the world !')
                if message != "":
                    print(message)
                    aws_send_sns_message(message, os.environ.get('SNS_ARN'), aws_key, aws_secret)

                if security_group['GroupId'] == test_sg:
                    return message

        sleep(1)  # Not to overload AWS API
    return message

def aws_send_sns_message(message, topic_arn, aws_key, aws_secret):
    # type: (string, string, string, string) -> string
    # TODO: Requires Unit Testing
    topic_keys = topic_arn.split(':', 5)
    region = topic_keys[3]
    aws_client = boto3.client(
        "sns",
        aws_access_key_id=aws_key,
        aws_secret_access_key=aws_secret,
        region_name=region
    )
    response = aws_client.publish(
        TopicArn=os.environ.get('SNS_ARN'),
        Message=message
    )
    return response


from .BasePostingCrawler import BasePostingCrawler
from Utilities import Network

class AkamaiPostingCrawler(BasePostingCrawler):
    def __init__(self, posting):
        super().__init__(posting)

    def get_description(self):
        raw = ''
        link = self.posting.get_url()
        for i in range(4):
            response = Network.get_html(link, allow_redirects=False)
            link = response.headers['Location']
        page = Network.get_page(link)
        self.posting.set_page(page)
        soup = self.posting.get_soup()
        tags = soup.find_all('p', style="margin-top:0px;margin-bottom:0px")
        for tag in tags:
            raw += tag.text + '\n'
        return raw


# DFS, BFS, Matrix
# Similar to Island Problem

class Solution:
    def hasPath(self, maze: List[List[int]], start: List[int], destination: List[int]) -> bool:

        m = len(maze)
        n = len(maze[0])

        def dfs(maz: List[List[int]], i: int, j: int, visited: List[List[bool]]) -> None:
            if visited[i][j]:
                return
            visited[i][j] = True
            l = j - 1
            r = j + 1
            u = i - 1
            d = i + 1
            while l >= 0 and maz[i][l] == 0:
                l -= 1
            l += 1
            while r < n and maz[i][r] == 0:
                r += 1
            r -= 1
            while u >= 0 and maz[u][j] == 0:
                u -= 1
            u += 1
            while d < m and maz[d][j] == 0:
                d += 1
            d -= 1
            dfs(maz, i, l, visited)
            dfs(maz, i, r, visited)
            dfs(maz, u, j, visited)
            dfs(maz, d, j, visited)

        visited = [[False for _ in range(n)] for _ in range(m)]
        dfs(maze, start[0], start[1], visited)

        return visited[destination[0]][destination[1]]


import tensorflow as tf
import tensorflow.contrib.slim as slim

import collections

class MLPModel(object):
  def __init__(self, config):
    self.config = config
    super(MLPModel, self).__init__()

  def arg_scope(self, reuse=None, is_training=None):
    with slim.arg_scope([slim.fully_connected],
                        weights_initializer=tf.truncated_normal_initializer(
                            stddev=0.01),
                        weights_regularizer=slim.l2_regularizer(1e-4),
                        activation_fn=tf.nn.relu,
                        reuse=reuse) as sc:
        return sc

  def compute(self, inputs):
    inputs = tf.reshape(inputs, shape=[-1, self.config.INPUT_DIM])
    outputs = slim.fully_connected(inputs, 20, scope="fc_0")
    outputs = slim.fully_connected(inputs, 20, scope="fc_1")
    outputs = slim.fully_connected(inputs, 20, scope="fc_2")
    outputs = slim.fully_connected(
        outputs, self.config.NUM_ACTIONS, activation_fn=None, scope="softmax")
    return outputs


# coding=utf-8
__author__ = 'chenlei'

from django import template
from django.utils.translation import ugettext as _

register = template.Library()

@register.filter
def bool_map(value):
    if value not in [True, False]:
        raise Exception("Bool Map Value should be a bool value")
    if value:
        return _(u"是")
    return _(u"否")


import sys
n = int(input())

mem_limit = 3*(10**6)
while True:
  st = sys.stdin.read(mem_limit)
  if len(st) == mem_limit:
    while True:
      app = sys.stdin.read(1)
      if app == "\n":
        break
      else:
        st += app
  else:
    if st in ["\n", ""]:
      break
  A = list(map(int, st.split()))

  for a, b in zip(A[::2], A[1::2]):
    print(abs(a)//abs(b)*(1-((a>0)^(b>0))*2))


from netsquid_simulationtools.parameter_set import Parameter, ParameterSet
from netsquid_nv.nv_state_delivery_model import prob_nuclear_dephasing_during_entgen_attempt
import numpy as np

def _gaussian_dephasing_fn(x):
    return 0.5 * (1. + np.exp(-0.5 * x ** 2))

def _inverse_gaussian_dephasing_fn(x):
    if x == 0.5:
        return np.inf
    elif x == 1.:
        return 0.
    else:
        return np.sqrt(-2. * np.log(2 * x - 1))

def _coherence_time_prob_fn(x):
    return np.exp(-1. / x)

def _inverse_coherence_time_prob_fn(x):
    if x == 1.:
        return np.inf
    elif x == 0.:
        return 0
    else:
        return - 1. / np.log(x)

def compute_product_tau_decay_delta_w_from_dephasing_prob(prob_deph, alpha):
    r"""Inverse of
    :fn:`~netsquid_nv.nv_state_delivery_model.prob_nuclear_dephasing_during_entgen_attempt`.

    Parameters
    ----------
    prob_deph :
        Probability of dephasing of a nuclear spin during a single entanglement
        generation attempt with the electron spin.
    alpha : float
        Bright state parameter

    Returns
    -------
    float
        Product of :math:`\tau` and :math:`\Delta \omega`.
    """
    log_content = 1 - (2 * prob_deph / (1 - alpha))
    return np.sqrt(-2 * np.log(log_content))

def compute_dephasing_prob_from_nodephasing_number(nodephasing_number):
    r"""Equivalent of
    :fn:`~netsquid_nv.nv_state_delivery_model.prob_nuclear_dephasing_during_entgen_attempt`,
    but in this case not the product of :math:`\tau` and :math:`\delta \omega` is specified,
    but instead one gives the "nondephasing number" :math:`N_{1/e}`, i.e. the number of entanglement attempts
    on the electron spins that make a nuclear qubit in the :math:`|+\rangle\langle+|` state
    reduce its Bloch sphere length from 1 to :math:`1/e` (where `e` is the natural constant).

    Parameters
    ----------
    nondephasing_number : float

    Returns
    -------
    float
        Dephasing probability of a carbon nuclear spin during a single entanglement
        generation attempt.

    Notes
    -----
    This function returns the probability :math:`p` for which the Bloch vector length of
    :math:`|+\rangle\langle +|` after `n` applications of the single-qubit dephasing channel,
    :math:`(1 - 2p)^n`, equals `1/e`.
    """
    return (1 - np.exp(-1 / nodephasing_number)) / 2

def compute_product_tau_decay_delta_w_from_nodephasing_number(nodephasing_number, alpha):
    r"""Equivalent of
    :fn:`~netsquid_nv.nv_parameter_set.compute_product_tau_decay_delta_w_from_dephasing_prob`
    where now the number :math:`N_{1/e}` is used as input, which is the number of entanglement
    attempts after which the Bloch vector length of :math:`|+\rangle\langle +|` has decreased
    to :math:`1/e`.

    Parameters
    ----------
    nondephasing_number : float
    alpha : float
        Bright state parameter

    Returns
    -------
    float
        Product of :math:`\tau` and :math:`\Delta \omega`.
    """
    prob_deph = compute_dephasing_prob_from_nodephasing_number(nodephasing_number)
    return compute_product_tau_decay_delta_w_from_dephasing_prob(
        prob_deph=prob_deph,
        alpha=alpha)

def compute_nodephasing_number_from_dephasing_probability(prob_deph):
    """Inverse of
    :fn:`~netsquid_nv.nv_parameter_set.compute_dephasing_prob_from_nodephasing_number`.
    """
    return -1 / np.log(1 - 2 * prob_deph)

class NVParameterSet(ParameterSet):

    _REQUIRED_PARAMETERS = [

        Parameter(name="visibility",
                  units=None,
                  perfect_value=1.,
                  type=float,
                  convert_to_prob_fn=lambda x: x,
                  convert_from_prob_fn=lambda x: x),

        # The probability of no dark counts at all
        # is computed as `exp(-1. * dark_count_rate * time_window)`
        # and hence the probability of having at least a single dark count
        # equals 1 minus this number.
        Parameter(name="prob_dark_count",
                  units=None,
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: 1 - x,
                  convert_from_prob_fn=lambda x: 1 - x),

        # The standard deviation of the phase on the produced state
        # caused by drift on the interferometer is turned into a
        # dephasing map with the following probability of no-dephasing:
        # `0.5 * (1 + exp(-0.5 * std_phase ** 2))`
        Parameter(name="std_electron_electron_phase_drift",
                  units="radians",
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=_gaussian_dephasing_fn,
                  convert_from_prob_fn=_inverse_gaussian_dephasing_fn),

        Parameter(name="p_double_exc",
                  units=None,
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: 1 - x,
                  convert_from_prob_fn=lambda x: 1 - x),

        Parameter(name="photon_emission_delay",
                  units="ns",
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=None,
                  convert_from_prob_fn=None),

        # The probability of the carbon memory qubit
        # *not* dephasing is given by
        #  1 / 2 * (1 + np.exp(-(some_constant_factor * delta_w * tau_decay) ** 2 / 2))
        Parameter(name="product_tau_decay_delta_w",
                  units="None, product of delta_w (units:Hz, i.e. without factor 1/2pi)" +
                  " & tau_decay (units:seconds)",
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: 1 - prob_nuclear_dephasing_during_entgen_attempt(alpha=0.5, delta_w=1, tau_decay=x),
                  convert_from_prob_fn=lambda x: compute_product_tau_decay_delta_w_from_dephasing_prob(1 - x, alpha=0.5)),

        Parameter(name="electron_init_depolar_prob",
                  units=None,
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: 1 - x,
                  convert_from_prob_fn=lambda x: 1 - x),

        Parameter(name="electron_single_qubit_depolar_prob",
                  units=None,
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: 1 - x,
                  convert_from_prob_fn=lambda x: 1 - x),

        Parameter(name="prob_error_0",
                  units=None,
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: 1 - x,
                  convert_from_prob_fn=lambda x: 1 - x),

        Parameter(name="prob_error_1",
                  units=None,
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: 1 - x,
                  convert_from_prob_fn=lambda x: 1 - x),

        Parameter(name="carbon_init_depolar_prob",
                  units=None,
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: 1 - x,
                  convert_from_prob_fn=lambda x: 1 - x),

        Parameter(name="carbon_z_rot_depolar_prob",
                  units=None,
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: 1 - x,
                  convert_from_prob_fn=lambda x: 1 - x),

        Parameter(name="ec_gate_depolar_prob",
                  units=None,
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: 1 - x,
                  convert_from_prob_fn=lambda x: 1 - x),

        Parameter(name="electron_T1",
                  units="nanoseconds",
                  perfect_value=np.inf,
                  type=float,
                  convert_to_prob_fn=_coherence_time_prob_fn,
                  convert_from_prob_fn=_inverse_coherence_time_prob_fn),

        Parameter(name="electron_T2",
                  units="nanoseconds",
                  perfect_value=np.inf,
                  type=float,
                  convert_to_prob_fn=_coherence_time_prob_fn,
                  convert_from_prob_fn=_inverse_coherence_time_prob_fn),

        Parameter(name="carbon_T1",
                  units="nanoseconds",
                  perfect_value=np.inf,
                  type=float,
                  convert_to_prob_fn=_coherence_time_prob_fn,
                  convert_from_prob_fn=_inverse_coherence_time_prob_fn),

        Parameter(name="carbon_T2",
                  units="nanoseconds",
                  perfect_value=np.inf,
                  type=float,
                  convert_to_prob_fn=_coherence_time_prob_fn,
                  convert_from_prob_fn=_inverse_coherence_time_prob_fn),

        # The transmissionless detection efficiency is the product of THREE probabilities:
        # - the probability that a photon is emitted from the electron spin into
        #   the zero-phonon-line
        # - the probability that the photon is collected into the fibre
        # - the probability that the detector clicks, given that a photon entered it
        # The transmittivity (i.e. the probability that the photon survives the
        # travelling through the fibre) is explicitly NOT included in this number
        # since the transmittivity is dependent on the length of the fibre.
        Parameter(name="prob_detect_excl_transmission_with_conversion_with_cavities",
                  units=None,
                  perfect_value=1.,
                  type=float,
                  convert_to_prob_fn=lambda x: x,
                  convert_from_prob_fn=lambda x: x),

        Parameter(name="prob_detect_excl_transmission_no_conversion_no_cavities",
                  units=None,
                  perfect_value=1.,
                  type=float,
                  convert_to_prob_fn=lambda x: x,
                  convert_from_prob_fn=lambda x: x),

        Parameter(name="coherent_phase",
                  units="radians",
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=None,
                  convert_from_prob_fn=None),

        Parameter(name="p_fail_class_corr",
                  units=None,
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=lambda x: x,
                  convert_from_prob_fn=lambda x: x),

        Parameter(name="initial_nuclear_phase",
                  units="radians",
                  perfect_value=0.,
                  type=float,
                  convert_to_prob_fn=_gaussian_dephasing_fn,
                  convert_from_prob_fn=_inverse_gaussian_dephasing_fn),

        # the probability that a photon is lost in the fibre
        Parameter(name="p_loss_lengths_with_conversion",
                  units="dB/km",
                  perfect_value=0.,
                  type=float,
                  # we assume that the transmittivity in the fibre
                  # as given and that it cannot be improved.
                  # We thus set the functions for conversion
                  # to a 'probability of no-error' and back to None
                  convert_to_prob_fn=None,
                  convert_from_prob_fn=None),

        Parameter(name="p_loss_lengths_no_conversion",
                  units="dB/km",
                  perfect_value=0.,
                  type=float,
                  # we assume that the transmittivity in the fibre
                  # as given and that it cannot be improved.
                  # We thus set the functions for conversion
                  # to a 'probability of no-error' and back to None
                  convert_to_prob_fn=None,
                  convert_from_prob_fn=None)]


import time
import torch
import numpy as np
from copy import deepcopy
from argparse import ArgumentParser
from torch.utils.data import DataLoader

from .incremental_learning import Inc_Learning_Appr
from datasets.exemplars_dataset import ExemplarsDataset

class Appr(Inc_Learning_Appr):
    """Class implementing the Bias Correction (BiC) approach described in
    http://openaccess.thecvf.com/content_CVPR_2019/papers/Wu_Large_Scale_Incremental_Learning_CVPR_2019_paper.pdf
    Original code available at https://github.com/wuyuebupt/LargeScaleIncrementalLearning
    """

    def __init__(self, model, device, nepochs=250, lr=0.1, lr_min=1e-5, lr_factor=3, lr_patience=5, clipgrad=10000,
                 momentum=0.9, wd=0.0002, multi_softmax=False, wu_nepochs=0, wu_lr_factor=1, fix_bn=False,
                 eval_on_train=False, logger=None, exemplars_dataset=None, val_exemplar_percentage=0.1,
                 num_bias_epochs=200, T=2, lamb=-1):
        # Sec. 6.1. CIFAR-100: 2,000 exemplars, ImageNet-1000: 20,000 exemplars, Celeb-10000: 50,000 exemplars
        # Sec. 6.2. weight decay for CIFAR-100 is 0.0002, for ImageNet-1000 and Celeb-10000 is 0.0001
        super(Appr, self).__init__(model, device, nepochs, lr, lr_min, lr_factor, lr_patience, clipgrad, momentum, wd,
                                   multi_softmax, wu_nepochs, wu_lr_factor, fix_bn, eval_on_train, logger,
                                   exemplars_dataset)
        self.val_percentage = val_exemplar_percentage
        self.bias_epochs = num_bias_epochs
        self.model_old = None
        self.T = T
        self.lamb = lamb
        self.bias_layers = []

        self.x_valid_exemplars = []
        self.y_valid_exemplars = []

        if self.exemplars_dataset.max_num_exemplars != 0:
            self.num_exemplars = self.exemplars_dataset.max_num_exemplars
        elif self.exemplars_dataset.max_num_exemplars_per_class != 0:
            self.num_exemplars_per_class = self.exemplars_dataset.max_num_exemplars_per_class

        have_exemplars = self.exemplars_dataset.max_num_exemplars + self.exemplars_dataset.max_num_exemplars_per_class
        assert (have_exemplars > 0), 'Error: BiC needs exemplars.'

    @staticmethod
    def exemplars_dataset_class():
        return ExemplarsDataset

    @staticmethod
    def extra_parser(args):
        """Returns a parser containing the approach specific parameters"""
        parser = ArgumentParser()
        # Sec. 3. "lambda is set to n / (n+m)" where n=num_old_classes and m=num_new_classes - so lambda is not a param
        # To use the original, set lamb=-1, otherwise, we allow to use specific lambda for the distillation loss
        parser.add_argument('--lamb', default=-1, type=float, required=False,
                            help='Forgetting-intransigence trade-off (default=%(default)s)')
        # Sec. 6.2. "The temperature scalar T in Eq. 1 is set to 2 by following [13,2]."
        parser.add_argument('--T', default=2, type=int, required=False,
                            help='Temperature scaling (default=%(default)s)')
        # Sec. 6.1. "the ratio of train/validation split on the exemplars is 9:1 for CIFAR-100 and ImageNet-1000"
        parser.add_argument('--val-exemplar-percentage', default=0.1, type=float, required=False,
                            help='Percentage of exemplars that will be used for validation (default=%(default)s)')
        # In the original code they define epochs_per_eval=100 and epoch_val_times=2, making a total of 200 bias epochs
        parser.add_argument('--num-bias-epochs', default=200, type=int, required=False,
                            help='Number of epochs for training bias (default=%(default)s)')
        return parser.parse_known_args(args)

    def bias_forward(self, outputs):
        """Utility function --- inspired by https://github.com/sairin1202/BIC"""
        bic_outputs = []
        for m in range(len(outputs)):
            bic_outputs.append(self.bias_layers[m](outputs[m]))
        return bic_outputs

    def train_loop(self, t, trn_loader, val_loader):
        """Contains the epochs loop
        Some parts could go into self.pre_train_process() or self.post_train_process(), but we leave it for readability
        """
        # add a bias layer for the new classes
        self.bias_layers.append(BiasLayer().to(self.device))

        # STAGE 0: EXEMPLAR MANAGEMENT -- select subset of validation to use in Stage 2 -- val_old, val_new (Fig.2)
        print('Stage 0: Select exemplars from validation')
        clock0 = time.time()

        # number of classes and proto samples per class
        num_cls = sum(self.model.task_cls)
        num_old_cls = sum(self.model.task_cls[:t])
        if self.exemplars_dataset.max_num_exemplars != 0:
            num_exemplars_per_class = int(np.floor(self.num_exemplars / num_cls))
            num_val_ex_cls = int(np.ceil(self.val_percentage * num_exemplars_per_class))
            num_trn_ex_cls = num_exemplars_per_class - num_val_ex_cls
            # Reset max_num_exemplars
            self.exemplars_dataset.max_num_exemplars = (num_trn_ex_cls * num_cls).item()
        elif self.exemplars_dataset.max_num_exemplars_per_class != 0:
            num_val_ex_cls = int(np.ceil(self.val_percentage * self.num_exemplars_per_class))
            num_trn_ex_cls = self.num_exemplars_per_class - num_val_ex_cls
            # Reset max_num_exemplars
            self.exemplars_dataset.max_num_exemplars_per_class = num_trn_ex_cls

        # Remove extra exemplars from previous classes -- val_old
        if t > 0:
            if self.exemplars_dataset.max_num_exemplars != 0:
                num_exemplars_per_class = int(np.floor(self.num_exemplars / num_old_cls))
                num_old_ex_cls = int(np.ceil(self.val_percentage * num_exemplars_per_class))
                for cls in range(num_old_cls):
                    assert (len(self.y_valid_exemplars[cls]) == num_old_ex_cls)
                    self.x_valid_exemplars[cls] = self.x_valid_exemplars[cls][:num_val_ex_cls]
                    self.y_valid_exemplars[cls] = self.y_valid_exemplars[cls][:num_val_ex_cls]

        # Add new exemplars for current classes -- val_new
        non_selected = []
        for curr_cls in range(num_old_cls, num_cls):
            self.x_valid_exemplars.append([])
            self.y_valid_exemplars.append([])
            # get all indices from current class
            cls_ind = np.where(np.asarray(trn_loader.dataset.labels) == curr_cls)[0]
            assert (len(cls_ind) > 0), "No samples to choose from for class {:d}".format(curr_cls)
            assert (num_val_ex_cls <= len(cls_ind)), "Not enough samples to store for class {:d}".format(curr_cls)
            # add samples to the exemplar list
            self.x_valid_exemplars[curr_cls] = [trn_loader.dataset.images[idx] for idx in cls_ind[:num_val_ex_cls]]
            self.y_valid_exemplars[curr_cls] = [trn_loader.dataset.labels[idx] for idx in cls_ind[:num_val_ex_cls]]
            non_selected.extend(cls_ind[num_val_ex_cls:])
        # remove selected samples from the validation data used during training
        trn_loader.dataset.images = [trn_loader.dataset.images[idx] for idx in non_selected]
        trn_loader.dataset.labels = [trn_loader.dataset.labels[idx] for idx in non_selected]
        clock1 = time.time()
        print(' > Selected {:d} validation exemplars, time={:5.1f}s'.format(
            sum([len(elem) for elem in self.y_valid_exemplars]), clock1 - clock0))

        # make copy to keep the type of dataset for Stage 2 -- not efficient
        bic_val_dataset = deepcopy(trn_loader.dataset)
        # add exemplars to train_loader -- train_new + train_old (Fig.2)
        if t > 0:
            trn_loader = torch.utils.data.DataLoader(trn_loader.dataset + self.exemplars_dataset,
                                                     batch_size=trn_loader.batch_size,
                                                     shuffle=True,
                                                     num_workers=trn_loader.num_workers,
                                                     pin_memory=trn_loader.pin_memory)

        # STAGE 1: DISTILLATION
        print('Stage 1: Training model with distillation')
        super().train_loop(t, trn_loader, val_loader)
        # From LwF: Restore best and save model for future tasks
        self.model_old = deepcopy(self.model)
        self.model_old.eval()
        self.model_old.freeze_all()

        # STAGE 2: BIAS CORRECTION
        if t > 0:
            print('Stage 2: Training bias correction layers')
            # Fill bic_val_loader with validation protoset
            if isinstance(bic_val_dataset.images, list):
                bic_val_dataset.images = sum(self.x_valid_exemplars, [])
            else:
                bic_val_dataset.images = np.vstack(self.x_valid_exemplars)
            bic_val_dataset.labels = sum(self.y_valid_exemplars, [])
            bic_val_loader = DataLoader(bic_val_dataset, batch_size=trn_loader.batch_size, shuffle=True,
                                        num_workers=trn_loader.num_workers, pin_memory=trn_loader.pin_memory)

            # bias optimization on validation
            self.model.eval()
            # Allow to learn the alpha and beta for the current task
            self.bias_layers[t].alpha.requires_grad = True
            self.bias_layers[t].beta.requires_grad = True

            # In their code is specified that momentum is always 0.9
            bic_optimizer = torch.optim.SGD(self.bias_layers[t].parameters(), lr=self.lr, momentum=0.9)
            # Loop epochs
            for e in range(self.bias_epochs):
                # Train bias correction layers
                clock0 = time.time()
                total_loss, total_acc = 0, 0
                for inputs, targets in bic_val_loader:
                    # Forward current model
                    with torch.no_grad():
                        outputs = self.model(inputs.to(self.device))
                        old_cls_outs = self.bias_forward(outputs[:t])
                    new_cls_outs = self.bias_layers[t](outputs[t])
                    pred_all_classes = torch.cat([torch.cat(old_cls_outs, dim=1), new_cls_outs], dim=1)
                    # Eqs. 4-5: outputs from previous tasks are not modified (any alpha or beta from those is fixed),
                    #           only alpha and beta from the new task is learned. No temperature scaling used.
                    loss = torch.nn.functional.cross_entropy(pred_all_classes, targets.to(self.device))
                    # However, in their code, they apply a 0.1 * L2-loss to the gamma variable (beta in the paper)
                    loss += 0.1 * ((self.bias_layers[t].beta[0] ** 2) / 2)
                    # Log
                    total_loss += loss.item() * len(targets)
                    total_acc += ((pred_all_classes.argmax(1) == targets.to(self.device)).float()).sum().item()
                    # Backward
                    bic_optimizer.zero_grad()
                    loss.backward()
                    bic_optimizer.step()
                clock1 = time.time()
                # reducing the amount of verbose
                if (e + 1) % (self.bias_epochs / 4) == 0:
                    print('| Epoch {:3d}, time={:5.1f}s | Train: loss={:.3f}, TAg acc={:5.1f}% |'.format(
                          e + 1, clock1 - clock0, total_loss / len(bic_val_loader.dataset.labels),
                          100 * total_acc / len(bic_val_loader.dataset.labels)))
            # Fix alpha and beta after learning them
            self.bias_layers[t].alpha.requires_grad = False
            self.bias_layers[t].beta.requires_grad = False

        # Print all alpha and beta values
        for task in range(t + 1):
            print('Stage 2: BiC training for Task {:d}: alpha={:.5f}, beta={:.5f}'.format(task,
                  self.bias_layers[task].alpha.item(), self.bias_layers[task].beta.item()))

        # STAGE 3: EXEMPLAR MANAGEMENT
        self.exemplars_dataset.collect_exemplars(self.model, trn_loader, val_loader.dataset.transform)

    def train_epoch(self, t, trn_loader):
        """Runs a single epoch"""
        self.model.train()
        if self.fix_bn and t > 0:
            self.model.freeze_bn()
        for images, targets in trn_loader:
            # Forward old model
            targets_old = None
            if t > 0:
                targets_old = self.model_old(images.to(self.device))
                targets_old = self.bias_forward(targets_old)  # apply bias correction
            # Forward current model
            outputs = self.model(images.to(self.device))
            outputs = self.bias_forward(outputs)  # apply bias correction
            loss = self.criterion(t, outputs, targets.to(self.device), targets_old)
            # Backward
            self.optimizer.zero_grad()
            loss.backward()
            torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clipgrad)
            self.optimizer.step()

    def eval(self, t, val_loader):
        """Contains the evaluation code"""
        with torch.no_grad():
            total_loss, total_acc_taw, total_acc_tag, total_num = 0, 0, 0, 0
            self.model.eval()
            for images, targets in val_loader:
                # Forward old model
                targets_old = None
                if t > 0:
                    targets_old = self.model_old(images.to(self.device))
                    targets_old = self.bias_forward(targets_old)  # apply bias correction
                # Forward current model
                outputs = self.model(images.to(self.device))
                outputs = self.bias_forward(outputs)  # apply bias correction
                loss = self.criterion(t, outputs, targets.to(self.device), targets_old)
                hits_taw, hits_tag = self.calculate_metrics(outputs, targets)
                # Log
                total_loss += loss.item() * len(targets)
                total_acc_taw += hits_taw.sum().item()
                total_acc_tag += hits_tag.sum().item()
                total_num += len(targets)
        return total_loss / total_num, total_acc_taw / total_num, total_acc_tag / total_num

    def cross_entropy(self, outputs, targets, exp=1.0, size_average=True, eps=1e-5):
        """Calculates cross-entropy with temperature scaling"""
        out = torch.nn.functional.softmax(outputs, dim=1)
        tar = torch.nn.functional.softmax(targets, dim=1)
        if exp != 1:
            out = out.pow(exp)
            out = out / out.sum(1).view(-1, 1).expand_as(out)
            tar = tar.pow(exp)
            tar = tar / tar.sum(1).view(-1, 1).expand_as(tar)
        out = out + eps / out.size(1)
        out = out / out.sum(1).view(-1, 1).expand_as(out)
        ce = -(tar * out.log()).sum(1)
        if size_average:
            ce = ce.mean()
        return ce

    def criterion(self, t, outputs, targets, targets_old):
        """Returns the loss value"""

        # Knowledge distillation loss for all previous tasks
        loss_dist = 0
        if t > 0:
            loss_dist += self.cross_entropy(torch.cat(outputs[:t], dim=1),
                                            torch.cat(targets_old[:t], dim=1), exp=1.0 / self.T)
        # trade-off - the lambda from the paper if lamb=-1
        if self.lamb == -1:
            lamb = (self.model.task_cls[:t].sum().float() / self.model.task_cls.sum()).to(self.device)
            return (1.0 - lamb) * torch.nn.functional.cross_entropy(torch.cat(outputs, dim=1),
                                                                    targets) + lamb * loss_dist
        else:
            return torch.nn.functional.cross_entropy(torch.cat(outputs, dim=1), targets) + self.lamb * loss_dist

class BiasLayer(torch.nn.Module):
    """Bias layers with alpha and beta parameters"""

    def __init__(self):
        super(BiasLayer, self).__init__()
        # Initialize alpha and beta with requires_grad=False and only set to True during Stage 2
        self.alpha = torch.nn.Parameter(torch.ones(1, requires_grad=False, device="cuda"))
        self.beta = torch.nn.Parameter(torch.zeros(1, requires_grad=False, device="cuda"))

    def forward(self, x):
        return self.alpha * x + self.beta


import copy
from datetime import datetime

from .db import db
from .helpers import pretty_date

class Toggle(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(128), unique=True)
    description = db.Column(db.String(256))
    enabled = db.Column(db.Boolean, nullable=False, default=False)
    last_changed_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)

    def __repr__(self) -> str:
        return f"<Toggle: {self.name},{self.enabled}>"

    def __eq__(self, other):
        if not isinstance(other, Toggle):
            return NotImplemented

        return self.name == other.name and self.enabled == other.enabled

    def serialize(self):
        return {self.name: self.enabled}

    @staticmethod
    def insert_all():
        for toggle in toggles:
            if Toggle.query.get(toggle.id) is None:
                print(f"Creating toggle: {toggle.name}")
                db.session.add(toggle)

        db.session.commit()

    def last_changed(self) -> str:
        return pretty_date(self.last_changed_at)

    def toggle(self):
        self.enabled = not self.enabled
        self.last_changed_at = datetime.utcnow()

toggles = [
    Toggle(
        id=1,
        name="game.tutorial.enable",
        description="Aktiviert das Tutorial",
    ),
    Toggle(
        id=2,
        name="game.announcements.enable",
        description="Aktiviert 'Ansagen'",
        enabled=True,
    ),
    Toggle(
        id=3,
        name="game.multiplayer.enable",
        description="Aktiviert den Multiplayer-Modus",
    ),
]


# !/usr/bin/env python
# Name:     standardValues.py
# By:       LA-Shill
# Date:     30.01.2022
# Version   0.3.1
# -----------------------------------------------

import pymongo
from os import environ, path, getcwd
from dotenv import load_dotenv
from datetime import timedelta

"""Load in .env file"""
basedir = path.abspath(path.dirname(__file__))
load_dotenv(path.join(basedir, '.env'))

host = str(environ.get('DB_HOST'))
CORE_MONGO_DB = str(environ.get('CORE_MONGO_DB'))
VUL_MONGO_DB = str(environ.get('VUL_MONGO_DB'))
port = int(environ.get('DB_PORT'))

client = pymongo.MongoClient(host, port)
mainDB = client[CORE_MONGO_DB.rsplit('/', 1)[-1]]

try:
    if mainDB['settings'].count_documents({}) > 0:
        print("[INFORMANT] Settings DB file detected")
    else:
        settings = {'CENSYS_API_ID' : '', 'CENSYS_API_SECRET' : '', 'SHODAN_API_KEY' : '', 'BINARY_EDGE_API_KEY' : '', 'ONYPHE_API_KEY' : '', 'FARSIGHT_API_KEY' : '', 'HIGH_RISK_PORTS' : [], 'GEO_LOCATION' : ''}
        mainDB['settings'].insert(settings)
except Exception as e:
    print("Error on first run, check you have all DB dependencies installed. Error: " + str(e))

class StandardValues:
    """
    Default vaules used across program,
    can be stored in volatile memory if required
    """
    db_r = mainDB['settings'].find({})

    CENSYS_API_ID: str = db_r[0]['CENSYS_API_ID']
    CENSYS_API_SECRET: str =  db_r[0]['CENSYS_API_SECRET']
    SHODAN_API_KEY: str =  db_r[0]['SHODAN_API_KEY']
    BINARY_EDGE_API_KEY: str = db_r[0]['BINARY_EDGE_API_KEY']
    ONYPHE_API_KEY: str =  db_r[0]['ONYPHE_API_KEY']
    FARSIGHT_API_KEY: str = db_r[0]['FARSIGHT_API_KEY']
    HIGH_RISK_PORTS: list = db_r[0]['HIGH_RISK_PORTS']
    GEO_LOCATION: str = db_r[0]['GEO_LOCATION']

    CENSYS_DEFAULT_RESULTS_QUANTITY: int = 10000
    SHODAN_DEFAULT_RESULTS_QUANTITY: int = 10
    BINARYEDGE_DEFAULT_RESULTS_PAGE: int = 1

    THREATMINER_API_DELAY: int = 6
    ROBTEX_API_DELAY: int = 20
    THREATCROWD_API_DELAY: int = 10
    DALOO_API_DELAY: int = 5
    FARSIGHT_API_DELAY: int = 1

    DB_HOST: str = host
    DB_PORT: int = port
    MAIN_DB_NAME: str = CORE_MONGO_DB.rsplit('/', 1)[-1]
    VUL_DB_NAME: str = VUL_MONGO_DB.rsplit('/', 1)[-1]


#Your config - you can also pass arguments
from config.intime_crt import *

#Boilerplate imports
import dash
from dash import dcc
from dash import html
import numpy as np
from variables import *
from Loader import Loader
import plotly.graph_objects as go
from time import time
import argparse
import os

#globals - passed between various update functions
global coatings #pds coatings
global cmax #max pe on cbar
global mcparts_lines
global crt_lines

#tpc0 globals
global muons_tpc0
global pds_tpc0
global pds_coordinates_tpc0
global muon_lines_tpc0

#tpc 1 globals
global muons_tpc1
global pds_tpc1
global pds_coordinates_tpc1
global muon_lines_tpc1

# Create the parser
parser = argparse.ArgumentParser(description="Load data - default values are stored in the config selected")

# Add the arguments
parser.add_argument('--data', type=str, default=DATA_DIR, required=False, help='Path to the data')
parser.add_argument('--hdump_name', default=HDUMP_NAME, required=False, help='hitdumper file name')
parser.add_argument('--sm_name', default=SM_NAME, required=False, help='pmt software metrics file name')
parser.add_argument('--wfm_name', default=WFM_NAME, required=False, help='waveform file name')
parser.add_argument('--load_muon',type=bool, default=LOAD_MUON, required=False, help='load muon info')
parser.add_argument('--load_crt',type=bool, default=LOAD_CRT, required=False, help='load crt info')
parser.add_argument('--load_mcpart',type=bool, default=LOAD_MCPART, required=False, help='load mcpart info')
parser.add_argument('--mode',type=str, default=MODE, help='Optical detector mode')

# Parse the arguments
args = parser.parse_args()

# Use the arguments
l = Loader(
    data_dir=args.data,
    hdump_name=args.hdump_name, #Set in config or parsers
    software_name=args.sm_name, #Set in config or parsers
    wfm_name=args.wfm_name, #Set in config or parsers
    load_muon=args.load_muon,
    load_crt=args.load_crt,
    load_mcpart=args.load_mcpart,
    mode=args.mode,
    hdrkeys=HDRKEYS, #Set in config
    pmt_ara_name=PMT_ARA_NAME, #set in config
)

#Initialize display
run_list = l.run_list #Get list of run,subrun,evt
run,subrun,event = run_list[0] #get first run for initialization
start_bin = t0+dt
end_bin = t1
l.get_event(run,subrun,event)
coatings = COATINGS #Select initial PDs to show
cmin = 0 #minimum PE count

#Muon init
if l.load_muon:
    muons_tpc0 = l.get_muon_list(tpc=0)
    muons_tpc1 = l.get_muon_list(tpc=1)
    muon_lines_tpc0 = [muons_tpc0[i].plot_line(i,0) for i in range(len(muons_tpc0))]
    muon_lines_tpc1 = [muons_tpc1[i].plot_line(i,1) for i in range(len(muons_tpc1))]
else:
    muon_lines_tpc0 = [go.Scatter()]
    muon_lines_tpc1 = [go.Scatter()]

#MCPart init
if l.load_mcpart:
    mcparts = l.get_mcpart_list()
    mcparts_lines = [mcparts[i].plot_line(i,max_color=len(mcparts)) for i in range(len(mcparts))]
else:
    mcparts_lines = [go.Scatter()]

#CRT track init
if l.load_crt:
    crt_trks = l.get_crt_list()
    crt_lines = [crt_trks[i].plot_line(i,max_color=len(crt_trks)) for i in range(len(crt_trks))]
else:
    crt_lines = [go.Scatter()]

#Display figures
WIDTH = 700
TPC_HEIGHT = WIDTH * 4/5
WAVEFORM_HEIGHT = WIDTH * 2/5
def get_tpc0():
    global pds_coordinates_tpc0,muon_lines_tpc0,mcparts_lines,cmax,crt_lines
    return go.Figure(
        data=pds_coordinates_tpc0+muon_lines_tpc0+mcparts_lines+crt_lines,
        layout=go.Layout(
            autosize=False,
            width=WIDTH,
            height=TPC_HEIGHT,
            showlegend=False,
            xaxis=dict(range=[0, 500]),  # Set x-axis limits
            yaxis=dict(range=[-200, 200]),   # Set y-axis limits
            coloraxis=dict(cmax=cmax),
            margin=dict(l=50, r=50, b=50, t=50, pad=0),
        )
    )
def get_tpc1():
    global pds_coordinates_tpc1,muon_lines_tpc1,mcparts_lines,cmax,crt_lines
    return go.Figure(
        data=pds_coordinates_tpc1+muon_lines_tpc1+mcparts_lines+crt_lines,
        layout=go.Layout(
            autosize=False,
            width=WIDTH,
            height=TPC_HEIGHT,
            showlegend=False,
            xaxis=dict(range=[0, 500]),  # Set x-axis limits
            yaxis=dict(range=[-200, 200]),   # Set y-axis limits
            coloraxis=dict(cmax=cmax),
            margin=dict(l=50, r=50, b=50, t=50, pad=0),
        )
    )
def get_waveform(waveform_data,pmt_id=None):
    return go.Figure(data=waveform_data,
                             layout=go.Layout(
                                autosize=False,
                                width=WIDTH,
                                height=WAVEFORM_HEIGHT,
                                title=dict(
                                    text=f'PDS ID : {pmt_id}'
                                ),
                                xaxis_title='t [us]',
                                yaxis_title='ADC',
                                margin=dict(l=50, r=50, b=50, t=50, pad=0),
                            )
                        )

#PDS init
def init_pds_dash():
    global pds_coordinates_tpc0
    global pds_tpc0

    global pds_coordinates_tpc1
    global pds_tpc1

    global cmax
    global coatings

    pds_tpc0 = l.get_pmt_list(tpc=0,coatings=coatings) #Get list of pmts
    pds_tpc1 = l.get_pmt_list(tpc=1,coatings=coatings) #Get list of pmts
    if CMAX == 'global':
        cmax = np.max([pds.op_pe.op_pe.sum() for pds in pds_tpc0+pds_tpc1])
    elif CMAX == 'dynamic':
        cmax = np.max([pds.get_pe_start_stop(start_bin,end_bin) for pds in pds_tpc0+pds_tpc1])
    else:
        if VERBOSE: print(f'{CMAX} is not a valid setting for setting pe color')
        cmax = None
    pds_ids = [pds.id for pds in pds_tpc0+pds_tpc1]
    pds_coordinates_tpc0 = [pds.plot_coordinates(start_bin,end_bin,pds_ids,cmin=cmin,cmax=cmax) for pds in pds_tpc0]
    pds_coordinates_tpc1 = [pds.plot_coordinates(start_bin,end_bin,pds_ids,cmin=cmin,cmax=cmax) for pds in pds_tpc1]

init_pds_dash()

#Start dash
app = dash.Dash(__name__)

# Layout for the Dash app
app.layout = html.Div(style={'display':'flex'},children=[
    html.Div(style={'width': '85%'},children=[
        html.H1('PAD'),
        html.Label('Run: '),
        dcc.Input(id='run-input', type='number', value=run),  # initialize with run from l.run_list

        html.Label('Subrun: '),
        dcc.Input(id='subrun-input', type='number', value=subrun),  # initialize with subrun from l.run_list

        html.Label('Event: '),
        dcc.Input(id='event-input', type='number', value=event),  # initialize with event from l.run_list
        html.Button('Submit', id='submit-button', n_clicks=0),
        html.Br(),
        html.Label('t0 [ns]: '),
        dcc.Slider(
            id='t0',
            min=t0,  # minimum value
            max=t1,  # maximum value
            step=dt,  # step size
            value=t0,  # current value at t0
            marks=None,
            tooltip={"placement": "bottom", "always_visible": True}
        ),
        html.Label('t1 [ns]: '),
        dcc.Slider(
            id='t1',
            min=t0,  # minimum value
            max=t1,  # maximum value
            step=dt,  # step size
            value=t1,  # current value at t0
            marks=None,
            tooltip={"placement": "bottom", "always_visible": True}
        ),
        # Put TPC0 and TPC1 side by side
        html.Div(style={'display': 'flex'}, children=[
            # TPC0 Div
            html.Div(children=[
                html.H2('TPC0'),
                dcc.Graph(
                    id='tpc0',
                    figure=get_tpc0(),
                ),
                dcc.Graph(
                    id='waveform-graph-tpc0',
                    style={'height': f'{WAVEFORM_HEIGHT}px', 'width': f'{WIDTH}px'}
                ),  # This will initialize an empty graph with correct size)
            ]),
            # TPC1 Div
            html.Div(children=[
                html.H2('TPC1'),
                dcc.Graph(
                    id='tpc1',
                    figure=get_tpc1()
                ),
                dcc.Graph(
                    id='waveform-graph-tpc1',
                    style={'height': f'{WAVEFORM_HEIGHT}px', 'width': f'{WIDTH}px'}
                ),  # This will initialize an empty graph with correct size)
            ]),
        ]),
    ]),
    # Right-hand side
    html.Div(style={'width': '15%'},children=[
        html.Img(src=dash.get_asset_url('sbnd_pretty.png'), style={'width': '100%', 'height': 'auto'}),
        html.H3(f'Mode : {l.mode}'),
        html.Img(src=dash.get_asset_url('muon_track.png'), style={'width': '100%', 'height': 'auto'}) if l.load_muon else html.H3('Muon Tracks : NA'),
        html.Img(src=dash.get_asset_url('crt_track.png'), style={'width': '100%', 'height': 'auto'}) if l.load_crt else html.H3('CRT Tracks : NA'),
        html.Img(src=dash.get_asset_url('g4.png'), style={'width': '100%', 'height': 'auto'}) if l.load_mcpart else html.H3('G4 Primaries : NA'),
        #Not supported yet
        #html.H3(f'Current event : '),
        #html.H5(f'    Run: {l.run}, Subrun {l.subrun}, Event {l.event}'),
        html.H3('Select PDS Coatings:'),
        dcc.Checklist(
            id='pds_coatings',
            options=[
                {'label': 'Undefined PDs', 'value': 0},
                {'label': 'Coated (VUV) PMTs', 'value': 1},
                {'label': 'Uncoated (VIS) PMTs', 'value': 2},
                {'label': 'Coated (VUV) XAs', 'value': 4},
                {'label': 'Uncoated (VIS) XAs', 'value': 3},
            ],
            value=coatings
        ),
        html.H2('Available Runs'),
        html.Ul(children=[
            html.Li(f'Run: {run}, Subrun: {subrun}, Event: {event}') for run, subrun, event in run_list
        ])
    ])
])

#Update PE and/or coatings
@app.callback(
    dash.dependencies.Output('tpc0', 'figure'),
    dash.dependencies.Output('tpc1', 'figure'),
    [dash.dependencies.Input('t0', 'value'),
     dash.dependencies.Input('t1', 'value'),
     dash.dependencies.Input('submit-button', 'n_clicks'),
     dash.dependencies.Input('pds_coatings', 'value')],
    [dash.dependencies.State('run-input', 'value'),
     dash.dependencies.State('subrun-input', 'value'),
     dash.dependencies.State('event-input', 'value')]
)

def update_tpcs(start_time_bin, end_time_bin, n_clicks,values,run, subrun, event):
    #Make graph objects global
    global muons_tpc0
    global pds_tpc0
    global pds_coordinates_tpc0
    global muon_lines_tpc0

    global muons_tpc1
    global pds_tpc1
    global pds_coordinates_tpc1
    global muon_lines_tpc1

    global coatings
    global cmax
    global mcparts_lines
    global crt_lines

    ctx = dash.callback_context
    # Check if the button triggered the callback or if a coating was changed
    if ctx.triggered[0]['prop_id'] == 'submit-button.n_clicks' or (coatings != values and values != []):
        coatings = values
        if [run,subrun,event] in l.run_list:
            if VERBOSE: print(f'Retrieving Run {run} Subrun {subrun} Event {event}')
            l.get_event(run, subrun, event)

            pds_tpc0 = l.get_pmt_list(tpc=0,coatings=coatings)
            pds_tpc1 = l.get_pmt_list(tpc=1,coatings=coatings)

            if l.load_muon:
                muons_tpc0 = l.get_muon_list(tpc=0)
                muons_tpc1 = l.get_muon_list(tpc=1)
                muon_lines_tpc0 = [muons_tpc0[i].plot_line(i,0) for i in range(len(muons_tpc0))]
                muon_lines_tpc1 = [muons_tpc1[i].plot_line(i,1) for i in range(len(muons_tpc1))]
            else:
                muon_lines_tpc0 = [go.Scatter()]
                muon_lines_tpc1 = [go.Scatter()]
            if l.load_mcpart:
                mcparts = l.get_mcpart_list()
                mcparts_lines = [mcparts[i].plot_line(i,max_color=len(mcparts)) for i in range(len(mcparts))]
            else:
                mcparts_lines = [go.Scatter()]
            if l.load_crt:
                crt_trks = l.get_crt_list()
                crt_lines = [crt_trks[i].plot_line(i,max_color=len(crt_trks)) for i in range(len(crt_trks))]
            else:
                crt_lines = [go.Scatter()]
        else:
            print(f'Run {run} Subrun {subrun} Event {event} not in file')
            if VERBOSE:
                print('List of run,subrun,events - ')
                #print(l.run_list)
                return None

    #PDS
    #if VERBOSE: print('Update time window')
    s0 = time()
    if CMAX == 'global':
        cmax = np.max([pds.op_pe.op_pe.sum() for pds in pds_tpc0+pds_tpc1])
    elif CMAX == 'dynamic':
        cmax = np.max([pds.get_pe_start_stop(start_time_bin,end_time_bin) for pds in pds_tpc0+pds_tpc1])
    else:
        print(f'{CMAX} is not a valid setting for setting pe color')
        cmax = None
    cmin = 0
    pds_ids = [pds.id for pds in pds_tpc0+pds_tpc1]
    pds_coordinates_tpc0 = [pds.plot_coordinates(start_time_bin,end_time_bin,pds_ids,cmin=cmin,cmax=cmax) for pds in pds_tpc0]
    pds_coordinates_tpc1 = [pds.plot_coordinates(start_time_bin,end_time_bin,pds_ids,cmin=cmin,cmax=cmax) for pds in pds_tpc1]
    s1 = time()
    if VERBOSE: print(f'Get new PE from {start_time_bin} to {end_time_bin} (ns): {s1-s0:.2f} s')

    #Muons
    s0 = time()
    if l.load_muon:
        muon_lines_tpc0 = [muons_tpc0[i].plot_line(i,0) for i in range(len(muons_tpc0))]
        muon_lines_tpc1 = [muons_tpc1[i].plot_line(i,1) for i in range(len(muons_tpc1))]
    else:
        muon_lines_tpc0 = [go.Scatter()]
        muon_lines_tpc1 = [go.Scatter()]
    s1 = time()
    if VERBOSE: print(f'Get new muon trajectories : {s1-s0:.2f} s')
    if l.load_mcpart:
        mcparts = l.get_mcpart_list()
        mcparts_lines = [mcparts[i].plot_line(i,max_color=len(mcparts)) for i in range(len(mcparts))]
    else:
        mcparts_lines = [go.Scatter()]
    if l.load_crt:
        crt_trks = l.get_crt_list()
        crt_lines = [crt_trks[i].plot_line(i,max_color=len(crt_trks)) for i in range(len(crt_trks))]
    else:
        crt_lines = [go.Scatter()]
    s2 = time()
    if VERBOSE: print(f'Get new mcpart trajectories : {s2-s1:.2f} s')

    return [get_tpc0(),get_tpc1()]

# Callback to update the waveform graph when the button is clicked
@app.callback(
    dash.dependencies.Output('waveform-graph-tpc0', 'figure'),
    [dash.dependencies.Input('tpc0', 'clickData')]
)

def update_waveform_graph(click_data):
    if click_data is not None:
        if VERBOSE: print('Drawing waveform')
        pmt_id = click_data['points'][0]['customdata']
        ind = l.pds_tpc0_ids.index(pmt_id) #find index of pmt
        s0 = time()
        waveform_data = pds_tpc0[ind].plot_waveform()
        s1 = time()
        if VERBOSE: print(f'Get waveform for PDS {pmt_id}: {s1-s0:.2f} s')

        if waveform_data is not None:
            if VERBOSE: print('Done')
            return get_waveform(waveform_data,pmt_id=pmt_id)

    return go.Figure()

# Callback to update the waveform graph when the button is clicked
@app.callback(
    dash.dependencies.Output('waveform-graph-tpc1', 'figure'),
    [dash.dependencies.Input('tpc1', 'clickData')]
)

def update_waveform_graph(click_data):
    if click_data is not None:
        if VERBOSE: print('Drawing waveform')
        pmt_id = click_data['points'][0]['customdata']
        ind = l.pds_tpc1_ids.index(pmt_id) #find index of pmt
        s0 = time()
        waveform_data = pds_tpc1[ind].plot_waveform()
        s1 = time()
        if VERBOSE: print(f'Get waveform for PDS {pmt_id}: {s1-s0:.2f} s')

        if waveform_data is not None:
            return get_waveform(waveform_data,pmt_id=pmt_id)

    return go.Figure()

if __name__ == '__main__':
    app.run_server(debug=False)


import torchvision.transforms as transforms
import matplotlib.pyplot as plt
from PIL import Image
import torch

def get_loader(imsize):
    loader = transforms.Compose([
        transforms.Resize(imsize),
        transforms.CenterCrop(imsize),
        transforms.ToTensor()])
    return loader

unloader = transforms.ToPILImage()

def image_loader(image_name, loader, device):
    image = Image.open(image_name)
    image = loader(image).unsqueeze(0)
    return image.to(device, torch.float)

def imshow(tensor, title=None):
    """

    :param tensor:
    :param title:
    :return:
    """
    image = tensor.cpu().clone()
    image = image.squeeze(0)
    image = unloader(image)
    plt.imshow(image)
    if title is not None:
        plt.title(title)
    plt.pause(0.001)


import random
import tkinter as tk

definitions = {
    'While': 'Цикл "while" используется для выполнения блока кода, пока условие истинно.',
    'For': 'Цикл "for" используется для итерации по элементам последовательности (например, списку или строке).',
    'If': 'Условие "if" позволяет выполнить определенный блок кода, если условие истинно.',
    'Function': 'Функция - это блок кода, который можно вызывать с определенными аргументами.',
    'List': 'Список - это упорядоченная коллекция элементов, которая может содержать разные типы данных.'
}

def show_random_definition():
    random_key = random.choice(list(definitions.keys()))
    definition_text.delete('1.0', tk.END)
    definition_text.insert(tk.END, definitions[random_key])

my_tk = tk.Tk()
my_tk.title('Определения Python')
my_tk.configure(bg='#00FFFF')

title = tk.Label(my_tk, text='Определения Python', font=('Arial', 16), bg='#00FFFF')
title.pack(pady=10)

definition_text = tk.Text(my_tk, height=5, width=50)
definition_text.pack()

show_definition = tk.Button(my_tk, text='Показать определение', command=show_random_definition)
show_definition.pack(pady=10)

my_tk.mainloop()


import numpy
import PIL
import math
import time
import picamera
import numpy as np
import cv2
import matplotlib as plt
import io

stream=io.BytesIO()
with picamera.PiCamera() as camera:
	camera.resolution=(320,240)
	camera.framerate=24
	time.sleep(1)
	camera.capture(stream,format='jpeg')
data=np.fromstring(stream.getvalue(),dtype=np.uint8)
image=cv2.imdecode(data,1)

# with picamera.PiCamera() as camera:
#     camera.resolution=(320,240)
#     time.sleep(3)
#     camera.capture('assets/pi_photo')
#     image = cv2.imread('assets/pi_photo')

window_name = '“lab 5.2'

cv2.imshow(window_name, image)
raw_key = cv2.waitKey(1000)
rgb=cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
height,width,channels=image.shape #Add this line here

for x in range (0,height):
	for y in range (0,width):
		r,g,b=rgb[x,y]
		if r>g and r>b and r>128:
			image[x,y]=0,0,255

window_name2 = 'lab 5.2 red filter'
cv2.imshow(window_name2, image)
raw_key = cv2.waitKey(20000)


"""
simple log parser like logstash
"""
import threading
import logging
import os
import re
import enum
from datetime import datetime
from queue import Queue, Empty
from time import sleep
from glob import glob
from weakref import WeakValueDictionary
from contextlib import contextmanager

from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine, Column, String, Integer, DateTime, Enum
from sqlalchemy.ext.declarative import declarative_base

from .outputs import *

__version__ = '1.0.0'
__author__ = 'thuhak.zhou@nio.com'
Base = declarative_base()

class LogStat(enum.Enum):
    ready = 0
    running = 1
    file_deleted = 2
    parse_error = 3
    no_update = 4
    read_file_error = 5

class Logfile(Base):
    __tablename__ = 'log'

    filename = Column(String(256), primary_key=True)
    inode = Column(Integer, nullable=False)
    pos = Column(Integer, nullable=False, default=0)
    last_update = Column(DateTime, nullable=True)
    status = Column(Enum(LogStat), default=LogStat.ready)

@contextmanager
def session_scope(DBSession):
    session = DBSession()
    try:
        yield session
        session.commit()
    except Exception as e:
        logging.error('db commit error, rolling back')
        session.rollback()
        raise e
    finally:
        session.close()

def is_thread_running(thread: threading.Thread) -> bool:
    return thread.is_alive() if thread is not None else False

class LogParser:
    """
    logfile parser
    """
    _instances = WeakValueDictionary()

    def __new__(cls, filename, *args, **kwargs):
        """flyweight"""
        if filename in cls._instances:
            instance = cls._instances[filename]
        else:
            instance = super().__new__(cls)
            cls._instances[filename] = instance
        return instance

    def __init__(self, filename: str, que: Queue, dbsession, encoding='utf-8', flush_interval=3, stop_time=30):
        """
        :param filename: path of logfile
        :param que: parsed result will be pushed to this queue
        :param encoding: encoding of logfile
        :param dbsession: database session
        :param flush_interval: every flush_interval seconds, latest running will be saved to database
        :param stop_time: stop parser when file not changed with stop_time seconds
        """
        self.filename = filename
        self.inode = os.stat(filename).st_ino
        self.dbsession = dbsession
        with session_scope(self.dbsession) as sess:
            data = sess.query(Logfile).filter_by(filename=filename).first()
            if data:
                if self.inode != data.inode:
                    data.inode = self.inode
                    data.pos = self.pos = 0
                    data.last_update = self.last_update = None
                else:
                    self.pos = data.pos
                    self.last_update = data.last_update
            else:
                self.pos = 0
                self.last_update = None
                data = Logfile(filename=filename, inode=self.inode)
                sess.add(data)
            self._data = data
        self.que = que
        self.flush_interval = flush_interval
        self.stop_time = stop_time
        self.encoding = encoding

        self.running = False
        self.status = LogStat.ready
        self.context = None
        self._working_thread = None
        self._flush_thread = None
        self._flag = threading.Event()
        self.start()

    def parser(self, line):
        """
        default parser, you can define you own parser, override the default one
        """
        return line

    def start(self):
        """start parser"""
        logging.info('starting logfile parser')
        self._working_thread = threading.Thread(target=self.run, daemon=True)
        self._working_thread.start()
        self._flush_thread = threading.Thread(target=self.flush, daemon=True)
        self._flush_thread.start()

    def stop(self):
        """stop parser"""
        logging.info('stopping logfile parser')
        self._flag.set()
        while True:
            if not is_thread_running(self._flush_thread) and not is_thread_running(self._working_thread):
                self.running = False
                break
            sleep(0.1)

    def _flush(self):
        """
        flush status to database
        """
        try:
            with session_scope(self.dbsession) as sess:
                self._data.pos = self.pos
                self._data.inode = self.inode
                self._data.last_update = datetime.now()
                self._data.status = self.status
                sess.merge(self._data)
        except Exception as e:
            logging.error('can not save parser status to database, reason {}'.format(str(e)))

    def flush(self):
        """
        run _flush job every self.flush_interval seconds
        """
        while True:
            if not is_thread_running(self._working_thread):
                self._flush()
                self.running = False
                break
            elif self._flag.is_set():
                sleep(0.1)
                continue
            else:
                sleep(self.flush_interval)
                self._flush()

    def run(self):
        """
        run parser
        """
        try:
            stream = open(self.filename, 'r', encoding=self.encoding)
            stream.seek(self.pos)
            self.running = True
        except Exception as e:
            logging.error('can not open file {}, reason: {}'.format(self.filename, str(e)))
            self.status = LogStat.read_file_error
            return
        while True:
            if self._flag.is_set():
                break
            line = stream.readline().strip()
            if line:
                try:
                    result = self.parser(line)
                except Exception as e:
                    logging.error('parser error for line {}, error is {}'.format(line, str(e)))
                    break
                else:
                    if result:
                        self.que.put(result)
                    self.status = LogStat.running
                    self.pos = stream.tell()
                    self.last_update = datetime.now()
            else:
                if not os.path.exists(self.filename):
                    logging.info('log file {} no longer exists'.format(self.filename))
                    self.status = LogStat.file_deleted
                    break
                else:
                    stat = os.stat(self.filename)
                    inode, mtime = stat.st_ino, stat.st_mtime
                    if inode != self.inode:
                        stream.close()
                        self.pos = 0
                        self.last_update = None
                        self.inode = inode
                        self.status = LogStat.ready
                        try:
                            stream = open(self.filename, 'r', encoding=self.encoding)
                        except Exception as e:
                            logging.error('can not open file {}, reason: {}'.format(self.filename, str(e)))
                            self.status = LogStat.read_file_error
                            raise e
                    elif self.last_update and \
                            self.last_update.timestamp() - mtime >= self.stop_time:
                        logging.info('{} has no changing over {} seconds,stop parsing'.format(self.filename,
                                                                                self.stop_time))
                        self.status = LogStat.no_update
                        break
                    else:
                        sleep(1)
        stream.close()

class LogSlice:
    """
    log parser manager
    """

    def __init__(self, path, output, dbpath='/var/lib/logslice', parser=LogParser, file_filter=None,
                 encoding='utf-8', flush_interval=3, rescan_interval=30, close_file_time=30,
                 cache_size=10):
        """
        :param path: search log files using globbing patterns
        :param output: callable object for output
        :param dbpath: path of database
        :param parser: parser class
        :param file_filter: regex filter
        :param encoding: file encoding
        :param flush_interval: every flush_interval seconds, latest running will be saved to database
        :param rescan_interval: every rescan_interval seconds, rescan logs
        :param close_file_time: close file which not changed within close_file_time seconds
        :param cache_size: a iterable object for output
        """
        if isinstance(path, str):
            self.path = [path]
        else:
            self.path = path
        self.output_callback = output
        self.parser = parser
        self.filter = None if file_filter is None else re.compile(file_filter)
        self.encoding = encoding
        self.flush_interval = flush_interval
        self.rescan_interval = rescan_interval
        self.close_file_time = close_file_time
        self.cache_size = cache_size

        if not os.path.exists(dbpath):
            os.makedirs(dbpath)
        dbpath = os.path.join(dbpath, 'logslice.db')
        engine = create_engine('sqlite:///{}'.format(dbpath), echo=False)
        self.dbsession = sessionmaker(bind=engine)
        Base.metadata.create_all(engine)

        self.que = Queue()
        self.jobs = []
        self._stop_input_flag = threading.Event()
        self._stop_output_flag = threading.Event()
        self.input_thread = None
        self.output_thread = None

    def scan_logs(self) -> set:
        """
        search logs in path
        :return: all log files
        """
        all_logs = set()
        for logpath in self.path:
            logs = set(glob(logpath, recursive=True))
            if self.filter:
                logs = {x for x in logs if self.filter.match(x)}
            all_logs |= logs
        return all_logs

    def start(self):
        self.input_thread = threading.Thread(target=self.scan, daemon=True)
        self.input_thread.start()
        self.output_thread = threading.Thread(target=self.output, daemon=True)
        self.output_thread.start()
        while True:
            try:
                if is_thread_running(self.input_thread) and is_thread_running(self.output_thread):
                    continue
            except KeyboardInterrupt:
                self.stop()
                break

    def stop(self):
        self._stop_input_flag.set()
        logging.info('stopping output')
        for job in self.jobs:
            job.stop()
        sleep(0.5)
        while True:
            self.clean_stopped_jobs()
            if not self.jobs:
                self._stop_output_flag.set()
                break
            else:
                logging.info('some job still working {}'.format([j.filename for j in self.jobs]))
                sleep(0.5)

    def clean_stopped_jobs(self):
        for job in self.jobs[:]:
            if not job.running:
                self.jobs.remove(job)

    def cache(self):
        for _ in range(self.cache_size):
            try:
                yield self.que.get(timeout=1)
            except Empty:
                break

    def output(self):
        while True:
            if self._stop_output_flag.is_set():
                logging.info('stop out put')
                break
            else:
                try:
                    self.output_callback(self.cache())
                except Exception as e:
                    logging.error('output error, reason: {}'.format(str(e)))

    def scan(self):
        while True:
            if self._stop_input_flag.is_set():
                break
            self._scan()
            sleep(self.rescan_interval)

    def _scan(self):
        self.clean_stopped_jobs()
        logs = self.scan_logs()
        no_update_logs = set()
        with session_scope(self.dbsession) as sess:
            check_logs = sess.query(Logfile).filter(Logfile.status == LogStat.no_update, Logfile.filename.in_(logs))
            for log in check_logs:
                try:
                    mtime = os.stat(log.filename).st_mtime
                    if mtime < log.last_update.timestamp():
                        no_update_logs.add(log.filename)
                except Exception as e:
                    logging.error('read file {} error, reason {}'.format(log.filename, str(e)))
                    continue
        jobs = {j.filename for j in self.jobs}
        logs = logs - no_update_logs - jobs
        for log in logs:
            self.jobs.append(self.parser(filename=log, que=self.que, dbsession=self.dbsession, encoding=self.encoding,
                                         flush_interval=self.flush_interval, stop_time=self.close_file_time))


# IMPORTS
import sys
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import torch.optim as optim
import matplotlib
import matplotlib.pyplot as plt
from timeit import default_timer as timer

# IMPORTS FROM OTHER PROJECT FILES
# Add the model folder to the sys path
sys.path.insert(1, '../Model')

# Do the imports from other files in the project
from DepthLSTM import DepthLSTM
from hyperparameters import *
from train_epoch import train_epoch
from test_epoch import test_epoch

# LOAD THE DATA
xyz_data = np.load('../Data/normalized_data.npy')

# PARAMETER DEFINITION
DATA_SIZE = xyz_data.shape[0]
NUM_JOINTS = 26
INPUT_SIZE = 2*NUM_JOINTS

# SEPARATION OF DATA AND GROUND TRUTH
# Separate x and y dimensions and convert into tensors
x_data = torch.tensor(xyz_data[:, :, 0].squeeze())
y_data = torch.tensor(xyz_data[:, :, 1].squeeze())
# Obtain the body_data tensor of shape [DATA_SIZE, 2*NUM_JOINTS]
body_data = torch.empty((DATA_SIZE, 2*NUM_JOINTS))
for i in range(0, 2*NUM_JOINTS):
  if i%2 == 0:
    body_data[:, i] = x_data[:, i//2]
  else:
    body_data[:, i] = y_data[:, i//2]
# Obtain the ground_data tensor of shape [DATA_SIZE, NUM_JOINTS]
ground_data = torch.tensor(xyz_data[:, :, 2].squeeze())

# TRAIN/TEST SPLIT
# Use the last TEST_FRACTION of the data for testing
TRAIN_SIZE = np.int(np.ceil(0.8*DATA_SIZE))
TEST_SIZE = DATA_SIZE - TRAIN_SIZE
train_data = body_data[0:TRAIN_SIZE,:]
train_ground = ground_data[0:TRAIN_SIZE,:]
test_data = body_data[TRAIN_SIZE:,:]
test_ground = ground_data[TRAIN_SIZE:,:]

# DIVIDE TRAINING DATA INTO BATCHES
# Compute the number of samples in each chunk or batch
CHUNK_SIZE = TRAIN_SIZE // BATCH_SIZE
# Obtain the list of BATCH_SIZE chunks where each element is a tensor
# of shape [CHUNK_SIZE, 2*NUM_JOINTS]
trainset = [ train_data[beg_i:end_i] \
            for beg_i, end_i in zip(range(0, TRAIN_SIZE - CHUNK_SIZE, CHUNK_SIZE),
                                    range(CHUNK_SIZE, TRAIN_SIZE, CHUNK_SIZE)) ]
groundset = [ train_ground[beg_i:end_i] \
            for beg_i, end_i in zip(range(0, TRAIN_SIZE - CHUNK_SIZE, CHUNK_SIZE),
                                    range(CHUNK_SIZE, TRAIN_SIZE, CHUNK_SIZE)) ]

# DIVIDE THE TESTING DATA INTO BATCHES
# Compute the number of samples in each chunk or batch
CHUNK_TEST_SIZE = TEST_SIZE // BATCH_SIZE
# Obtain the list of BATCH_SIZE batches where each element is a tensor
# of shape [CHUNK_SIZE, 2*NUM_JOINTS]
testset = [ test_data[beg_i:end_i] \
            for beg_i, end_i in zip(range(0, TEST_SIZE - CHUNK_TEST_SIZE, CHUNK_TEST_SIZE),
                                    range(CHUNK_TEST_SIZE, TEST_SIZE, CHUNK_TEST_SIZE)) ]
groundtestset = [ test_ground[beg_i:end_i] \
            for beg_i, end_i in zip(range(0, TEST_SIZE - CHUNK_TEST_SIZE, CHUNK_TEST_SIZE),
                                    range(CHUNK_TEST_SIZE, TEST_SIZE, CHUNK_TEST_SIZE)) ]

# TRAIN AND TEST THE MODEL
# Initialize the model
model = DepthLSTM(HIDDEN_SIZE, NUM_LAYERS, SEQ_LEN, BATCH_SIZE, NUM_JOINTS)
model.to(device)
# Define the loss function and the optimizer
loss_function = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
tr_loss = []
tst_loss = []
state = None
timer_beg = timer()
# Train the model for NUM_EPOCHS epochs
for epoch in range(NUM_EPOCHS):
  print('Starting epoch: ', epoch)
  train_epoch_loss = train_epoch(model, trainset, groundset, optimizer, loss_function, CHUNK_SIZE)
  test_epoch_loss, predY = test_epoch(model, testset, groundtestset, CHUNK_TEST_SIZE)
  timer_end = timer()
  if (epoch + 1) % 10 == 0:
    # Print the training loss of this epoch
    # It is calculated as the average of losses of every window
    print('Training loss in epoch {} is: {}'.format(epoch, sum(train_epoch_loss)/len(train_epoch_loss) ))
    print('Test loss in epoch {} is: {}'.format(epoch, sum(test_epoch_loss)/len(test_epoch_loss) ))
  tr_loss.append(train_epoch_loss)
  tst_loss.append(test_epoch_loss)
  timer_beg = timer()


import numpy as np
from scipy.signal import convolve2d

def markers_evaluator(game, is_max, *, markers=8):
    """ Evaluation based on the number of markers already placed """
    me = game.current_player if is_max else game.previous_player
    opponent = game.previous_player if is_max else game.current_player

    my_placed = np.count_nonzero(game.board == me)
    opponent_placed = np.count_nonzero(game.board == opponent)

    def f(p, m):
        return (2 ^ 63 - 1) if m - p == 0 else -m / (m - p)

    return f(my_placed, markers) - f(opponent_placed, markers)

def combination_evaluator(game, is_max, weight=10.0):
    """ Evaluation of consecutive markers of the same player """
    me = game.current_player if is_max else game.previous_player
    opponent = game.previous_player if is_max else game.current_player

    score = 0
    kernels = [np.ones(shape=(1, 2)), np.ones(shape=(2, 1)), np.eye(2), np.fliplr(np.eye(2))]
    for kernel in kernels:
        score += np.count_nonzero(convolve2d(game.board == me, kernel, mode='valid') == 2) * weight
        score -= np.count_nonzero(convolve2d(game.board == opponent, kernel, mode='valid') == 2) * weight

    return score

def mix_evaluator(game, is_max, *, markers=8, weight=10.0):
    """ Better evaluation function that combines the markers already placed and their combinations """
    markers_score = markers_evaluator(game, is_max, markers=markers)
    combination_score = combination_evaluator(game, is_max, weight=weight)

    return markers_score + combination_score


import torch
import torch.nn as nn
import pandas as pd
import numpy as np
from .useless_ids import useless_ids
from collections import Iterable

class FBaselineConfig:
    def __init__(self,
            anova_info_path="../data/anova_info.csv",
            f_cut = 150,
            out_dim = 3,
            dropout = 0.1
        ):
        self.anova_info_path = anova_info_path
        self.f_cut = f_cut
        self.out_dim = out_dim
        self.dropout = dropout

    def initialize(self):
        self.x_ids = self.cal_x_ids()
        self.in_dim = len(self.x_ids)
        self.n_hidden = min(self.in_dim // 6,512)

    def cal_x_ids(self):
        anova_info = pd.read_csv(self.anova_info_path)
        anova_info = anova_info[anova_info["f"] > self.f_cut]
        dims = []
        fs = []
        for i in range(len(anova_info)):
            f = "%.7f"%(anova_info["f"][i])
            if f not in fs and anova_info["dim"][i] not in useless_ids:
                fs.append(f)
                dims.append(anova_info["dim"][i])
        return np.array(dims)

class FullResLayer(nn.Module):
    def __init__(self,in_dim,dropout):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(in_dim,in_dim),
            nn.ReLU(),
            nn.Dropout(dropout),
            nn.Linear(in_dim,in_dim)
        )
        self.relu = nn.ReLU()

    def forward(self,x):
        return self.relu(self.net(x) + x)

class FBaseline(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.x_ids = config.x_ids

        self.net = nn.Sequential(
            nn.Dropout(0.65),
            nn.Linear(self.config.in_dim, self.config.n_hidden),
            nn.ReLU(),
            nn.Dropout(self.config.dropout),
            FullResLayer(self.config.n_hidden,self.config.dropout),
            nn.Dropout(self.config.dropout),
            FullResLayer(self.config.n_hidden,self.config.dropout),
            nn.Dropout(self.config.dropout),
            FullResLayer(self.config.n_hidden,self.config.dropout),
            nn.Dropout(self.config.dropout),
            nn.Linear(self.config.n_hidden, self.config.out_dim),
            nn.Softmax(dim=-1)
        )

    def _preprocess(self,x):
        if len(x.shape) == 1:
            return x[self.x_ids]
        if len(x.shape) == 2:
            return x[:,self.x_ids]

    def forward(self, x):
        """
        args:
            x : input of feaure
        return:
            x : possibility of three class
        Shape:
            input : [batch_size, in_dim]
            output : [batch_size,out_dim]
        """
        x = self._preprocess(x)
        x = self.net(x)
        return x

def test():
    config = FBaselineConfig()
    config.initialize()
    print(config.__dict__)
    model = FBaseline(config)
    x = torch.randn((32,28169))
    y = model(x)
    print(y)

if __name__ == "__main__":
    test()