#Clases
class Nodo(object):
    def __init__(self,hij=None,her=None,lin=0,cat=None,tip=None,cont=None,tabs=0):
        self.hijos=[]
        if hij is not None:
            for i in hij:
                self.hijos.append(i)
        self.hermano=her
        self.linea=lin
        self.categoria=cat
        self.tipo=tip
        self.contenido=cont
        self.tabuladores=tabs
        #Nuevo
        #Aqui le cambie
        self.value=None
        self.type=None

#clase para las lineas de los datos de la tabla hash
class LineListRec:
    linea=0;
    sig=None;

#clase para los datos de la tabla hash
class BucketListRec:
    name="";
    lines=LineListRec();
    memloc="";
    valor=None
    ttype=None
    sig=None;

#FUNCIONES

#funcion hash
def hasha(key):
    global size;
    global shift;
    temp=0;
    for i in key :
        temp=((temp << shift) + ord(i)) %size;
    return temp

#Funcion que inserta los numeros de linea en la tabla hash
def st_insert(name,linea,loc,val,ty):
    global hashtable;
    h=hasha(name)
    l=BucketListRec();
    l=hashtable[h];
    while((l!=None) and (name is not l.name)):
        l=l.sig;
    if(l==None): #Si la variable no se encuentra en la tabla
        l=BucketListRec()
        l.name=name
        l.lines=LineListRec();
        l.lines.linea=linea
        l.memloc=loc
        l.valor=val
        l.ttype=ty
        l.lines.sig=None
        l.sig=hashtable[h]
        hashtable[h]=l;
    else: #Si si se encuentra en la tabla
        t=LineListRec();
        t=l.lines;
        while(t.sig != None):
            t=t.sig;
        t.sig=LineListRec();
        t.sig.linea=linea;
        t.sig.sig=None;

#Funcion que actualiza en la tabla hash el valor que tiene la variable
def st_value(name,val,linea):
    global hashtable;
    global size;
    h=hasha(name);
    l=BucketListRec();
    l=hashtable[h];
    while((l!=None) and (name is not l.name)):
        l=l.sig;
    if(l==None): #Si no se encuentra en la tabla --> No esta declarada
        errores.append("Error en linea "+str(linea)+" : "+"Variable no declarada"+"  "+name);
        #print("Error en linea "+str(linea)+" : "+"Variable no declarada"+"  "+name);
    else: #Si se encuentra en la tabla cambia el valor de la variable
        l.valor=val
        #if(name=="a"):
         #   print("..."+str(val))

#Funcion que retorna la posicion en memoria de la variable
#Si esta no se encuentra retorna un -1
def st_lookup(name):
    global hashtable;
    global size;
    h=hasha(name);
    l=BucketListRec();
    l=hashtable[h];
    while((l!=None) and (name is not l.name)):
        l=l.sig;
    if(l==None): #Si la variable no se encuentra en la tabla
        return -1;
    else: #Si la variable se encuentra en la tabla regresa la posicion en memoria
        return l.memloc;

#Funcion que imprime en un archivo txt la tabla de simbolos y su contenido
def printSymTab(listing):
    i=0
    listing.write("VariableName    Location       Value    Type    LineNumbers\n")
    listing.write("------------    --------       -----    ----    -----------\n")
    for i in range(0,size):
        if(hashtable[i]!=None):
            l=BucketListRec();
            l=hashtable[i];
            while(l!=None): #Va imprimiendo el contenido de la tabla
                t=LineListRec();
                t=l.lines;
                listing.write("\t"+str(l.name))
                listing.write("\t   "+str(l.memloc))
                listing.write("\t\t"+str(l.valor))
                listing.write("\t"+l.ttype)
                listing.write("\t"+str(t.linea))
                t=t.sig;
                while(t!=None): #imprime todas las lineas en donde se encuentra la variable
                    listing.write(" "+t.linea)
                    t=t.sig;
                listing.write("\n")
                l=l.sig;

#Funcion recursiva que hace el recorrido a traves del arbol ya sea preorden o postorden
#de acuerdo a como se le indique
def traverse(raiz,preproc=None,postproc=None):
    if(raiz!=None):
        if(postproc==None):
            preproc(raiz)
            i=0
            for i in range(len(raiz.hijos)):
                traverse(raiz.hijos[i],preproc,postproc)
        else:
            i=0
            for i in range(len(raiz.hijos)):
                traverse(raiz.hijos[i],preproc,postproc)
            postproc(raiz)
        traverse(raiz.hermano,preproc,postproc)

#Funcion que inserta valores en la tabla
def insertNode(t):
    global location
    #Caso sentencia
    if(t.categoria=="Sentencia"):
        if(t.tipo=="asignacion" or t.tipo=="sentcin"):
            if(st_lookup(t.contenido) != -1):
                st_insert(t.contenido,t.linea,0,t.value,"")
    #Caso expresion
    elif(t.categoria=="Expresion"):
        if(t.tipo=="identificador"):
            if(st_lookup(t.contenido) != -1):
                st_insert(t.contenido,t.linea,0,t.value,"")
    #Caso declaracion
    elif(t.categoria=="Declaracion"):
        #Caso variable entera
        if(t.tipo=="tipo" and t.contenido=="int"):
            if(st_lookup(t.hijos[0].contenido) == -1):
                location+=1
                st_insert(t.hijos[0].contenido,t.linea,location,t.value,"int")
            else:
                typeError(t.hijos[0],"Variable ya declarada")
                t.hijos[0].type="int"
        #Caso variable real
        elif(t.tipo=="tipo" and t.contenido=="real"):
            if(st_lookup(t.hijos[0].contenido) == -1):
                location+=1
                st_insert(t.hijos[0].contenido,t.linea,location,t.value,"real")
            else:
                typeError(t.hijos[0],"Variable ya declarada")
                t.hijos[0].type="real"
        #Caso variable booleana
        elif(t.tipo=="tipo" and t.contenido=="boolean"):
            if(st_lookup(t.hijos[0].contenido) == -1):
                location+=1
                st_insert(t.hijos[0].contenido,t.linea,location,t.value,"boolean")
            else:
                typeError(t.hijos[0],"Variable ya declarada")
                t.hijos[0].type="boolean"
        #Caso declaraciones en una misma linea
        #Va pasandole su tipo a sus hermanos
        elif(t.tipo=="variable"):
            if(t.hermano!=None):
                if(st_lookup(t.hermano.contenido) == -1):
                    location+=1
                    st_insert(t.hermano.contenido,t.linea,location,t.value,t.type)
                else:
                    typeError(t.hermano,"Variable ya declarada")
                    t.hermano.valor="Error"

#Funcion que construye la tabla de simbolos recorriendo el arbol en preorden
def buildSymtab(t):
    traverse(t,insertNode,None)

#Funcion que recibe un nodo y un mensaje de error y lo imprime en el archivo de errores
def typeError(t,mensaje):
    errores.append("Error en linea "+t.linea+" : "+mensaje+"  "+t.contenido);
    #print("Error en linea "+t.linea+" : "+mensaje+"  "+t.contenido);

#Funcion que pasa todos los tipos en el arbol
def pasaTipo(t):
    if(t!=None):
        if(t.categoria=="Declaracion"):
            if(t.tipo=="tipo"):
                t.type=t.contenido
                t.hijos[0].type=t.contenido
            elif(t.tipo=="variable"):
                if(t.hermano!=None):
                    t.hermano.type=t.type

#Funcion que retorna el tipo de dato de la variable
#Si esta no se encuentra retorna un -1
def checkType(name):
    global hashtable;
    global size;
    h=hasha(name);
    l=BucketListRec();
    l=hashtable[h];
    while((l!=None) and (name is not l.name)):
        l=l.sig;
    if(l==None): #No se encuentra en la tabla hash
        return -1;
    else: #Retorna el tipo de dato de la variable
        return l.ttype;

#Funcion que checa que la variable este en la tabla de simbolos
#y retorna su valor, si no la encuentra retorna un -1
def checkValue(name):
    global hashtable;
    global size;
    h=hasha(name);
    l=BucketListRec();
    l=hashtable[h];
    while((l!=None) and (name is not l.name)):
        l=l.sig;
    if(l==None): #La variable no se encuentra en la tabla hash
        return -1;
    else: #Retorna el valor de la variable
        return l.valor;

#Funcion que hace todo el paso de tipos, de valores y chequeo de tipos
#en el arbol de analisis sintactico
def semantic(t):
    if(t!=None):
        #Pasa tipo a numeros
        if(t.tipo=="numero"):
            t.type="int"
            t.value=int(t.contenido)
        #Pasa tipo a reales
        if(t.tipo=="flotante"):
            t.type="real"
            t.value=float(t.contenido)
        #Si es una signacion o un identificador busca su tipo de dato en la tabla
        #y se lo asigna
        if(t.tipo=="asignacion" or t.tipo=="identificador"):
            if(st_lookup(t.contenido) != -1):
                t.type=checkType(t.contenido)
            else: #Caso variable no declarada
                typeError(t,"Variable no declarada")
                t.value="Error"
        #Asigna valores a las variables ya declaradas
        if(t.tipo=="identificador"):
            t.value=checkValue(t.contenido)

        #Caso expresion
        if(t.categoria=="Expresion"):
            if(t.tipo=="operador"):
                #Caso operadores ( +, -, *, /)
                if(t.contenido=="+" or t.contenido=="-" or t.contenido=="*" or t.contenido=="/"):
                    semantic(t.hijos[0])
                    semantic(t.hijos[1])

                    #Pasa el tipo al operador segun el tipo de sus hijos
                    if(t.hijos[0].type=="int" and t.hijos[1].type=="int"):
                        t.type="int"
                    elif(t.hijos[0].type=="real" and t.hijos[1].type=="int"):
                        t.type="real"
                    elif(t.hijos[0].type=="int" and t.hijos[1].type=="real"):
                        t.type="real"
                    elif(t.hijos[0].type=="real" and t.hijos[1].type=="real"):
                        t.type="real"

                    #else:
                     #   typeError(t,"Los tipos de dato no coinciden")
                      #  t.type="Error"

                    #Realiza operaciones de +, -, * y / y le asigna el valor al operador
                    if(t.hijos[0].value!="Error" and t.hijos[1].value!="Error"):
                        if(t.hijos[0].value != None and t.hijos[1].value != None):
                            if(t.contenido=="+"): #Caso suma
                                t.value = t.hijos[0].value + t.hijos[1].value
                            elif(t.contenido=="-"): #Caso resta
                                t.value = t.hijos[0].value - t.hijos[1].value
                            elif(t.contenido=="*"): #Caso multiplicacion
                                t.value = t.hijos[0].value * t.hijos[1].value
                            elif(t.contenido=="/"): #Caso division
                                if(t.hijos[1].value==0): #Caso division entre 0
                                    typeError(t.hijos[1],"Division entre 0")
                                else:
                                    t.value = t.hijos[0].value / t.hijos[1].value
                            #Caso resultado flotante y variable entera se castea
                            """
                            if(t.type=="int"):
                                t.value = int(t.value)
                            else:
                                t.value = float(t.value)"""
                        #Caso error, los hijos del operador no tienen un valor asignado
                        else:
                            if(t.hijos[0].value==None):
                                typeError(t.hijos[0],"No se ha asignado previamente valor a la variable ")
                            else:
                                typeError(t.hijos[1],"No se ha asignado previamente valor a la variable ")
                            t.value="Error"
                    else:
                        t.value="Error" #Caso los hijos son un error

                #Caso operadores ( <, >, <=, >=, !=, ==)
                elif(t.contenido=="<" or t.contenido=="<=" or t.contenido=="==" or t.contenido=="!=" or t.contenido==">=" or t.contenido==">"):
                    semantic(t.hijos[0])
                    semantic(t.hijos[1])
                    t.type="boolean" #Le asigna tipo de dato boolean al operador
                    #Hace las respectivas comparaciones entre los hijos del operador
                    if(t.hijos[0].value!="Error" and t.hijos[1].value!="Error"):
                        if(t.contenido=="<"): #Caso menor que
                            t.value = t.hijos[0].value < t.hijos[1].value
                        elif(t.contenido=="<="): #Caso menor o igual que
                            t.value = t.hijos[0].value <= t.hijos[1].value
                        elif(t.contenido=="=="): #Caso igual que
                            t.value = t.hijos[0].value == t.hijos[1].value
                        elif(t.contenido=="!="): #Caso diferente que
                            t.value = t.hijos[0].value != t.hijos[1].value
                        elif(t.contenido==">="): #Caso mayor o igual que
                            t.value = t.hijos[0].value >= t.hijos[1].value
                        elif(t.contenido==">"): #Caso mayor que
                            t.value = t.hijos[0].value > t.hijos[1].value
                    #Caso en que los hijos no tienen un valor asignado
                    elif(t.hijos[0].value!=None and t.hijos[1].value!=None):
                        typeError(t,"No se ha asignado valor a una variable")
                        t.value="Error"
                    else:
                        typeError(t,"No se puede comparar un valor")
                        t.value="Error"

        #Caso asignacion
        if(t.tipo=="asignacion"):
            if(t.type=="real" and t.hijos[0].type=="int"):
                if(t.hijos[0].value!="Error"):
                    t.value=float(t.hijos[0].value)
                    st_value(t.contenido,t.value,t.linea)

            elif(t.value !="Error" and t.hijos[0].value!="Error"):
                if(t.type!=t.hijos[0].type):
                    typeError(t,"No se puede asignar un tipo de dato distinto")
                else:
                    t.value = t.hijos[0].value
                    st_value(t.contenido,t.value,t.linea)

#Funcion que imprime el arbol en el archivo
def imprime(arbol,tabs):
    if(arbol != None):
        arbol.tabs = tabs
        texto = ""
        for i in range(tabs):
            texto+="\t"
        if(arbol.type==None): #Caso tipos None
            arbol.type="_"
        if(arbol.value==None): #Caso valores None
            arbol.value="_"
        print( texto + " " + arbol.contenido+".."+arbol.type+".."+str(arbol.value))
        outfile = open('MuestraSemantico.txt', 'a')
        outfile.write( texto + " " + arbol.contenido+" ["+arbol.type+"] ["+str(arbol.value)+ "]\n")
        outfile.close()
        for i in range (len(arbol.hijos)):
            imprime(arbol.hijos[i],tabs+1)
        imprime(arbol.hermano,tabs)

def semantica():
    global hashtable
    #Funciones a realizar
    arbol=sintaxis() #Recupera el arbol del sintactico
    traverse(arbol,pasaTipo,None) #Hace el recorrido preorden pasando los tipos
    buildSymtab(arbol) #Construye la tabla de simbolos
    traverse(arbol,None,semantic) #Hace el recorrido postorden haciendo las operaciones y recuperando errores
    return arbol,hashtable

def archivos_errores():
    global hashtable
    arbol,hashtable=semantica()
    #Abre el archivo de la tabla hash, la imprime y lo cierra
    listing=open('hashtable.txt','w')
    listing.write("\nSymbol table:\n\n")
    printSymTab(listing);
    listing.close()

    #Crea el archivo con el arbol que se mostrara en el ide
    outfile2 = open('MuestraSemantico.txt', 'w')

    #Cre el archivo de errores
    outfile = open('Errores.txt', 'a')
    outfile.write("\n\n\tERRORES SEMANTICO\n\n")

    #Imprime los errores en el archivo
    for i in range(0,len(errores)):
        outfile.write(errores[i]+"\n")

    outfile.close()
    print(type(arbol))
    imprime(arbol,0) #imprime el arbol en el archivo

    outfile2.close()

    #imprime los errores en consola
    for i in errores:
        print("["+i+"]")

#PRINCIPAL
from sintactico import sintaxis

#Variables globales
global actual
actual=0
errores=[]
global location
location=0
global size #Variable que guarda el tamanio de la tabla hash
size=211
global shift #Variable usada como multiplicador de la funcion hash
shift=4
global hashtable #Tabla hash
hashtable=[]
hashtable=[None]*size
global arbol
arbol = Nodo()


# page 33
import numpy as np

# Passed in gradient from the next layer for the purpose of this example
# we're going to use an array of an incremental gradient values
dvalues = np.array([
    [1., 1., 1],
    [2., 2., 2.],
    [3., 3., 3.]
])

# We have 3 sets of weights - one set for each neuron
# we have 4 inputs, thus 4 weights
# recall that we keep weights tranposed
weights = np.array([
    [0.2, 0.8, -0.5, 1],
    [0.5, -0.91, 0.26, -0.5],
    [-0.26, -0.27, 0.17, 0.87]
]).T

# sum weights of given input
# and multiply by the passed in gradient for this neuron
dinputs = np.dot(dvalues, weights.T)
print(dinputs)


z.destroy()
del z

import DBPy.dbzmq
import ctypes
z = DBPy.dbzmq.DBzmq()

z.send(b'\x0A\x01'+ctypes.c_ulong(0xBA1350))
z.send(b'\x0A\x03'+ctypes.c_ulong(0xBA1350))

for i in z.recv_all():print(i)

[str(i[1:-1]) for i in z.recv_all() if i[0]==2]


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.optim as optim
import torchtext.vocab as vocab

import numpy as np
import time
import os
from six.moves import cPickle
import torch.backends.cudnn as cudnn
import yaml

import opts
import misc.eval_utils
import misc.utils as utils
import misc.AttModel as AttModel
import yaml

# from misc.rewards import get_self_critical_reward
import torchvision.transforms as transforms
import pdb
import argparse
import torch.nn.functional as F
import matplotlib.pyplot as plt
from PIL import Image
plt.switch_backend('agg')
import json
import pickle
import sys
sys.path.append('../faster-rcnn.pytorch/')
from get_ppls import prep_model, get_caption

def decode_sequence_simple(itow, itod, ltow, itoc, wtod, seq, bn_seq, fg_seq, vocab_size, opt):
    N, D = seq.size()
    itod = np.asarray(['__background__',
                       'aeroplane', 'bicycle', 'bird', 'boat',
                       'bottle', 'bus', 'car', 'cat', 'chair',
                       'cow', 'diningtable', 'dog', 'horse',
                       'motorbike', 'person', 'pottedplant',
                       'sheep', 'sofa', 'train', 'tvmonitor'])
    out = []
    for i in range(N):
        txt = ''
        for j in range(D):
            if j >= 1:
                txt = txt + ' '
            ix = seq[i,j].item()
            if ix > vocab_size:
                word = itod[int(fg_seq[i,j].item())]
                # word = '[ ' + word + ' ]'
            else:
                if ix == 0:
                    break
                else:
                    word = itow[str(ix)]
            txt = txt + word
        out.append(txt)
    return out

def demo_simple(obj_det_model, model, img_name):
    model.eval()
    file_path = img_name
    max_proposal = 200

    num_proposal = 6
    num_nms = 6

    # load the image.
    input_imgs = torch.FloatTensor(1)
    img = Image.open(file_path).convert('RGB')
    # resize the image.
    img = transforms.Resize((opt.image_crop_size, opt.image_crop_size))(img)

    ppls = get_caption(obj_det_model, img_name)
    pad_proposals = torch.cat([i.unsqueeze(dim=0) for i in ppls]).unsqueeze(dim=0)
#     pad_proposals = torch.from_numpy(pad_proposals).float().unsqueeze(dim=0)
    num = torch.FloatTensor([1, len(ppls), len(ppls)]).unsqueeze(dim=0)

    img = transforms.ToTensor()(img)
    img = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(img).unsqueeze(dim=0)

    img = img.cuda()
    pad_proposals = pad_proposals.cuda()
    num = num.cuda()

    eval_opt = {'sample_max':1, 'beam_size': opt.beam_size, 'inference_mode' : True, 'tag_size' : opt.cbs_tag_size}
    seq, bn_seq, fg_seq, _, _, _ = model._sample(img, pad_proposals, num, eval_opt)
    sents = utils.decode_sequence(dataset.itow, dataset.itod, dataset.ltow, dataset.itoc, dataset.wtod, \
                                    seq.data, bn_seq.data, fg_seq.data, opt.vocab_size, opt)
    return sents

def init_model(opt):
    cudnn.benchmark = True

    ####################################################################################
    # Build the Model
    ####################################################################################
    from misc.dataloader_coco import DataLoader
    dataset = DataLoader(opt, split='train')
    opt.vocab_size = dataset.vocab_size
    opt.detect_size = dataset.detect_size
    opt.seq_length = opt.seq_length
    opt.fg_size = dataset.fg_size
    opt.fg_mask = torch.from_numpy(dataset.fg_mask).byte()
    opt.glove_fg = torch.from_numpy(dataset.glove_fg).float()
    opt.glove_clss = torch.from_numpy(dataset.glove_clss).float()
    opt.glove_w = torch.from_numpy(dataset.glove_w).float()
    opt.st2towidx = torch.from_numpy(dataset.st2towidx).long()

    opt.itow = dataset.itow
    opt.itod = dataset.itod
    opt.ltow = dataset.ltow
    opt.itoc = dataset.itoc

    if not opt.finetune_cnn: opt.fixed_block = 4 # if not finetune, fix all cnn block

    if opt.att_model == 'topdown':
        model = AttModel.TopDownModel(opt)
    elif opt.att_model == 'att2in2':
        model = AttModel.Att2in2Model(opt)

    infos = {}
    histories = {}
    if opt.start_from is not None:
        if opt.load_best_score == 1:
            model_path = os.path.join(opt.start_from, 'model-best.pth')
            info_path = os.path.join(opt.start_from, 'infos_'+opt.id+'-best.pkl')
        else:
            model_path = os.path.join(opt.start_from, 'model.pth')
            info_path = os.path.join(opt.start_from, 'infos_'+opt.id+'.pkl')

            # open old infos and check if models are compatible
        with open(info_path, 'rb') as f:
            infos = pickle.load(f, encoding='latin1')
            saved_model_opt = infos['opt']

        # opt.learning_rate = saved_model_opt.learning_rate
        print('Loading the model %s...' %(model_path))
        model.load_state_dict(torch.load(model_path))

        if os.path.isfile(os.path.join(opt.start_from, 'histories_'+opt.id+'.pkl')):
            with open(os.path.join(opt.start_from, 'histories_'+opt.id+'.pkl'), 'rb') as f:
                histories = pickle.load(f, encoding='latin1')

    if opt.decode_noc:
        model._reinit_word_weight(opt, dataset.ctoi, dataset.wtoi)

    best_val_score = infos.get('best_val_score', None)
    iteration = infos.get('iter', 0)
    start_epoch = infos.get('epoch', 0)

    val_result_history = histories.get('val_result_history', {})
    loss_history = histories.get('loss_history', {})
    lr_history = histories.get('lr_history', {})
    ss_prob_history = histories.get('ss_prob_history', {})

    if opt.cuda:
        model.cuda()

    params = []
    # cnn_params = []
    for key, value in dict(model.named_parameters()).items():
        if value.requires_grad:
            if 'cnn' in key:
                params += [{'params':[value], 'lr':opt.cnn_learning_rate,
                        'weight_decay':opt.cnn_weight_decay, 'betas':(opt.cnn_optim_alpha, opt.cnn_optim_beta)}]
            else:
                params += [{'params':[value], 'lr':opt.learning_rate,
                    'weight_decay':opt.weight_decay, 'betas':(opt.optim_alpha, opt.optim_beta)}]

    print("Use %s as optmization method" %(opt.optim))
    if opt.optim == 'sgd':
        optimizer = optim.SGD(params, momentum=0.9)
    elif opt.optim == 'adam':
        optimizer = optim.Adam(params)
    elif opt.optim == 'adamax':
    	optimizer = optim.Adamax(params)

    obj_det_model = prep_model('../faster-rcnn.pytorch/data_fr/pretrained_model/faster_rcnn_coco.pth')

    return obj_det_model, model

#     img_name = 'test_images/COCO_val2014_000000184791.jpg'
#     pics = ['COCO_val2014_000000184791.jpg','COCO_val2014_000000550529.jpg',
#             'COCO_val2014_000000579664.jpg','COCO_val2014_000000348881.jpg','COCO_val2014_000000560623.jpg']
#     obj_det_model = prep_model('../faster-rcnn.pytorch/data_fr/pretrained_model/faster_rcnn_coco.pth')
#     for pic in pics:
#         sents = demo_simple(obj_det_model, model, 'test_images/'+pic)
#         print(sents)
if __name__=='__main__':
    sents = demo_simple(obj_det_model, model, 'data/coco/images/val2014/COCO_val2014_000000184791.jpg')
    print(sents)


# -*- coding: UTF-8 -*-
""""
Created on 27.01.21

Module containing buffers.

:author:     Martin Dočekal
"""
from typing import TextIO

class PrintBuffer:
    """
    Stores and prints data marked with serial numbers in sorted order.

    Use unique serial numbers else the data will be overwritten.

    Usage:

        pBuffer = PrintBuffer()
        pBuffer.print(1, "B")   # stores B because the next printed string should have serial number 0
        pBuffer.print(2, "C")   # stores C because the next printed string should have serial number 0
        pBuffer.print(0, "A")   # prints
        ->  A
            B
            C
        pBuffer.print(3, "C")  # print immediately because 3 is the next serial number
        ->  C

    :var file_out: Where the output should be printed.
    :var print_flush: flush parameter for print method
    :var end: String appended to the last character before print. Default is a newline.
    """

    def __init__(self, file_out: TextIO, print_flush: bool = False, end: str = "\n"):
        """
        initialization of buffer

        :param file_out: Where the output should be printed.
        :type file_out: TextIO
        :param print_flush: flush parameter for print method
        :type print_flush: bool
        :param end: String appended to the last character before print. Default is a newline.
        :type end: str
        """
        self._waiting_for = 0
        self._buffer = {}
        self.fileOut = file_out
        self.printFlush = print_flush
        self.end = end

    def _print(self, value: str):
        """
        Just wrapper for print.

        :param value: Value that should be printed.
        :type value: str
        """
        print(value, file=self.fileOut, flush=self.printFlush, end=self.end)

    @property
    def waiting_for(self) -> int:
        """
        Serial number the buffer is waiting for.
        """
        return self._waiting_for

    def print(self, serial_number: int, value: str) -> bool:
        """
        Prints value if serial number is the one it is waiting for, else stores that value for later.
        If serial number is the one it is waiting for and it also has stored values with consequent serial numbers
        it will print them too.

        Use unique serial numbers else the data will be overwritten.

        :param serial_number: Serial number of actual value.
        :type serial_number: int
        :param value: String that can be printed.
        :type value: str
        :return: True if value (or multiple values) was printed. False when the value was buffered.
        :rtype: bool
        """

        if serial_number == self._waiting_for:
            self._print(value)

            self._waiting_for += 1

            while self._waiting_for in self._buffer:
                self._print(self._buffer[self._waiting_for])
                del self._buffer[self._waiting_for]
                self._waiting_for += 1

            return True
        else:
            self._buffer[serial_number] = value
            return False

    def flush(self):
        """
        Prints out all data stored in buffer in ascending sorted order according to their serial number.

        Sets the waitingFor to the next serial number after the biggest one in memory.
        Do nothing if buffer is empty.
        """
        serial_number = self._waiting_for - 1  # will guarantee that if the buffer is empty the waitingFor does not change

        for serial_number in sorted(self._buffer.keys()):
            self._print(self._buffer[serial_number])
            del self._buffer[serial_number]

        self._waiting_for = serial_number + 1

    def __len__(self):
        """
        Number of stored items.
        """

        return len(self._buffer)

    def clear(self):
        """
        Clears buffer without printing items inside and set waitingFor to zero.
        """
        self._buffer = {}
        self._waiting_for = 0


import functools

def _cross_join():
    df = self.players_hands[self.other_players_list[0][0]].assign(key=1).query('posterier_prob > 0')
    for player in self.other_players_list[1:]:
        df_next = self.players_hands[player[0]].assign(key=1).query('posterier_prob > 0')
        df = df.merge(df_next, how='left', on='key')
    df.drop('key', 1, inplace=True)
    return df

def _find_plausible_combos(df):
    print(df.head())

def envelope_distribution(hand_dict):
    # todo: 1. toy hands, 2. check output, 3. plausible combos

    hand_dist_lengths = [len(self.players_hands[player[0]].query('posterior_prob > 0')) for player in
                         self.other_players_list]
    if functools.reduce(lambda x, y: x * y, hand_dist_lengths) < 1000:
        df = self._cross_join().pipe(self._find_plausible_combos)

if __name__ == '__main__':
    hands = {'Calvin': '(a, b, c)', 'Kay': '(d, e, f'}

    envelope_distribution()

# # players = 2
# # cards_num = 2
# # def hand1(h):
# #     player_hand = []
# #     for envelope in itertools.combinations(h, 3):
# #         h_minus_envelope = [card for card in h if card not in envelope]
# #         for hand in hand2(envelope, h_minus_envelope):
# #             player_hand.append([str(envelope)] + hand)
# #
# #     return player_hand
# #
# # def hand2(e, h):
# #     player_hand_lst = []
# #     while len(h) > cards_num:
# #         for player_hand in itertools.combinations(h, 2):
# #             h_minus_envelope = [card for card in h if card not in player_hand]
# #             h2 = hand2(e, h_minus_envelope)
# #             if type(h2) == tuple:
# #                 player_hand_lst.append([str(player_hand), str(h2)])
# #             else:
# #                 for hand in h2:
# #                     player_hand_lst.append([str(player_hand)] + hand)
# #         return player_hand_lst
# #     return tuple(h)
#
# # todo: convert to pandas dataframe
# # todo: clean up this nasty code
#
#
#
#
#
#
#
#
#
#
#     def _possible_hands(self):
#         player_hand = []
#         # 112
#         # 924 for 3 players, ~84k for 4, ~something huge for 6 (just think, 18 players would be 17!)
#         for envelope in itertools.product(self.remaining_rooms, self.remaining_suspects, self.remaining_weapons):
#             remaining_cards = [card for card in self.remaining_rooms + self.remaining_suspects + self.remaining_weapons if card not in envelope]
#
#             for hand in self._hands_create(remaining_cards):
#                 player_hand.append([str(envelope)] + hand)
#
#             # columns = ['envelope'] + [players[0] for players in self.other_players_list]
#             # print(pd.DataFrame(player_hand, columns=columns).head(50))
#             # sys.exit()
#             print(len(player_hand))
#             sys.exit()
#         columns = ['envelope'] + [players[0] for players in self.other_players_list]
#         return pd.DataFrame(player_hand, columns=columns)
#
#     def _hands_create(self, cards):
#         hands_lst = []
#
#         last_player_card_count = self.other_players_list[-1][1]
#         while len(cards) > last_player_card_count:
#             next_player = next(self.player_generator)[1]
#             for player_hand in itertools.combinations(cards, next_player):
#                 remaining_cards = [card for card in cards if card not in player_hand]
#                 hand_combination = self._hands_create(remaining_cards)
#                 if type(hand_combination) == tuple:
#                     hands_lst.append([str(player_hand), str(hand_combination)])
#                 else:
#                     for hand in hand_combination:
#                         hands_lst.append([str(player_hand)] + hand)
#             return hands_lst
#         return tuple(cards)
#
#     def hands_update(self, player, cards, answer='no'):
#         pass
#
#
#     self.yes_dict = {player[0]: [] for player in self.other_players_list}
#     self.yes_uncertain_dict = {player[0]: [] for player in self.other_players_list}
#     self.no_dict = {player[0]: [] for player in self.other_players_list}
#     def card_reveal(self, player, card):
#         """player shows me card"""
#         self.yes_dict[player] += card
#
#     def turn(self, inquiry, player, answer='no'):
#         """'player' responds to the 'inquiry' with a 'no' or 'yes' by showing one of his cards. Cards are added to a
#         list in either yes_dict or no_dict. If a 'no' response is given, the card values are added directly to the list.
#         Otherwise, the list of cards is added to the list."""
#         if answer == 'no':
#             self.no_dict[player] += inquiry
#         else:
#             self.yes_uncertain_dict[player].append(inquiry)
#
#
#     def players_hand_create(self):
#
#         self.remaining_rooms + self.remaining_suspects + self.remaining_weapons
#
# # why don't I simply, for each player, (18 choose whatever).
# # for each player, create a player hand class.
#
#
# # what if I had a hand of potential cards for each of the players and as they say no I widdle each down. Then I somehow use this to construct the possibilities
# # I should calculate the number of possibly hands as well and when it gets down to something reasonable, create the dataframe
# # It should be pretty straightforward to calculate the possible number of hands
# # Then I'd update the dataframe using the historical and future data
# # don't be explicit about possible envelopes...I can back that out.
#


from waldur_core.core.log import AuthEventMixin
from waldur_core.core.models import User
from waldur_core.logging.loggers import EventLogger, event_logger

class Saml2AuthEventLogger(AuthEventMixin, EventLogger):
    user = User

    class Meta:
        event_types = ('auth_logged_in_with_saml2', 'auth_logged_out_with_saml2')
        event_groups = {'users': event_types}

    @staticmethod
    def get_scopes(event_context):
        return {event_context['user']}

event_logger.register('saml2_auth', Saml2AuthEventLogger)


# This class is used to create a file list of all the .gz data filenames

import os

class FileList():
	"""Create a object of raw data file path list."""

	def __init__(self,path):

		# Path of file to open.
		# File path = dataDirectory + '201607/' + 'SPTCC-20160701.csv.gz'

		self.file_list=[]

		# Date format: 20160709
		self.date = []

		self.sys_dir = os.path.join(os.path.expanduser('~'), path, 'smartcard', '')
		self.data_dir = os.path.join(os.path.expanduser('~'), path, 'smartcard', 'data', '')

		months = ['201607', '201608', '201609']
		for mon in months:
			for day in range(31):

				if not (mon == '201609' and day == 30): # September has no 31th day

					if (day + 1) <= 9:
						tempstr = '0'
					else:
						tempstr = ''

					self.file_list.append(os.path.join(self.data_dir + mon, 'SPTCC-' + mon + tempstr + str(day + 1) + '.csv.gz'))

					if day < 9:
						self.date.append(mon + '0' + str(day + 1))
					else:
						self.date.append(mon + str(day + 1))


import maya.cmds as cmds
import os
import random
import pymel.core as pm
import socket
import sys
import maya.mel as mm
import csv

recstart = 0
selectedBlend = ""
dataarray = []
gnumcurrenttime = 1

"""
menulabelarr = ['Brow Left UP','Brow Left Down','Brow Right UP','Brow Right Down','Brow Centering','Brow outer left down','Brow outer right down','Eye Close Left','Eye Close Right','Mouse Open',
                'Mouse Left Smile','Mouse Right Smile','Mouse Left Spread','Mouse Right Spread','Mouse Left Frawn','Mouse Right Frawn','Mouse Left Centering','Mouse Right Centering','Cheek Left UP',
                'Cheek Right UP']
"""

# Our mel global proc.
melproc = """
global proc portData(string $arg){
    python(("portData(\\"" + $arg + "\\")"));
}
"""

mm.eval(melproc)

def edit_cell(row, column, value):
    return 1

def startrealtimeexp(*args):
    global recstart
    global recend
    #numrow = cmds.scriptTable('scrtable', query=True, rows=True)
    if len(selectedBlend) >= 0:
        if recstart == 0:
            recstart = 1
            cmds.button( 'realtimecomm' ,edit=True, label = 'Stop Real Time Expression' )
	    #Start Comm
            cmds.commandPort(name="127.0.0.1:7777", echoOutput=False, noreturn=False,prefix="portData", returnNumCommands=True)
            cmds.commandPort(name=":7777", echoOutput=False, noreturn=False,prefix="portData", returnNumCommands=True)
        else:
            recstart = 0
            cmds.button( 'realtimecomm' ,edit=True, label = 'Start Real Time Expression' )
	    #Stop Comm
            deactivateCommandPort('127.0.0.1', '7777')
'''
def delete_sel_row(*args):
    if recstart == 0:
        try:
            selected_row = cmds.scriptTable('scrtable', query=True, selectedRows=True)[0]
            if selected_row == None:
                print('Select Row to Delete')
            else:
                cmds.scriptTable('scrtable', edit=True,deleteRow=selected_row)
        except:
            print('Select Row to Delete')
'''

'''
def savepresetfile(*args):
    blendshapetxt = cmds.textField('selectedBlendShapeText', q=True, tx=True )
    filename = blendshapetxt + "_mayapipeline_setting.txt"
    o_file = os.path.join(os.path.join(os.environ['USERPROFILE']), 'Desktop') + "\\"+ filename

    all_rows = cmds.scriptTable('scrtable', query=True, rows=True)

    if all_rows < 2:
        return 0

    if not (cmds.file(o_file,query=True, exists=True)):
        tmp_csv_file = open(o_file, 'w' ,os.O_CREAT)
    else:
        tmp_csv_file = open(o_file, 'w')
    writer = csv.writer(tmp_csv_file, lineterminator='\n')

    all_rows = cmds.scriptTable('scrtable', query=True, rows=True)
    for i in range(all_rows):
        all_colums = cmds.scriptTable('scrtable', query=True, columns=True)
        data_list = []
        for j in range( all_colums - 1):
            if i == 0:
                blendshapetxt = cmds.textField('selectedBlendShapeText', q=True, tx=True )
                headbonenametxt = cmds.textField('HeadbonenameF', q=True, tx=True )
                headxtxt = cmds.textField('HeadbonenameXF', q=True, tx=True )
                headytxt = cmds.textField('HeadbonenameYF', q=True, tx=True )
                headztxt = cmds.textField('HeadbonenameZF', q=True, tx=True )
                porttxt = cmds.textField('Portnum', q=True, tx=True )
                data_list = [blendshapetxt, headbonenametxt, headxtxt, headytxt, headztxt, porttxt]

            else:
                cell_list = cmds.scriptTable('scrtable', cellIndex=(i,j + 1), query=True, cellValue=True)
                if j == 0:
                    if type(cell_list) == list:
                        cell_text = "".join(cell_list)
                        print(cell_text)
                    elif cell_list == None:
                        cell_text = u''
                    else:
                        cell_text = cell_list
                    data_list.append(cell_text)
                elif j == 1:
                    if type(cell_list) == list:
                        cell_text = "".join(cell_list)
                        print(cell_text)
                    elif cell_list == None:
                        cell_text = u''
                    else:
                        cell_text = cell_list
                    data_list.append(cell_text)
                else:
                    if type(cell_list) == list:
                        cell_text = "".join(cell_list)
                    elif cell_list == None:
                        cell_text = u''
                    else:
                        cell_text = cell_list
                    data_list.append(cell_text)
        writer.writerow(data_list)
    tmp_csv_file.close()
    cmds.warning( "Saved file (" +o_file+ ")" )
'''

'''
def loadpresetfile(*args):
    if recstart != 0:
        return 0

    presetfile = pm.fileDialog2(fileMode=1)

    if presetfile:
        cmds.scriptTable('scrtable', edit=True, selectedRows=[])

        o_file = open(str(presetfile[0]), 'r')
        reader = csv.reader(o_file)

        all_rows = cmds.scriptTable('scrtable', query=True, rows=True)
        for i in range(all_rows):
            cmds.scriptTable('scrtable', edit=True,deleteRow= i)

        row_no = 0
        for row in reader:
            if row_no > 0:
                cmds.scriptTable('scrtable', edit=True,insertRow=row_no)
                cmds.scriptTable('scrtable', cellIndex=(row_no,1), edit=True, cellValue=row[0])
                cmds.scriptTable('scrtable', cellIndex=(row_no,2), edit=True, cellValue=row[1])
                cmds.scriptTable('scrtable', cellIndex=(row_no,3), edit=True, cellValue="0")
                cmds.scriptTable('scrtable', cellIndex=(row_no,4), edit=True, cellValue=row[3])
                cmds.scriptTable('scrtable', cellIndex=(row_no,5), edit=True, cellValue=row[4])
                cmds.scriptTable('scrtable', cellIndex=(row_no,6), edit=True, cellValue=row[5])
            else:
                cmds.textField('selectedBlendShapeText', edit=True, text=row[0] )
                cmds.textField('HeadbonenameF', edit=True, text=row[1] )
                cmds.textField('HeadbonenameXF', edit=True, text=row[2] )
                cmds.textField('HeadbonenameYF', edit=True, text=row[3] )
                cmds.textField('HeadbonenameZF', edit=True, text=row[4] )
                #cmds.textField('Portnum', edit=True, text=row[5] )
                loadTargetList()
            row_no = 1 + row_no
        o_file.close()
'''

def expTrackerWindow():
    if cmds.window('expTrackerWindow', exists = True):
        cmds.deleteUI('expTrackerWindow')

    #window def
    cmds.window('expTrackerWindow',widthHeight=(900,400),title='Mustache_Boy_Facial_Mocap-conelab',minimizeButton=False,maximizeButton=False,resizeToFitChildren = True, sizeable = True)
    #cmds.rowColumnLayout(numberOfColumns=3,columnWidth=[(1,300),(2,300),(3,300)],backgroundColor=[200,200,0])

    '''
    #select an existing blendShape node
    cmds.rowColumnLayout(numberOfColumns=6,columnWidth=[(1,150),(2,25),(3,300),(4,25),(5,200),(6,200)],backgroundColor=[1,0.9843,0.7961])
    cmds.text( label='Blend Shape node name : ' ,align='right' )
    cmds.text( label='' )
    selectedBlendShapeTextField = cmds.textField( 'selectedBlendShapeText' ,backgroundColor=[1,1,1])
    cmds.text( label='' )
    cmds.button( label = 'Load Targets', command = 'loadTargetList()', width=20, align='left',backgroundColor=[0.3412,0.8196,0.7882])
    cmds.text( label='' )

    cmds.columnLayout('temp1', width=900)
    cmds.rowColumnLayout(numberOfColumns=6,columnWidth=[(1,300),(2,40),(3,300),(4,40),(5,200),(6,20)])
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )

    targetList = cmds.optionMenu( 'targetObjectMenu', label='Target: ',backgroundColor=[1,0.9098,0.4118])
    cmds.menuItem(label=' ' )
    cmds.text( label='' )
    paramList = cmds.optionMenu( 'paramObjectMenu', label='Link: ',backgroundColor=[1,0.9098,0.4118] )
    cmds.menuItem(label='Please select param' )
    cmds.menuItem(label='Brow Left UP')
    cmds.menuItem(label='Brow Left Down')
    cmds.menuItem(label='Brow Right UP')
    cmds.menuItem(label='Brow Right Down')
    cmds.menuItem(label='Brow Centering')
    cmds.menuItem(label='Brow outer left down')
    cmds.menuItem(label='Brow outer right down')
    cmds.menuItem(label='Eye Close Left')
    cmds.menuItem(label='Eye Close Right')
    cmds.menuItem(label='Mouse Open')
    cmds.menuItem(label='Mouse Left Smile')
    cmds.menuItem(label='Mouse Right Smile')
    cmds.menuItem(label='Mouse Left Spread')
    cmds.menuItem(label='Mouse Right Spread')
    cmds.menuItem(label='Mouse Left Frawn')
    cmds.menuItem(label='Mouse Right Frawn')
    cmds.menuItem(label='Mouse Left Centering')
    cmds.menuItem(label='Mouse Right Centering')
    cmds.menuItem(label='Cheek Left UP')
    cmds.menuItem(label='Cheek Right UP')
    cmds.text( label='' )
    cmds.button( label = 'Add link', command = 'addlink()' ,backgroundColor=[0.3412,0.8196,0.7882])
    cmds.text( label='' )

    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    '''

    # start Real Time Expression 뒷 배경.
    cmds.columnLayout('mytable', width=900,backgroundColor=[1,0.9098,0.4118])

    '''
    table =  cmds.scriptTable('scrtable',rows=0, columns=6,columnWidth=([1,300],[2,300],[3,145],[4,145],[5,1],[6,1]),
    	label=[(1,"Target"), (2,"Link"), (3,"Receive val"), (4,"Strength"), (5,"ID(Target)"), (6,"ID(Link)")], width=900,
    	cellChangedCmd=edit_cell)
    '''
    #cmds.rowColumnLayout(numberOfColumns = 3, columnWidth = [(1,100),(2, 300), (3, 500)])

    '''
    cmds.text( label='' )
    cmds.text( label='' )
    #cmds.button( label = 'Delete select row',command=delete_sel_row,backgroundColor=[0.3412,0.8196,0.7882] )

    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )

    #cmds.columnLayout('temp2', width=900)
    cmds.rowColumnLayout(numberOfColumns = 4, columnWidth = [(1,100),(2, 300), (3, 300),(4,200)],backgroundColor=[1,0.9098,0.4118])

    cmds.text( label='' )
    #cmds.text( label='Head bone name' )
    #Headbonename = cmds.textField('HeadbonenameF')
    cmds.text( label='' )

    cmds.text( label='' )
    #cmds.text( label='Head bone rotation compensation X' )
    #HeadbonenameX = cmds.textField('HeadbonenameXF',text='0')
    cmds.text( label='' )

    cmds.text( label='' )
    #cmds.text( label='Head bone rotation compensation Y' )
    #HeadbonenameY = cmds.textField('HeadbonenameYF',text='0')
    cmds.text( label='' )

    cmds.text( label='' )
    #cmds.text( label='Head bone rotation compensation Z' )
    #HeadbonenameZ = cmds.textField('HeadbonenameZF',text='0')
    cmds.text( label='' )
    '''

    cmds.columnLayout('temp3', width=900)
    cmds.rowColumnLayout(numberOfColumns=5,columnWidth=[(1,200),(2,50),(3,200),(4,150),(5,300)])

    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )

    #cmds.button( label = 'Save preset', command = savepresetfile,backgroundColor=[0.3412,0.8196,0.7882] )
    cmds.text( label='' )
    #cmds.button( label = 'Load preset', command = loadpresetfile,backgroundColor=[0.3412,0.8196,0.7882] )
    cmds.text( label='' )
    cmds.button( 'realtimecomm', label = 'Start Real Time Expression', command = startrealtimeexp ,backgroundColor=[0.9294,0.3294,0.5216])

    #cmds.button(label='close', command=('cmds.deleteUI(\"' + window + '\", window=True)'), backgroundColor=[0.9294,0.3294,0.5216])

    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )
    cmds.text( label='' )

    cmds.showWindow( 'expTrackerWindow' )

expTrackerWindow()

'''
if (cmds.window("expTrackerWindow", exists=True)):
    cmds.deleteUI("expTrackerWindow")
'''

'''
#function that loads target list to the UI

def loadTargetList():
    global selectedBlend
    selectedBlend = cmds.textField('selectedBlendShapeText', q=True, tx=True )

    existingItems = cmds.optionMenu( 'targetObjectMenu', q=True, itemListLong=True )
    if existingItems != None and existingItems != []:
        cmds.deleteUI(existingItems)

    cmds.menuItem(label='Please select blendShape', parent='targetObjectMenu')

    blendTargets = cmds.listAttr (selectedBlend + '.w', m = True)
    for target in blendTargets:
        cmds.menuItem(label='%s' %(target), parent='targetObjectMenu')
'''

'''
def addlink():
    if recstart == 0:
        selectedBlend = cmds.optionMenu('targetObjectMenu', query=True, value=True )
        selectedBlendI = cmds.optionMenu('targetObjectMenu', query=True, select=True )
        selectedLinkBlend = cmds.optionMenu('paramObjectMenu', query=True, value=True )
        selectedLinkBlendI = cmds.optionMenu('paramObjectMenu', query=True, select=True )

        if selectedBlendI > 1 and selectedLinkBlendI > 1:
            add_row();
'''

'''
def add_row(*args):
    cmds.scriptTable('scrtable', edit=True, selectedRows=[])

    selectedBlend = cmds.optionMenu('targetObjectMenu', query=True, value=True )
    selectedBlendI = cmds.optionMenu('targetObjectMenu', query=True, select=True )
    selectedLinkBlend = cmds.optionMenu('paramObjectMenu', query=True, value=True )
    selectedLinkBlendI = cmds.optionMenu('paramObjectMenu', query=True, select=True )

    last_row_num = cmds.scriptTable('scrtable', query=True, rows=True)
    cmds.scriptTable('scrtable', edit=True,insertRow=last_row_num)

    cmds.scriptTable('scrtable', e=True, ci=[last_row_num, 1], cv=selectedBlend)
    cmds.scriptTable('scrtable', e=True, ci=[last_row_num, 2], cv=selectedLinkBlend)
    cmds.scriptTable('scrtable', e=True, ci=[last_row_num, 3], cv='0')
    cmds.scriptTable('scrtable', e=True, ci=[last_row_num, 4], cv='1')
    cmds.scriptTable('scrtable', e=True, ci=[last_row_num, 5], cv=selectedBlendI)
    cmds.scriptTable('scrtable', e=True, ci=[last_row_num, 6], cv=selectedLinkBlendI)
'''

def deformface():
    global dataarray
    if len(dataarray) < 0:
        # dataarray의 인자의 첫 세개는 목뼈의 x,y,z좌표 이므로 이것이 없다면 facial_mocap의 의미 없으므로 0을 리턴.
    	return 0

    #all_rows = cmds.scriptTable('scrtable', query=True, rows=True)
    global recstart
    global gnumcurrenttime

    all_rows = 1;
    if all_rows > 0 and recstart == 1:
        ornum = 1;

        numcurrenttime = gnumcurrenttime
        #bonename = cmds.textField('HeadbonenameF', q=True, tx=True )

        # bjoint = pm.PyNode(Jaw_CTRL)

        '''
        # LowerLipMid_CTRL (dataarray[0] ~ dataarray[2])

        global LowerLipMid_X
        global LowerLipMid_Y
        global LowerLipMid_Z

        LowerLipMid_X = 0.089617
        LowerLipMid_Y = 0.177767
        LowerLipMid_Z = 0.075221

        print((float(dataarray[2])*(-1)) + LowerLipMid_Y)

        pm.move(float(dataarray[0])*-1 + LowerLipMid_X, float(dataarray[1])*-1 + LowerLipMid_Y, float(dataarray[2])*-1 + LowerLipMid_Z, 'LowerLipMid_CTRL', relative=True, objectSpace=True, worldSpaceDistance=True )
            #if recnow ==1
             # numcurrenttime 증가시키는 함수는 어딨지?
        pm.setKeyframe('LowerLipMid_CTRL', v=float(dataarray[0])*-1 + LowerLipMid_X, attribute='TranslateX', t=[numcurrenttime])
        pm.setKeyframe('LowerLipMid_CTRL', v=float(dataarray[1])*-1 + LowerLipMid_Y, attribute='TranslateY', t=[numcurrenttime])
        pm.setKeyframe('LowerLipMid_CTRL', v=float(dataarray[2])*-1 + LowerLipMid_Z, attribute='TranslateZ', t=[numcurrenttime])
        '''

        '''
        # UpperLipMid_CTRL (dataarray[3] ~ dataarray[5])

        global UpperLipMid_X
        global UpperLipMid_Y
        global UpperLipMid_Z

        UpperLipMid_X = 0.087832
        UpperLipMid_Y = 0.216568
        UpperLipMid_Z = 0.074157

        pm.move(float(dataarray[3])*-1 + UpperLipMid_X, float(dataarray[4])*-1 + UpperLipMid_Y, float(dataarray[5])*-1 + UpperLipMid_Z, 'UpperLipMid_CTRL', relative=True, objectSpace=True, worldSpaceDistance=True )
        pm.setKeyframe('UpperLipMid_CTRL', v=float(dataarray[3])*-1 + UpperLipMid_X, attribute='TranslateX', t=[numcurrenttime])
        pm.setKeyframe('UpperLipMid_CTRL', v=float(dataarray[4])*-1 + UpperLipMid_Y, attribute='TranslateY', t=[numcurrenttime])
        pm.setKeyframe('UpperLipMid_CTRL', v=float(dataarray[5])*-1 + UpperLipMid_Z, attribute='TranslateZ', t=[numcurrenttime])
        '''

        # L_LowerLipMid_CTRL (dataarray[6] ~ dataarray[8])

        global L_LowerLipMid_X
        global L_LowerLipMid_Y
        global L_LowerLipMid_Z

        L_LowerLipMid_X = 0.097783
        L_LowerLipMid_Y = 0.184848
        L_LowerLipMid_Z = 0.106411

        pm.move(float(dataarray[6])*-1 + L_LowerLipMid_X, float(dataarray[7])*-1 + L_LowerLipMid_Y, float(dataarray[8])*-1 + L_LowerLipMid_Z, 'L_LipsBottom_CTRL', relative=True, objectSpace=True, worldSpaceDistance=True )
        pm.setKeyframe('L_LowerLipMid_CTRL', v=float(dataarray[6])*-1 + L_LowerLipMid_X, attribute='TranslateX', t=[numcurrenttime])
        pm.setKeyframe('L_LowerLipMid_CTRL', v=float(dataarray[7])*-1 + L_LowerLipMid_Y, attribute='TranslateY', t=[numcurrenttime])
        pm.setKeyframe('L_LowerLipMid_CTRL', v=float(dataarray[8])*-1 + L_LowerLipMid_Z, attribute='TranslateZ', t=[numcurrenttime])

        # L_UpperLipMid_CTRL (dataarray[9] ~ dataarray[11])

        global L_UpperLipMid_CTRL_X
        global L_UpperLipMid_CTRL_Y
        global L_UpperLipMid_CTRL_Z

        L_UpperLipMid_CTRL_X = 0.092385
        L_UpperLipMid_CTRL_Y = 0.2153
        L_UpperLipMid_CTRL_Z = 0.095308

        pm.move(float(dataarray[9])*-1 + L_UpperLipMid_CTRL_X, float(dataarray[10])*-1 + L_UpperLipMid_CTRL_Y, float(dataarray[11])*-1 + L_UpperLipMid_CTRL_Z, 'L_LipsTop_CTRL', relative=True, objectSpace=True, worldSpaceDistance=True )
        pm.setKeyframe('L_UpperLipMid_CTRL', v=float(dataarray[9])*-1 + L_UpperLipMid_CTRL_X, attribute='TranslateX', t=[numcurrenttime])
        pm.setKeyframe('L_UpperLipMid_CTRL', v=float(dataarray[10])*-1 + L_UpperLipMid_CTRL_Y, attribute='TranslateY', t=[numcurrenttime])
        pm.setKeyframe('L_UpperLipMid_CTRL', v=float(dataarray[11])*-1 + L_UpperLipMid_CTRL_Z, attribute='TranslateZ', t=[numcurrenttime])

        # R_LowerLipMid_CTRL (dataarray[12] ~ dataarray[14])

        global R_LowerLipMid_CTRL_X
        global R_LowerLipMid_CTRL_Y
        global R_LowerLipMid_CTRL_Z

        R_LowerLipMid_CTRL_X = 0.101421
        R_LowerLipMid_CTRL_Y = 0.185972
        R_LowerLipMid_CTRL_Z = 0.043421

        pm.move(float(dataarray[12])*-1 + R_LowerLipMid_CTRL_X, float(dataarray[13])*-1 + R_LowerLipMid_CTRL_Y, float(dataarray[14])*-1 + R_LowerLipMid_CTRL_Z, 'R_LipsBottom_CTRL', relative=True, objectSpace=True, worldSpaceDistance=True )
        pm.setKeyframe('R_LowerLipMid_CTRL', v=float(dataarray[12])*-1 + R_LowerLipMid_CTRL_X, attribute='TranslateX', t=[numcurrenttime])
        pm.setKeyframe('R_LowerLipMid_CTRL', v=float(dataarray[13])*-1 + R_LowerLipMid_CTRL_Y, attribute='TranslateY', t=[numcurrenttime])
        pm.setKeyframe('R_LowerLipMid_CTRL', v=float(dataarray[14])*-1 + R_LowerLipMid_CTRL_Z, attribute='TranslateZ', t=[numcurrenttime])

        # R_UpperLipMid_CTRL (dataarray[15] ~ dataarray[17])

        global R_UpperLipMid_CTRL_X
        global R_UpperLipMid_CTRL_Y
        global R_UpperLipMid_CTRL_Z

        R_UpperLipMid_CTRL_X = 0.094769
        R_UpperLipMid_CTRL_Y = 0.214611
        R_UpperLipMid_CTRL_Z = 0.053779

        pm.move(float(dataarray[15])*-1 + R_UpperLipMid_CTRL_X, float(dataarray[16])*-1 + R_UpperLipMid_CTRL_Y, float(dataarray[17])*-1 + R_UpperLipMid_CTRL_Z, 'R_LipsTop_CTRL', relative=True, objectSpace=True, worldSpaceDistance=True )
        pm.setKeyframe('R_UpperLipMid_CTRL', v=float(dataarray[15])*-1 + R_UpperLipMid_CTRL_X, attribute='TranslateX', t=[numcurrenttime])
        pm.setKeyframe('R_UpperLipMid_CTRL', v=float(dataarray[16])*-1 + R_UpperLipMid_CTRL_Y, attribute='TranslateY', t=[numcurrenttime])
        pm.setKeyframe('R_UpperLipMid_CTRL', v=float(dataarray[17])*-1 + R_UpperLipMid_CTRL_Z, attribute='TranslateZ', t=[numcurrenttime])

def portData(arg):
    """
    Read the 'serial' data passed in from the commandPort
    """
    global recend
    global dataarray

    dataarray=[]
        # 이차 배열의 첫 배열 dataarray.
    recVal = str(arg)
        # 받은 arg 스트링을 recVal에 넣음.
    strArray = recVal.split(",")
        # recVal 스트링을 , 경계로 쪼갬.
    for i in range(0,18):
        # 24가지의 데이터가 들어옴.
        dataarray.append(strArray[i])
            # dataarray에 strArray[i] 첨부.
        #print(strArray[i])
    #createTimer(0.03, deformface)
    deformface()

def deactivateCommandPort(host, port):
    path = host + ":" + port
    active = cmds.commandPort(path, q=True)
    if active:
        cmds.commandPort(name=path, cl=True)
    else:
        print("%s is was not active" % path)

'''
def createTimer(seconds, function, *args, **kwargs):
    def isItTime():
        now = time.time()
        if now - isItTime.then > seconds:
            isItTime.then = now			# swap the order of these two lines ...
            function(*args, **kwargs)	  # ... to wait before restarting timer

        isItTime.then = 0           #time.time() # set this to zero if you want it to fire once immediately
        cmds.scriptJob(event=("idle", isItTime))
'''


import os
import pandas as pd
from gensim.models import word2vec

from utils import load_training_data
from utils import load_testing_data

# path and filename
path_prefix = "/home/shannon/Downloads/dataset"
train_w_filename = os.path.join(path_prefix, 'training_label.txt')
train_wo_filename = os.path.join(path_prefix, 'training_nolabel.txt')
testing_filename = os.path.join(path_prefix, 'testing_data.txt')

def train_word2vec(x):
    # training word to vector by word embedding
    model = word2vec.Word2Vec(x, size=200, window=5, min_count=5, workers=12, iter=8, sg=1)
    return model

if __name__ == "__main__":
    print("loading training data ...")
    train_x, y = load_training_data(train_w_filename)
    train_x_no_label = load_training_data(train_wo_filename)

    print("loading testing data ...")
    test_x = load_testing_data(testing_filename)

    #embedding_x = train_x + train_x_no_label + test_x #1578614
    embedding_x = train_x + test_x #400000
    print("embedding 2d list ... lenght=",len(embedding_x))
    model = train_word2vec(embedding_x)

    print("saving model ...")
    model.save(os.path.join('model/w2v_200.model'))


from idm.objects import dp, LongpollEvent, MySignalEvent

@dp.longpoll_event_register('+префикс', '-префикс')
def binds_info(event: LongpollEvent):
    event.msg_op(2, 'ℹ Для добавления биндов используй префиксы лп модуля ' +
                 '(по умолчанию ".лп", "!лп")')

@dp.longpoll_event_register('префиксы')
def binds_list(event: LongpollEvent):
    prefixes = event.db.lp_settings['prefixes']
    if not prefixes:
        message = ('Я не знаю как ты этого достиг, но у тебя нет ни одного ' +
                   'LP префикса. На всякий случай добавил префикс "!л", ' +
                   'можешь пользоваться им (возможно понадобится ' +
                   'перезапуск LP модуля)')
        event.db.lp_settings['prefixes'].append('!л')
        event.db.save()
    else:
        message = 'Префиксы LP сигналов:'
        for prefix in prefixes:
            message += f'\n-- "{prefix}"'
    event.msg_op(2, message)

@dp.my_signal_event_register('+префикс', '-префикс')
def prefixes_callback_info(event: MySignalEvent):
    event.msg_op(2, 'ℹ Менять префиксы можно только через LP модуль\n' +
                 'https://github.com/Elchinchel/ICAD-Longpoll')


from django.urls import include, path
from rest_framework import routers

from .views import AnalyticsView, PostViewSet, post_like

router = routers.DefaultRouter(trailing_slash=True)
router.register(r'post', PostViewSet)

urlpatterns = [
    path('', include(router.urls)),
    path('analytics/', AnalyticsView.as_view(), name='analytics'),
    path('post/<post_id>/like/', post_like, name='post-like'),
]


# Loading data so that we have one huge list of player_performances where each player_performance will reference a
# gameweek in which it was player

import os
import pandas as pd
import pymongo

data_dir = '../Fantasy-Premier-League-master/data/'
seasons = ['2016-17', '2017-18', '2018-19', '2019-20']

client = pymongo.MongoClient()
db = client['FPL_Database_v2']
for season in seasons:
    print(season)
    season_collection = db[season]
    all_player_jsons = []
    # In our gameweek .csv that we are going to use we have all of the information that we need except for the player
    # position. We are going to get that info by parsing 'players_raw.csv', and store it in a player to position dict
    player_to_position_dict = {}
    data = pd.read_csv(data_dir + season + '/players_raw.csv', usecols=['element_type', 'first_name', 'second_name'])
    for _, row in data.iterrows():
        player_to_position_dict[row['first_name'] + ' ' + row['second_name']] = row['element_type']
    # Going through each player in 'players' subfolder and and creating a object that represents their performance for
    # each gameweek
    players_dir = data_dir + season + '/players/'
    player_subfolders = os.listdir(players_dir)
    test = 5
    for player in player_subfolders:
        player_data = pd.read_csv(players_dir + player + '/gw.csv',
                                  usecols=['assists',  'goals_conceded', 'goals_scored', 'ict_index', 'creativity',
                                           'threat', 'influence', 'minutes', 'red_cards', 'yellow_cards', 'round',
                                           'selected', 'total_points', 'was_home', 'value', 'transfers_in',
                                           'transfers_out'])
        # Our data source is not uniform for all 4 seasons. Some player folders have id number at the end of their
        # name, and some dont, so we need to do a bit of preprocessing
        name = ''.join(i for i in player if not i.isdigit()).replace('_', ' ').strip()
        try:
            position = player_to_position_dict[name]
        except KeyError:
            print('Name ' + name + ' does not exist in players_raw.csv')
            continue

        for _, row in player_data.iterrows():
            all_player_jsons.append({
                'complex_stats': {
                    'creativity': row['creativity'],
                    'influence': row['influence'],
                    'threat': row['threat'],
                    'ict_index': row['ict_index']
                },
                'transfers': {
                    'in': row['transfers_in'],
                    'out': row['transfers_out']
                },
                'name': name,
                'position': position,
                'value': row['value'],
                'assists': row['assists'],
                'minutes': row['minutes'],
                'selected': row['selected'],
                'goals_scored': row['goals_scored'],
                'goals_conceded': row['goals_conceded'],
                'red_cards': row['red_cards'],
                'yellow_cards': row['yellow_cards'],
                'was_home': row['was_home'],
                'total_points': row['total_points'],
                'gw': row['round']
            })

    season_collection.insert_many(all_player_jsons)
    # We have to delete the list because pymongo uses object_id as document_id in its database, so inserting the
    # same list more than once will cause BulkWriteError
    # del all_player_jsons


import torch
import torch.nn as nn
from torch.autograd import Variable
import numpy as np
import random

from TransformerBlock import TransformerDecoderBlock as DecoderBlock
from TransformerBlock import get_sinusoid_encoding_table
# from bert_embedding import get_bert_embedding
from layers import BERT, LinearProjection, LinearNet_TwoLayer, FFN, PostNet, LinearProjection
from text.symbols import symbols
import hparams as hp
# print(len(symbols))
from module import *

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# device = torch.device("cpu")

if_parallel = True

class WESS_Encoder(nn.Module):
    """
    Encoder
    (pre-transformer replaced by GRU)
    """

    def __init__(self,
                 vocab_max_size=2000,
                 embedding_size=256,
                 GRU_hidden_size=768,
                 GRU_num_layers=1,
                 GRU_batch_first=True,
                 GRU_bidirectional=True,
                 bert_prenet_hidden=1024,
                 bert_prenet_output=256,
                 bert_hidden=256,
                 bert_n_layers=2,
                 bert_attn_heads=4,
                 #  embedding_postnet_hidden=1024,
                 #  embedding_postnet_output=256,
                 dropout=0.1):
        """
        :param encoder_hparams
        """

        super(WESS_Encoder, self).__init__()
        self.vocab_max_size = vocab_max_size
        self.embedding_size = embedding_size
        self.GRU_hidden = GRU_hidden_size
        self.GRU_num_layers = GRU_num_layers
        self.GRU_batch_first = GRU_batch_first
        self.GRU_bidirectional = GRU_bidirectional
        self.bert_prenet_hidden = bert_prenet_hidden
        self.bert_prenet_output = bert_prenet_output
        self.bert_hidden = bert_hidden
        self.bert_n_layers = bert_n_layers
        self.bert_attn_heads = bert_attn_heads
        # self.embedding_postnet_hidden = embedding_postnet_hidden
        # self.embedding_postnet_output = embedding_postnet_output
        self.dropout = dropout

        # Embeddings
        self.pre_embedding = nn.Embedding(len(symbols)+1, self.embedding_size)
        self.position_embedding = nn.Embedding.from_pretrained(
            get_sinusoid_encoding_table(self.vocab_max_size, self.embedding_size), freeze=True)

        # self.pre_GRU = nn.GRU(input_size=self.embedding_size,
        #                       hidden_size=self.GRU_hidden,
        #                       num_layers=self.GRU_num_layers,
        #                       batch_first=self.GRU_batch_first,
        #                       dropout=self.dropout,
        #                       bidirectional=self.GRU_bidirectional)

        self.pre_GRU = nn.GRU(input_size=self.embedding_size,
                              hidden_size=self.GRU_hidden,
                              num_layers=self.GRU_num_layers,
                              batch_first=self.GRU_batch_first,
                              bidirectional=self.GRU_bidirectional)

        self.bert_pre_net = LinearNet_TwoLayer(
            self.GRU_hidden, self.bert_prenet_hidden, self.bert_prenet_output)

        self.bi_Transformer = BERT(hidden=self.bert_hidden,
                                   n_layers=self.bert_n_layers,
                                   attn_heads=self.bert_attn_heads,
                                   dropout=self.dropout)

        # self.EmbeddingNet = LinearNet_TwoLayer(
        #     self.embedding_size, self.embedding_postnet_hidden, self.embedding_postnet_output)

    def init_GRU_hidden(self, batch_size, num_layers, hidden_size):
        if self.GRU_bidirectional:
            return torch.zeros(num_layers*2, batch_size,  hidden_size).to(device)
        else:
            return torch.zeros(num_layers*1, batch_size,  hidden_size).to(device)

    def get_GRU_embedding(self, GRU_output):
        # print(GRU_output.size())
        out_1 = GRU_output[:, 0:1, :]
        out_2 = GRU_output[:, GRU_output.size(1)-1:, :]

        out = out_1 + out_2
        out = out[:, :, 0:out.size(2)//2] + out[:, :, out.size(2)//2:]

        # print(out.size())
        return out

    def cal_P_GRU(self, batch, gate_for_words_batch):
        list_input = list()
        list_output = list()

        for ind in range(len(gate_for_words_batch)-1):
            list_input.append(
                batch[gate_for_words_batch[ind]:gate_for_words_batch[ind+1]])

        # print(len(list_input))
        for one_word in list_input:
            one_word = torch.stack([one_word])

            # pos_input = torch.Tensor(
            #     [i for i in range(one_word.size(1))]).long().to(device)
            # position_embedding = self.position_embedding(pos_input)
            # position_embedding = position_embedding.unsqueeze(0)

            # one_word = one_word + position_embedding
            # output_one_word = self.P_transformer_block(one_word)
            # print(output_one_word.size())

            # self.pre_GRU.flatten_parameters()
            output_one_word = self.pre_GRU(one_word)[0]
            output_one_word = self.get_GRU_embedding(output_one_word)
            word = output_one_word.squeeze(0)
            # word = output_one_word[output_one_word.size()[0]-1]
            list_output.append(word)

        output = torch.stack(list_output)
        output = output.squeeze(1)
        # print(output.size())

        return output

    # def pad_by_word(self, words_batch):
    #     len_arr = np.array(list())
    #     for ele in words_batch:
    #         len_arr = np.append(len_arr, ele.size(0))
    #     max_size = int(len_arr.max())
    #     # print(max_size)

    #     def pad(tensor, target_length):
    #         embedding_size = tensor.size(1)
    #         pad_tensor = torch.zeros(1, embedding_size).to(device)

    #         for i in range(target_length-tensor.size(0)):
    #             tensor = torch.cat((tensor, pad_tensor))

    #         return tensor

    #     padded = list()
    #     for one_batch in words_batch:
    #         one_batch = pad(one_batch, max_size)
    #         padded.append(one_batch)
    #     padded = torch.stack(padded)

    #     return padded

    # def pad_all(self, word_batch, embeddings):
    #     # print(word_batch.size())
    #     # print(embeddings.size())
    #     if word_batch.size(1) == embeddings.size(1):
    #         return word_batch, embeddings

    #     if word_batch.size(1) > embeddings.size(1):
    #         pad_len = word_batch.size(1) - embeddings.size(1)
    #         pad_vec = torch.zeros(word_batch.size(
    #             0), pad_len, embeddings.size(2)).float().to(device)
    #         embeddings = torch.cat((embeddings, pad_vec), 1)
    #         return word_batch, embeddings

    #     if word_batch.size(1) < embeddings.size(1):
    #         pad_len = embeddings.size(1) - word_batch.size(1)
    #         pad_vec = torch.zeros(word_batch.size(
    #             0), pad_len, embeddings.size(2)).float().to(device)
    #         word_batch = torch.cat((word_batch, pad_vec), 1)
    #         return word_batch, embeddings

    def pad_bert_embedding_and_GRU_embedding(self, bert_embedding, GRU_embedding):
        """One batch"""

        len_bert = len(bert_embedding)
        len_GRU = GRU_embedding.size(0)
        max_len = max(len_bert, len_GRU)

        # pad_embedding = torch.zeros(1, GRU_embedding.size(1)).to(device)

        # for i in range(max_len - len_GRU):
        #     GRU_embedding = torch.cat((GRU_embedding, pad_embedding), 0)

        # for i in range(max_len - len_bert):
        #     bert_embedding = torch.cat((bert_embedding, pad_embedding), 0)

        # print(GRU_embedding.size())
        # print(bert_embedding.size())

        GRU_embedding = torch.cat((GRU_embedding, torch.zeros(
            max_len - len_GRU, GRU_embedding.size(1)).to(device)), 0)
        bert_embedding = torch.cat((bert_embedding, torch.zeros(
            max_len - len_bert, GRU_embedding.size(1)).to(device)), 0)

        # output = bert_embedding + GRU_embedding

        return bert_embedding, GRU_embedding

    def pad_all(self, bert_transformer_input):
        len_list = list()
        for batch in bert_transformer_input:
            len_list.append(batch.size(0))

        max_len = max(len_list)

        # pad_embedding = torch.zeros(
        #     1, bert_transformer_input[0].size(1)).to(device)

        for index, batch in enumerate(bert_transformer_input):
            bert_transformer_input[index] = torch.cat((bert_transformer_input[index], torch.zeros(
                max_len - bert_transformer_input[index].size(0), bert_transformer_input[index].size(1)).to(device)), 0)

        bert_transformer_input = torch.stack(bert_transformer_input)

        return bert_transformer_input

    def forward(self, x, bert_embeddings, gate_for_words, indexs_list):
        """
        :param: x: (batch, length)
        :param: bert_embeddings: (batch, length, 768)
        :param: gate_for_words: (batch, indexs)
        """

        # Embedding
        x = self.pre_embedding(x)
        # print("x:", x.size())

        # P_GRU
        words_batch = list()
        for index, batch in enumerate(x):
            words_batch.append(self.cal_P_GRU(batch, gate_for_words[index]))

        # words_batch = self.pad_by_word(words_batch)
        # bert_embeddings = self.pad_by_word(bert_embeddings)
        # words_batch, bert_embeddings = self.pad_all(
        #     words_batch, bert_embeddings)
        # # print(words_batch.size())
        # # print(bert_embeddings.size())
        # bert_input = words_batch + bert_embeddings

        # # Add Position Embedding
        # pos_input = torch.stack([torch.Tensor([i for i in range(
        #     bert_input.size(1))]).long() for i in range(bert_input.size(0))]).to(device)
        # pos_embedding = self.position_embedding(pos_input)
        # bert_input = bert_input + pos_embedding

        # encoder_output = self.bert_encoder(bert_input)

        # New

        # If you need data parallel, you will need some extra processing.
        if self.training and if_parallel:
            # print(indexs_list)
            bert_embeddings = [bert_embeddings[i] for i in indexs_list]

        if not (len(words_batch) == len(bert_embeddings)):
            raise ValueError(
                "the length of bert embeddings is not equal to the length of GRU embeddings.")

        bert_transformer_input = list()

        for batch_index in range(len(words_batch)):

            # print("bert_embedding:", bert_embeddings[batch_index])
            # print(words_batch[batch_index])

            bert_embedding, GRU_embedding = self.pad_bert_embedding_and_GRU_embedding(
                bert_embeddings[batch_index], words_batch[batch_index])
            add_bert_GRU = bert_embedding + GRU_embedding
            bert_transformer_input.append(add_bert_GRU)

        bert_transformer_input = self.pad_all(bert_transformer_input)

        pos_input_one_batch = torch.Tensor(
            [i for i in range(bert_transformer_input.size(1))]).long()
        pos_input = torch.stack([pos_input_one_batch for _ in range(
            bert_transformer_input.size(0))]).to(device)
        position_embedding = self.position_embedding(pos_input)

        #  print("position_embedding:", position_embedding.size())
        # print("position embedding:", position_embedding)

        # bert_transformer_input = bert_transformer_input + position_embedding

        # print("bert_transformer_input:", bert_transformer_input.size())
        bert_transformer_input = self.bert_pre_net(bert_transformer_input)
        # print("bert_transformer_input:", bert_transformer_input.size())

        bert_transformer_input = bert_transformer_input + position_embedding

        encoder_output_word = self.bi_Transformer(bert_transformer_input)

        # encoder_output_alpha = self.EmbeddingNet(x)
        pos_input_alpha = torch.stack([torch.Tensor([i for i in range(x.size(1))]).to(
            device) for _ in range(x.size(0))]).long()
        position_embedding_alpha = self.position_embedding(pos_input_alpha)
        encoder_output_alpha = x + position_embedding_alpha

        return encoder_output_word, encoder_output_alpha

class TacotronDecoder(nn.Module):
    """
    Tacotron Decoder
    """

    def __init__(self, hidden_size=128):
        super(TacotronDecoder, self).__init__()
        self.prenet = Prenet(hp.num_mels, hidden_size * 2, hidden_size)
        self.attn_decoder = AttentionDecoder(hidden_size * 2)
        self.postnet = PostNet()

    def forward(self, decoder_input, memory):
        # Initialize hidden state of GRUcells
        attn_hidden, gru1_hidden, gru2_hidden = self.attn_decoder.inithidden(
            decoder_input.size()[0])
        outputs = list()

        # Training phase
        if self.training:
            # Prenet
            dec_input = self.prenet.forward(decoder_input)
            timesteps = dec_input.size()[2] // hp.outputs_per_step

            # [GO] Frame
            # prev_output = dec_input[:, :, 0]
            # print(prev_output.size())
            GO_frame = torch.zeros(dec_input.size(
                0), dec_input.size(1)).to(device)
            # print(GO_frame.size())

            prev_output, attn_hidden, gru1_hidden, gru2_hidden = self.attn_decoder.forward(
                GO_frame, memory, attn_hidden=attn_hidden, gru1_hidden=gru1_hidden, gru2_hidden=gru2_hidden)
            outputs.append(prev_output)

            for i in range(timesteps):
                # Teacher Forced
                if random.random() < hp.teacher_forcing_ratio:
                    # Get spectrum at rth position
                    prev_output = dec_input[:, :, (i+1)*hp.outputs_per_step-1]
                else:
                    # Get last output
                    prev_output = prev_output[:, :, -1]
                    # prev_output = output[-1]

                prev_output, attn_hidden, gru1_hidden, gru2_hidden = self.attn_decoder.forward(
                    prev_output, memory, attn_hidden=attn_hidden, gru1_hidden=gru1_hidden, gru2_hidden=gru2_hidden)

                outputs.append(prev_output)

                # if random.random() < hp.teacher_forcing_ratio:
                #     # Get spectrum at rth position
                #     prev_output = dec_input[:, :, i * hp.outputs_per_step]
                # else:
                #     # Get last output
                #     prev_output = prev_output[:, :, -1]

            # Concatenate all mel spectrogram

            outputs = torch.cat(outputs, 2)

            # print(outputs.size())
            # print(decoder_input.size())

            outputs = outputs[:, :, 0:decoder_input.size(2)]

            # outputs_postnet = self.postnet(outputs) + outputs

        # Test
        else:
            # [GO] Frame
            prev_output = decoder_input

            for i in range(hp.max_iters):
                prev_output = self.prenet.forward(prev_output)
                prev_output = prev_output[:, :, 0]
                prev_output, attn_hidden, gru1_hidden, gru2_hidden = self.attn_decoder.forward(
                    prev_output, memory, attn_hidden=attn_hidden, gru1_hidden=gru1_hidden, gru2_hidden=gru2_hidden)
                outputs.append(prev_output)
                prev_output = prev_output[:, :, -1].unsqueeze(2)

            outputs = torch.cat(outputs, 2)

            # outputs_postnet = self.postnet(outputs) + outputs

        # return outputs, outputs_postnet
        return outputs

# class PostProcessingNet(nn.Module):
#     """
#     Post-processing Network
#     """

#     def __init__(self, hidden_size=125, num_freq=1025):
#         super(PostProcessingNet, self).__init__()
#         self.postcbhg = CBHG(
#             hidden_size, K=8, projection_size=hp.num_mels, is_post=True)
#         self.linear = SeqLinear(hidden_size * 2, num_freq)

#     def forward(self, decoder_output):
#         out = self.postcbhg.forward(decoder_output)
#         out = self.linear.forward(torch.transpose(out, 1, 2))

#         return out

class WETaSS(nn.Module):
    """
    WESS combine with Tacotron
    """

    def __init__(self):
        super(WETaSS, self).__init__()
        self.encoder = WESS_Encoder()
        self.decoder_word = TacotronDecoder()
        self.decoder_alpha = TacotronDecoder()
        # self.postnet = PostProcessingNet()
        self.postnet = PostNet()

    def forward(self, x, bert_embeddings, gate_for_words, mel_target=None, indexs_list=None):
        if self.training:
            encoder_output_word, encoder_output_alpha = self.encoder(
                x, bert_embeddings, gate_for_words, indexs_list)
            decoder_output_word = self.decoder_word(
                mel_target, encoder_output_word)
            decoder_output_alpha = self.decoder_alpha(
                mel_target, encoder_output_alpha)

            mel_output = decoder_output_word + decoder_output_alpha
            mel_output = mel_output.contiguous().transpose(1, 2)

            mel_output_postnet = self.postnet(mel_output) + mel_output

            mel_target = mel_target.contiguous().transpose(1, 2)

            return mel_output, mel_output_postnet, mel_target
        else:
            decoder_input = torch.zeros(1, hp.num_mels, 1).to(device)

            encoder_output_word, encoder_output_alpha = self.encoder(
                x, bert_embeddings, gate_for_words, indexs_list)
            decoder_output_word = self.decoder_word(
                decoder_input, encoder_output_word)
            decoder_output_alpha = self.decoder_alpha(
                decoder_input, encoder_output_alpha)

            mel_output = decoder_output_word + decoder_output_alpha
            mel_output = mel_output.contiguous().transpose(1, 2)

            mel_output_postnet = self.postnet(mel_output) + mel_output

            return mel_output, mel_output_postnet

# if __name__ == "__main__":
#     # Test
#     test_decoder = TacotronDecoder().to(device)
#     output = test_decoder(torch.randn(2, 80, 121).to(
#         device), torch.randn(2, 56, 256).to(device))
#     print(output.size())

#     # test_postnet = PostProcessingNet().to(device)
#     # out = test_postnet(output)
#     # print(out.size())

#     test_WETaSS = WETaSS().to(device)
#     print(test_WETaSS)


from __future__ import print_function
from neo4j.v1 import GraphDatabase, basic_auth
import json
import base64

print('Loading function')

def lambda_handler(event, context):
    output = []

    for record in event['records']:
        print(record['recordId'])
        payload = base64.b64decode(record['data'])

        payload = transform_data(payload)
        encode_str_base = json.dumps(payload)
        encoded_bytes = base64.b64encode(encode_str_base.encode("utf-8"))
        encoded_str = str(encoded_bytes, "utf-8")

        output_record = {
            'recordId': record['recordId'],
            'result': 'Ok',
            'data': encoded_str
        }
        output.append(output_record)
        print(output)

    print('Successfully processed {} records.'.format(len(event['records'])))

    return {'records': output}

def transform_data(data):
    node = mapper(data)
    reduce_outcome = reducer(node)
    return reduce_outcome

def mapper(data):
    data = json.loads(data)
    print(data)
    nodeStruct={
        'id': data['_id'],
        'projectCode': data['projectCode']
    }
    print(nodeStruct)
    return nodeStruct

def reducer(node):
    print(node["id"])
    neo4j_url = 'bolt://db-xvwxuy811zirtlf14lmz.graphenedb.com:24787'
    driver = GraphDatabase.driver(neo4j_url, auth=basic_auth('dbopsuser', 'b.G6oUmNKKs9MD.3Gxn1WfHSoUw0OGa'))
    session = driver.session()
    session.run("MERGE(n:Project {id: $idProject, projectCode: $projectCode}) RETURN n",
                idProject=node["id"], projectCode=node["projectCode"])
    session.close()
    print(node)
    return node


import time
from multiprocessing.context import Process

from Pyro5.api import Daemon as PyroDaemon
from panoptes.utils.library import load_module
from panoptes.utils.config.client import get_config, server_is_running
from huntsman.pocs.utils.logger import logger
from huntsman.pocs.utils.pyro.nameserver import get_running_nameserver
from huntsman.pocs.utils.config import get_own_ip

from huntsman.pocs.utils.pyro import serializers  # Required to set up the custom (de)serializers

def pyro_service_process(auto_start=False, *args, **kwargs):
    """Start a pyro service in a separate process."""
    # Set up nameserver process.
    logger.debug(f'Setting up Pyro service process.')
    service_process = Process(target=pyro_service, args=args, kwargs=kwargs)

    if auto_start:  # noqa
        logger.info("Auto-starting pyro service")
        service_process.start()
        logger.success("Pyro service started, will block until finished...(Ctrl-c/Cmd-c to exit)")
        service_process.join()

    return service_process

def pyro_service(service_class=None,
                 service_name=None,
                 host=None,
                 port=None,
                 metadata=None, **kwargs):
    """Creates and runs a Pyro Service.

    This is the "service" portion of the Pyro communication, which should be
    started on a remote device (not on the central control computer).

    Parameters
    ----------
    service_class (str): The class that has been exposed by Pyro.
    service_name (str): A name for the service that will be registered with the nameserver. Will
        attempt to use service_name to query the config server for the device config.
    host (str): The host or ip of the device running the service.
    port (str or int): The port to attach the device to, default 0/None for auto-select.
    metadata (list of str): Metadata to be stored on the NS. If `None`, will create metadata
        based on service class using the config server.

    Returns:
        multiprocess.Process: The process responsible for running the service. Note that if the
            service was started via `autostart=True`, the function will block until terminated,
            but still return the completed process.
    """
    while not server_is_running():
        logger.debug("Waiting for config server to start up before starting pyro service.")
        time.sleep(5)

    # Specify address
    host = host or get_config(f'pyro.{service_name}.host', default=None)
    if host is None:
        host = get_own_ip(verbose=True)
    # If port is not in config set to 0 so that Pyro will choose a random one.
    port = int(port or get_config(f'pyro.{service_name}.port', default=0))
    # Specify metadata to be stored on NS
    if metadata is None:
        service_class_base = service_class.split(".")[-1]
        metadata = get_config(f"pyro.{service_class_base}.metadata", None)
    if metadata is not None:
        metadata = set(metadata)

    # Get the class instance to expose
    service_name = service_name or get_config('name', 'Generic Pyro Server')
    service_instance = load_module(service_class)(device_name=service_name)

    # Locate the NS
    # NOTE: Moving this block outside start_service can lead to broken pipe errors
    try:
        nameserver = get_running_nameserver()
    except Exception as e:
        logger.warning(f"Pyro nameserver not running, can't create server. "
                       f"See 'huntsman-pyro nameserver' for details. {e!r}")
        return

    logger.info(f'Starting service_name={service_name} on host={host}:{port}')
    with PyroDaemon(host=host, port=port) as daemon:
        logger.info(f'Creating pyro daemon service for service_class={service_class}')
        uri = daemon.register(service_instance)
        logger.info(f'Registered {service_class} pyro daemon with uri={uri}'
                    f' and metadata={metadata}.')

        # Register service with the nameserver.
        nameserver.register(service_name, uri, safe=True, metadata=metadata)
        logger.info(f'Registered {service_class} with nameserver as {service_name}')

        try:
            logger.info(f'Pyro service {service_name} started. Ctrl-C/Cmd-C to quit...')
            daemon.requestLoop()
        except KeyboardInterrupt:
            logger.info(f'Pyro service {service_name} requested shutdown by user.')
        except Exception as e:  # noqa
            logger.info(f'{service_name} died unexpectedly: {e!r}')
        finally:
            logger.info(f'Shutting down {service_name} pyro server...')
            nameserver.remove(name=service_name)
            logger.info(f'Unregistered {service_name} from pyro nameserver')

    return


#使用dict2 = dict1.copy()对一个字典重新赋值
dict1 = {'a':'apple','b':'banana','c':["c",'cd']}
dict2 = dict1.copy()       #这种方法在dict1父对象是处于'深拷贝',但是在数组部分(子对象)处于浅拷贝
dict3 = dict1              #这种方法在dict1改变的时候也会同时改变dict3:浅拷贝

del(dict1['b'])            #改变dict1的同时改变dict3
dict1['c'].remove('cd')    #改变dict1['c']的元素时同时改变dict2和dict3

print('结果如下:')
print('dict1结果是:',dict1)
print('dict2结果是:',dict2)
print('dict3结果是:',dict3)
del(dict1)
del(dict2)
del(dict3)
#深拷贝操作:import copy
import copy
dict1 = {'a': 'apple','b':{'g':'grape','b':'banana'}}
print('原来的dict1:',dict1)
dict2 = copy.deepcopy(dict1)    #深拷贝:copy.deepcopy()
dict3 = copy.copy(dict1)        #与dict1.copy()相同
del(dict1['b']['g'])
#dict1['b'].setdefault('o','orange')
dict1['b']['o'] = 'orange'
dict1['a'] = dict1['a'] + 'Watch'
print ('现在的dict1:',dict1)
print ('dict2(深拷贝)与dict3(浅拷贝)对比如下:')
print ('dict2:',dict2)
print ('dict3:',dict3)

基本知识了解一下/字典的拷贝操作.py


import pandas as pd
import csv

# 进行数据的预处理和特征归一化

pd.set_option('precision', 5) #设置精度
pd.set_option('display.float_format', lambda x: '%.5f' % x) #为了直观的显示数字，不采用科学计数法

label_csv = pd.read_csv('./flows_label.csv')
feature_csv = pd.read_csv('./flows_feature.csv')

featureName=['PacketTotalNum','DurationTime','PacketLenMean', 'PayloadLenMean', 'TimeGapMean']

flows_feature = []
flows_label = []

# print(type(feature_csv))
print(feature_csv['PacketTotalNum'].describe())
true_label = label_csv.iloc[0:1000000] # 只使用所有数据的前1000000行
true_feature = feature_csv.iloc[0:1000000] # 只使用所有数据的前1000000行

features_csv=true_feature[['PacketTotalNum','DurationTime','PacketLenMean', 'PayloadLenMean', 'TimeGapMean']]
features_csv = (features_csv - features_csv.min()) / (features_csv.max() - features_csv.min())

true_label.to_csv("last_label.csv")
features_csv.to_csv("last_feature.csv")


def maxDistance(arr, n):
    # Code here
    # using hashing O(n)
    freq = {}
    for i in range(len(arr)):
        if arr[i] not in freq:
            freq[arr[i]] = [i]
        else:
            freq[arr[i]].append(i)
    max_dif = 0
    for key in freq:
        dif = freq[key][-1]-freq[key][0]
        if dif > max_dif:
            max_dif = dif
    return max_dif

Maximum distance between two occurrences of same element in array.py