# -*- coding:utf-8 -*-
"""
   File Name：     visual.py
   Description :   可视化
   Author :        royce.mao
   date：          2020/3/2 16:20
"""
import cv2
import json
import numpy as np
from PIL import Image, ImageDraw, ImageFont

# CTPN定位之后的数字串小图
class DigitPic(object):
    def __init__(self, ):
        super(DigitPic, self).__init__()

    @staticmethod
    def padding(img, pad_size):
        """
        顶部加padding（黑色背景，便于put_txt）
        :param img: numpy
        :param pad_size: size标量
        :return:
        """
        # 增加padding
        padding_size = int(max(0, pad_size))
        img_padding = np.pad(img, ((padding_size, 0), (0, 0)), mode='constant', constant_values=0)

        return img_padding

    def to_txt_image(self, img, pred):
        """
        put_txt英文
        :param img: 加了padding的img
        :param pred: 预测的字符串
        :return:
        """
        left_up = (0, 0)
        text_size, baseline = cv2.getTextSize(
            pred, cv2.FONT_HERSHEY_SIMPLEX, 0.3, 1)

        p1 = (left_up[0], left_up[1] + text_size[1])
        img_padding = self.padding(img, p1[1] + baseline)

        cv2.putText(img_padding, 'pred:' + pred, (p1[0], p1[
            1] + baseline), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (255, 255, 255), 1, 8)

        return img_padding

# CTPN定位数字串之前的原图
class OriginPic(object):
    def __init__(self, ):
        super(OriginPic, self).__init__()

    @staticmethod
    def json_parse(annotation_condidate):
        """
        labeme标注的json解析
        :param annotation_condidate:
        :return:
        """
        with open(annotation_condidate, 'r') as fp:
            data = json.load(fp)  # 加载json文件
            boxes = []
            for shape in data['shapes']:
                x1, y1 = shape['points'][0]  # 第一个点是左上角
                x2, y2 = shape['points'][1]  # 第二个点是右下角
                boxes.append([min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)])
            bbox = [int(np.min(np.array(boxes)[:, 0])),
                    int(np.min(np.array(boxes)[:, 1])),
                    int(np.max(np.array(boxes)[:, 2])),
                    int(np.max(np.array(boxes)[:, 3])),
                    ]

        return bbox

    @staticmethod
    def add_cn_text(img, text, left_top, color=(0, 0, 255), size=50):
        """
        中文支持
        :param img:
        :param text:
        :param left_top:
        :param color:
        :param size:
        :return:
        """
        if isinstance(img, np.ndarray):
            img = Image.fromarray(img)
        draw = ImageDraw.Draw(img)
        font = ImageFont.truetype('./simhei.ttf', size, encoding="utf-8")
        draw.text(left_top, text, color, font=font)

        return np.asarray(img)

    def to_bbox_txt_image(self, img, json_path, pred, score):
        """
        put_bbox and put_txt中文
        :param img:
        :param json_path:
        :param pred:
        :param score:
        :return:
        """
        pt = self.json_parse(json_path)

        # 画框
        left_up, right_bottom = (pt[0], pt[1]), (pt[2], pt[3])
        cv2.rectangle(img, left_up, right_bottom, (0, 0, 255), 5)

        # 添加中文文字
        conf = "{:.2f}".format(score)

        text_size, baseline = cv2.getTextSize(
            'Pred:' + pred + '丨' + 'Score:' + conf, cv2.FONT_HERSHEY_SIMPLEX, 5, 1)

        p1 = (left_up[0], left_up[1] - text_size[1])

        return self.add_cn_text(img, '预测:' + pred + ' ' + '得分:' + conf, (p1[0], p1[1]), color=(0, 0, 255), size=100)


import socket, threading
tcp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp_socket.connect(('192.168.67.94',6666))
print('connected with server')

def send():
    while True:
        send = input('pls input something: ')
        tcp_socket.send(send.encode('gbk'))

def reve():
    while True:
        text = tcp_socket.recv(1024)
        print('rece:',text.decode('gbk'))

if __name__ == '__main__':
    sub_send = threading.Thread(target=send)
    sub_reve = threading.Thread(target=reve)
    sub_send.start()
    sub_reve.start()


# coding: utf-8

# ## For Colab

# In[ ]:

from IPython.display import HTML, display

def set_css():
  display(HTML('''
  <style>
    pre {
        white-space: pre-wrap;
    }
  </style>
  '''))
get_ipython().events.register('pre_run_cell', set_css)

# In[ ]:

from google.colab import drive
drive.mount('/content/drive')

# ## Module Imports

# In[ ]:

import torch
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim.lr_scheduler import ReduceLROnPlateau
import numpy as np
import random
from tqdm import tqdm
import os
import nltk
nltk.download('punkt')
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from matplotlib import pyplot as plt
from torch.nn.utils.rnn import pad_sequence
import glob
import itertools
import pickle
import re
import pandas as pd
from tqdm import tqdm
from sklearn.model_selection import train_test_split
get_ipython().system('pip install rouge-score')

# ## Constants
# - remember to point to the right dataset

# In[ ]:

drive_prefix = '/content/drive/MyDrive/ese546/'
PATH_TRAIN_X = drive_prefix + 'cnndm/train.txt.src'
PATH_TRAIN_Y = drive_prefix + 'cnndm/train.txt.tgt.tagged'

# PATH_TRAIN_X = drive_prefix + 'cnndm/small.src'
# PATH_TRAIN_Y = drive_prefix + 'cnndm/small.tagged'

# PATH_VAL_X = drive_prefix + 'cnndm/val.txt.src'
# PATH_VAL_Y = drive_prefix + 'cnndm/val.txt.tgt.tagged'

# CHANGE THIS AFTER
# PATH_TRAIN_X = drive_prefix + 'cnndm/test.txt.src'
# PATH_TRAIN_Y = drive_prefix + 'cnndm/test.txt.tgt.tagged'

STOP_WORDS = set()
EMB_SIZE = 300
GLOVE_EMB = drive_prefix + f'glove.6B.{EMB_SIZE}d.txt'
BATCH_SIZE = 32
NUM_EVAL_BATCHES = 4
NUM_HEADS = 10
DROPOUT_RATE = 0.2

START_CHAR = 'starttoken'
END_CHAR = 'endtoken'
PAD_CHAR = 'padtoken'

MAX_ARTICLE_LEN = 300 # the article can have at most 300 tokens
MAX_LABEL_LEN = 40 # labels can have at most 100 tokens

# NN Hyper-parameters
E_HIDDEN_DIM = 512
D_HIDDEN_DIM = 512
KEY_DIM = 64
VALUE_DIM = 64
LR = 1e-3

EP = 100000
PRINT_EVERY_EP = 200
SAVE_MODEL_EVERY_EP = 10000
FORCE_CREATE_DICT = True # force to recreate the word features from scratch, but doesn't recreate the embedding dict, since it doesn't change

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)
torch.cuda.empty_cache()

# ## Helper Functions

# In[ ]:

def create_word_vec(input_text, prefix='train'):
    word_idx_dict = {} # { word : the index of that word in the dictionary}
    idx_word_dict = {} # { index of word : word }

    # 1. Create mapping between words and the corresponding embedding values
    embed_file_path = drive_prefix + f'{EMB_SIZE}d_embed_dict'
    if os.path.exists(embed_file_path):
        print('Embedding dictionary exists, loading from file...')
        embedding_dict = pickle.load(open(embed_file_path, 'rb'))
    else:
        embedding_dict = {}

        for line in glove:
            tokens = line.split()
            embedding_dict[tokens[0]] = np.array(tokens[1:], dtype='float32')
        pickle.dump(embedding_dict, open(embed_file_path, 'wb'))
        print('Saved embedding dictionary')

    # 2. Tokenize the input_text and labels
    if os.path.exists(drive_prefix + f'{prefix}_word_idx_dict') and not FORCE_CREATE_DICT:
        print('Word-to-index dictionary exists, loading from file...')
        word_idx_dict = pickle.load(open(drive_prefix + f'{prefix}_word_idx_dict', 'rb'))
    if os.path.exists(drive_prefix + f'{prefix}_idx_word_dict') and not FORCE_CREATE_DICT:
        print('Index-to-word dictionary exists, loading from file...')
        idx_word_dict = pickle.load(open(drive_prefix + f'{prefix}_idx_word_dict', 'rb'))
    else:
        unique_tokens = set([])
        for text, label in input_text:
            unique_tokens = unique_tokens.union(word_tokenize(text))
            unique_tokens = unique_tokens.union(word_tokenize(label))

        for token in unique_tokens:
            word_idx_dict[token] = len(word_idx_dict)

        # 2.1 Add in the special tokens to the dictionary, note that the START_CHAR and END_CHAR have been added
        # during the preprocessing stage
        word_idx_dict[PAD_CHAR] = len(word_idx_dict)

        idx_word_dict = dict(zip(word_idx_dict.values(), word_idx_dict.keys()))

    # 3. Build the word vector for all the words in our dictionary
    if os.path.exists(drive_prefix + f'{prefix}_word_vector') and not FORCE_CREATE_DICT:
        print('Word Vector exists, loading from file...')
        word_vector = pickle.load(open(drive_prefix + f'{prefix}_word_vector', 'rb'))
    else:
        word_vector = []
        for idx, token in idx_word_dict.items():
            if token in embedding_dict:
                word_vector.append(embedding_dict[token])
            # Append the special tokens to the word vector and assign random values
            elif token in [START_CHAR, END_CHAR, PAD_CHAR]:
                word_vector.append(np.random.normal(0, 1, EMB_SIZE))
            # if the token doesn't have an embedding, we set to 0
            else:
                word_vector.append(np.zeros([EMB_SIZE]))

    ## Save the dictionaries
    pickle.dump(word_idx_dict, open(drive_prefix + f'{prefix}_word_idx_dict', 'wb'))
    pickle.dump(idx_word_dict, open(drive_prefix + f'{prefix}_idx_word_dict', 'wb'))
    pickle.dump(word_vector, open(drive_prefix + f'{prefix}_word_vector', 'wb'))

    # The index in worvalue_dimec corresponds to the article index in the original X_Train array
    return np.array(word_vector), word_idx_dict, idx_word_dict

def sentence_to_idx(sentence, word_to_idx):
    text, label = sentence
    text_tokens = [word_to_idx[token] for token in word_tokenize(text)]
    label_tokens = [word_to_idx[token] for token in word_tokenize(label)]
    return np.array(text_tokens).astype(int), np.array(label_tokens).astype(int)

def decontracted(text):
    '''Clean text by removing unnecessary characters and altering the format of words.'''
    text = text.lower()
    text = re.sub(r"i'm", "i am", text)
    text = re.sub(r"he's", "he is", text)
    text = re.sub(r"she's", "she is", text)
    text = re.sub(r"it's", "it is", text)
    text = re.sub(r"that's", "that is", text)
    text = re.sub(r"what's", "that is", text)
    text = re.sub(r"where's", "where is", text)
    text = re.sub(r"how's", "how is", text)
    text = re.sub(r"\'ll", " will", text)
    text = re.sub(r"\'ve", " have", text)
    text = re.sub(r"\'re", " are", text)
    text = re.sub(r"\'d", " would", text)
    text = re.sub(r"\'re", " are", text)
    text = re.sub(r"won't", "will not", text)
    text = re.sub(r"can't", "cannot", text)
    text = re.sub(r"n't", " not", text)
    text = re.sub(r"n'", "ng", text)
    text = re.sub(r"'bout", "about", text)
    text = re.sub(r"'til", "until", text)
    text = re.sub(r"[-()\"#/@;:<>{}`+=~|.!?,]", "", text)
    return text

def _process(text):
    text = decontracted(text)
    text = text.replace('\\r', ' ')
    text = text.replace('\\"', ' ')
    text = text.replace('\\n', ' ')
    text = re.sub(r"\b[a-zA-Z]\b", " ", text)
    text = re.sub("^\d+\s|\s\d+\s|\s\d+$", " ", text)
    text = re.sub('[^A-Za-z0-9]+', ' ', text)
    text = re.sub('<[^>]*>', '', text)
    return text

def pre_process(articles, labels):
    preprocessed_text = []
    # tqdm is for printing the status bar
    for i in tqdm(range(len(articles))):
        sent = articles[i]
        label = labels[i]

        sent = _process(sent)
        label = _process(label)

        sent = sent.lower().strip().split()
        label = label.lower().strip().split()

        # trim longer items and ignore shorter ones
        if len(sent) < MAX_ARTICLE_LEN or len(label) < MAX_LABEL_LEN:
            continue

        sent = ' '.join(sent[:MAX_ARTICLE_LEN])
        label = ' '.join(label[:MAX_LABEL_LEN])
        label = f"{START_CHAR} {label} {END_CHAR}"
        preprocessed_text.append([sent, label])
    return preprocessed_text

def get_batch(data):
    while True:
        for i in range(int((data.shape[0] - BATCH_SIZE) / BATCH_SIZE)):
            yield data[i * BATCH_SIZE: i * BATCH_SIZE + BATCH_SIZE]

def patch_trg(trg):
    # Need to understand
    trg, gold = trg[:, :-1], trg[:, 1:].contiguous().view(-1)
    return trg, gold

def plot(train_loss, val_loss):
    plt.plot(train_loss, label='Train')
    plt.plot(val_loss, label='Val')
    plt.title('Training vs Validation Loss')
    plt.xlabel('Episodes')
    plt.ylabel('Loss')
    plt.legend()
    plt.show()

# ## Read Data

# In[ ]:

with open(PATH_TRAIN_X, 'r', encoding='utf-8') as f:
    X = f.readlines()[:100000]

with open(PATH_TRAIN_Y, 'r', encoding='utf-8') as f:
    Y = f.readlines()[:100000]

with open(GLOVE_EMB, 'r', encoding='utf-8') as f:
    glove = f.readlines()

X_Train, X_Val, Y_Train, Y_Val = train_test_split(X, Y, test_size=0.3, shuffle=True)

# ## Features

# In[ ]:

def get_features(processed_data, prefix):
    # note that the prefix variable is only used for naming dictionaries that are
    # saved to the disk
    print(f'\nTotal # of stories: {len(processed_data)}')
    word_vector, word_idx_dict, idx_word_dict = create_word_vec(processed_data, prefix=prefix)
    print(f'Word Vector Shape: {word_vector.shape}')
    assert word_vector.shape == (len(idx_word_dict.keys()), EMB_SIZE)

    return word_vector, word_idx_dict, idx_word_dict

# USING THE CELL BELOW TO SPEED THINGS UP
# # Full
full_processed_data = pre_process(X, Y)
full_word_vector, full_word_idx_dict, full_idx_word_dict = get_features(full_processed_data, prefix='train')

# Training
train_processed_data = pre_process(X_Train, Y_Train)
# convert sentences into indicies corresponding to the word vector indices
train_data_indices = np.array([sentence_to_idx(sentence, full_word_idx_dict) for sentence in train_processed_data])

# Validation
val_processed_data = pre_process(X_Val, Y_Val)
val_data_indices = np.array([sentence_to_idx(sentence, full_word_idx_dict) for sentence in val_processed_data])

# In[ ]:

if os.path.exists(drive_prefix + 'full_processed_data'):
    print('full_processed_data exists, loading from file...')
    full_processed_data = pickle.load(open(drive_prefix + 'full_processed_data', 'rb'))
else:
    full_processed_data = pre_process(X, Y)
    full_word_vector, full_word_idx_dict, full_idx_word_dict = get_features(full_processed_data, prefix='train')

if os.path.exists(drive_prefix + 'train_processed_data'):
    print('train_processed_data exists, loading from file...')
    train_processed_data = pickle.load(open(drive_prefix + 'train_processed_data', 'rb'))
else:
    train_data_indices = np.array([sentence_to_idx(sentence, full_word_idx_dict) for sentence in train_processed_data])

if os.path.exists(drive_prefix + 'val_processed_data'):
    print('val_processed_data exists, loading from file...')
    val_processed_data = pickle.load(open(drive_prefix + 'val_processed_data', 'rb'))
else:
    val_processed_data = pre_process(X, Y)
    val_data_indices = np.array([sentence_to_idx(sentence, full_word_idx_dict) for sentence in val_processed_data])

# In[ ]:

## Using this to speed things up
def dump(data, name):
    pickle.dump(data, open(drive_prefix + name, 'wb'))

dump(full_processed_data, 'full_processed_data')
dump(train_processed_data, 'train_processed_data')
dump(train_data_indices, 'train_data_indices')
dump(val_processed_data, 'val_processed_data')
dump(val_data_indices, 'val_data_indices')

# In[ ]:

val_data_indices.shape

# ## Model

# In[ ]:

# Create the embedding layer weights based on the pre-trained word vector
def create_pretrained_emb_layer(word_vector):
    # vocab_size, embed_dim = word_vector.shape
    embed_layer = nn.Embedding.from_pretrained(torch.tensor(word_vector).float(), freeze=False)
    return embed_layer

def get_pad_mask(seq, pad_idx):
    return (seq != pad_idx).unsqueeze(-2)

class PositionEmbedding(nn.Module):
    def __init__(self, dropout=DROPOUT_RATE, max_len=500):
        super(PositionEmbedding, self).__init__()

        self.dropout = nn.Dropout(dropout)
        self.pos_embedding = self.compute_pos_embedding(max_len)

    def compute_pos_embedding(self, max_len):

        pos = torch.arange(max_len).unsqueeze(1)
        index = torch.arange(EMB_SIZE).unsqueeze(0)
        angles = 1 / np.power(10000, (2 * (index // 2)) / np.float32(EMB_SIZE))

        pos_to_angle = pos * angles

        # apply sin to even indices in the array; 2i
        pos_to_angle[:, 0::2] = np.sin(pos_to_angle[:, 0::2])

        # apply cos to odd indices in the array; 2i+1
        pos_to_angle[:, 1::2] = np.cos(pos_to_angle[:, 1::2])

        # pos_embedding = pos_to_angle.unsqueeze(0).transpose(0, 1)
        pos_embedding = pos_to_angle.unsqueeze(0)

        return torch.FloatTensor(pos_embedding).to(device)

    def forward(self, x):
        x += self.pos_embedding[:, :x.shape[1]].clone().detach() # adding another feature to the word embedding
        return self.dropout(x)

class PositionwiseFeedForward(nn.Module):
    def __init__(self, d_in, hidden_dim, dropout=DROPOUT_RATE):
        super(PositionwiseFeedForward, self).__init__()
        self.pos1 = nn.Linear(d_in, hidden_dim)
        self.pos2 = nn.Linear(hidden_dim, d_in)
        self.relu = nn.ReLU()
        self.normalization = nn.LayerNorm(d_in, eps=1e-6)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):

        residual = x
        out = self.pos1(x)
        out = self.relu(out)
        out = self.pos2(out)
        out = self.dropout(out)
        out += residual

        out = self.normalization(out)

        return out

class ScaledDotProductAttention(nn.Module):
    def __init__(self, temperature, attn_dropout=DROPOUT_RATE):
        super(ScaledDotProductAttention, self).__init__()
        self.temperature = temperature
        self.dropout = nn.Dropout(attn_dropout)
        self.softmax = nn.Softmax(dim=-1)

    def forward(self, q, k, v, mask):

        attn = torch.matmul(q / self.temperature, k.transpose(2, 3))

        attn = attn.masked_fill(mask == 0, -1e7)

        attn = self.softmax(attn)

        return torch.matmul(attn, v), self.dropout(attn)

class MultiHeadAttention(nn.Module):
    def __init__(self, n_head, key_dim, value_dim, dropout=DROPOUT_RATE):
        super().__init__()

        self.n_head = n_head
        self.key_dim = key_dim
        self.value_dim = value_dim

        self.query_layer = nn.Linear(EMB_SIZE, n_head * key_dim, bias=False)
        self.key_layer = nn.Linear(EMB_SIZE, n_head * key_dim, bias=False)
        self.value_layer = nn.Linear(EMB_SIZE, n_head * value_dim, bias=False)
        self.fc = nn.Linear(n_head * value_dim, EMB_SIZE, bias=False)

        self.attention = ScaledDotProductAttention(temperature=key_dim ** 0.5).to(device)

        self.dropout = nn.Dropout(dropout)
        self.layer_norm = nn.LayerNorm(EMB_SIZE, eps=1e-6)

    def forward(self, q, k, v, attn_mask):

        key_dim, value_dim, n_head = self.key_dim, self.value_dim, self.n_head
        batch_size, q_len = q.size(0), q.size(1)
        k_len = k.size(1)
        v_len = v.size(1)

        residual = q

        q = self.query_layer(q).view(batch_size, q_len, n_head, key_dim)
        k = self.key_layer(k).view(batch_size, k_len, n_head, key_dim)
        v = self.value_layer(v).view(batch_size, v_len, n_head, value_dim)

        q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)

        attn_mask = attn_mask.unsqueeze(1)

        q, attn = self.attention(q, k, v, mask=attn_mask)

        q = q.transpose(1, 2).contiguous().view(batch_size, q_len, -1)
        q = self.fc(q)
        q = self.dropout(q)
        q += residual

        q = self.layer_norm(q)

        return q, attn

class EncoderLayer(nn.Module):
    def __init__(self, d_inner, n_head, key_dim, value_dim, dropout=DROPOUT_RATE):
        super(EncoderLayer, self).__init__()
        self.slf_attn = MultiHeadAttention(n_head=n_head, key_dim=key_dim, value_dim=value_dim, dropout=dropout).to(device)
        self.pos_ffn = PositionwiseFeedForward(EMB_SIZE, d_inner, dropout=dropout).to(device)

    def forward(self, enc_input, mask=None):
        enc_output, enc_slf_attn = self.slf_attn(enc_input, enc_input, enc_input, attn_mask=mask)
        enc_output = self.pos_ffn(enc_output)
        return enc_output, enc_slf_attn

class DecoderLayer(nn.Module):
    def __init__(self, d_model, d_inner, n_head, key_dim, value_dim, dropout=DROPOUT_RATE):
        super(DecoderLayer, self).__init__()
        self.slf_attn = MultiHeadAttention(n_head, key_dim, value_dim, dropout=dropout).to(device)
        self.enc_attn = MultiHeadAttention(n_head, key_dim, value_dim, dropout=dropout).to(device)
        self.pos_ffn = PositionwiseFeedForward(d_model, d_inner, dropout=dropout).to(device)

    def forward(self, d_in, e_out,y_mask=None, x_mask=None):
        d_out, d_slf_attn = self.slf_attn(d_in, d_in, d_in, attn_mask=y_mask)
        d_out, d_e_attn = self.enc_attn(d_out, e_out, e_out, attn_mask=x_mask)
        d_out = self.pos_ffn(d_out)
        return d_out, d_slf_attn, d_e_attn

class Decoder(nn.Module):
    def __init__(self, n_layers, n_head, key_dim, value_dim,d_model, d_inner, dropout=DROPOUT_RATE):
        super(Decoder, self).__init__()

        self.dropout = nn.Dropout(p=dropout)
        self.layer_stack = nn.ModuleList([DecoderLayer(d_model, d_inner, n_head, key_dim, value_dim, dropout=dropout).to(device) for _ in range(n_layers)])
        self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)

    def forward(self, y, y_mask, e_out, x_mask):
        d_out = self.dropout(y)
        d_out = self.layer_norm(d_out)
        for dec_layer in self.layer_stack:
            d_out, dec_slf_attn, dec_enc_attn = dec_layer(d_out, e_out, y_mask=y_mask, x_mask=x_mask)

        return d_out,

class Encoder(nn.Module):
    def __init__(self, n_layers, n_head, key_dim, value_dim, dropout=DROPOUT_RATE):
        super(Encoder, self).__init__()
        self.dropout = nn.Dropout(dropout)
        self.layers = nn.ModuleList([ EncoderLayer(E_HIDDEN_DIM, n_head, key_dim, value_dim, dropout=dropout).to(device) for _ in range(n_layers)])
        self.normalization = nn.LayerNorm(EMB_SIZE, eps=1e-6)

    def forward(self, x, x_mask):
        e_out = self.dropout(x)
        e_out = self.normalization(e_out)

        for enc_layer in self.layers:
            e_out, enc_slf_attn = enc_layer(e_out, mask=x_mask)

        return e_out,

class Transformer(nn.Module):
    def __init__(self, num_heads, word_vector, word_idx_dict, idx_word_dict):
        super(Transformer, self).__init__()
        self.word_idx_dict = word_idx_dict
        self.pos_embedding = PositionEmbedding(max_len=MAX_ARTICLE_LEN, dropout=DROPOUT_RATE).to(device)
        self.word_embedding = create_pretrained_emb_layer(word_vector).to(device)

        self.encoder = Encoder(n_layers=3, n_head=num_heads, key_dim=KEY_DIM, value_dim=VALUE_DIM, dropout=DROPOUT_RATE).to(device)
        self.decoder = Decoder(n_layers=3, n_head=num_heads, key_dim=KEY_DIM, value_dim=VALUE_DIM, d_model=EMB_SIZE, d_inner=D_HIDDEN_DIM, dropout=DROPOUT_RATE).to(device)

        self.initial_probs = self.get_initial_probs(word_vector.shape[0], word_idx_dict[START_CHAR])

        self.fc_to_logits = nn.Linear(EMB_SIZE, len(idx_word_dict), bias=False)

    def mask_future_seq(self, seq):
        # Our attention is focused only allowed on the past sequence, so we mask
        sz_b, len_s = seq.size()
        subsequent_mask = (1 - torch.triu(torch.ones((1, len_s, len_s), device=seq.device), diagonal=1)).bool()
        return subsequent_mask

    def get_initial_probs(self, vocab_size, initial_token_idx):
        probs = torch.zeros(1, vocab_size)
        probs[0, initial_token_idx] = 1
        return probs.float()

    def forward(self, x, y):
        x_mask = get_pad_mask(x, self.word_idx_dict[PAD_CHAR])
        y_mask = get_pad_mask(y, self.word_idx_dict[PAD_CHAR]) & self.mask_future_seq(y)

        x = self.word_embedding(x)
        x = self.pos_embedding(x)

        y = self.word_embedding(y)
        y = self.pos_embedding(y)

        # x = (batchsize, seqlen, embed dim)
        # y.shape = (batchsize, seqlen)

        # x_mask = (batchsize, 1, sequence length)
        # y_mask = (batchsize, sequence length, sequence length)
        # e_output = (batchsize, sequence length, embedding dimension)
        # d_output = (batchsize, sequence length, embedding dimension)

        e_out, *_ = self.encoder(x, x_mask)
        d_out, *_ = self.decoder(y, y_mask, e_out, x_mask)

        # self.fc_to_logits(dec_output) = (batchsize, seqlen, vocab len)
        logits = self.fc_to_logits(d_out) * 1 # no weight sharing

        return logits.view(-1, logits.size(2))

# In[ ]:

class ScheduledOptim():
    '''A simple wrapper class for learning rate scheduling'''

    def __init__(self, optimizer, init_lr, d_model, n_warmup_steps):
        self._optimizer = optimizer
        self.init_lr = init_lr
        self.d_model = d_model
        self.n_warmup_steps = n_warmup_steps
        self.n_steps = 0

    def step_and_update_lr(self):
        "Step with the inner optimizer"
        self._update_learning_rate()
        self._optimizer.step()

    def zero_grad(self):
        "Zero out the gradients with the inner optimizer"
        self._optimizer.zero_grad()

    def _get_lr_scale(self):
        d_model = self.d_model
        n_steps, n_warmup_steps = self.n_steps, self.n_warmup_steps
        return (d_model ** -0.5) * min(n_steps ** (-0.5), n_steps * n_warmup_steps ** (-1.5))

    def _update_learning_rate(self):
        ''' Learning rate scheduling per step '''

        self.n_steps += 1
        lr = self.init_lr * self._get_lr_scale()

        for param_group in self._optimizer.param_groups:
            param_group['lr'] = lr

# ## Train Functions

# In[ ]:

def _construct_summary(tokens, mapping):
    summary = []
    for token in tokens:
        token = token.item()
        if token in mapping and mapping[token] != PAD_CHAR:
            summary.append(mapping[token])
    return summary

def one_pass(x, y, gold, model, optim, criterion):
    optim.zero_grad()

    y_hat = model.forward(x, y) # (batchsize * seq len, vocab size)

    # NO LABEL SMOOTHING
    # loss = criterion(y_hat, gold)

    # LABEL SMOOTHING
    alpha = 0.1
    n_class = y_hat.size(1)

    y_hat_hot = torch.zeros_like(y_hat).scatter(1, gold.view(-1, 1), 1)
    y_hat_smooth = (1 - alpha) * y_hat_hot + (1 - y_hat_hot) * alpha / (n_class - 1) # y_ls = (1 - α) * y_hot + α / K
    log_prb = F.log_softmax(y_hat, dim=1)

    non_pad_mask = gold.ne(full_word_idx_dict[PAD_CHAR])
    loss = -(y_hat_smooth * log_prb).sum(dim=1)
    loss = loss.masked_select(non_pad_mask).sum()

    pred = y_hat.max(1)[1]

    pred_summary = ' '.join(_construct_summary(pred, full_idx_word_dict)[1:-1])
    true_summary = ' '.join(_construct_summary(y.contiguous().view(-1), full_idx_word_dict)[1:-1])

    loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1)

    # optim.step() #using lr-scheduler outside in train()
    optim.step_and_update_lr() # custom lr scheduler

    return loss, true_summary, pred_summary

## Evaluation One Pass
def one_pass_eval(model, criterion):
    model.eval()
    np.random.shuffle(val_data_indices)
    val_loader = get_batch(val_data_indices)
    eval_loss = 0
    eval_pass_pred = []
    eval_pass_true = []
    for _ in range(NUM_EVAL_BATCHES):
        data = val_loader.__next__()
        x, y = np.array(data[:, 0]), np.array(data[:, 1])
        x = torch.tensor(np.array([x[i] for i in range(len(x))])).long()
        y = torch.tensor(np.array([y[i] for i in range(len(y))])).long()

        x = x.to(device)
        y, gold = map(lambda x: x.to(device), patch_trg(y))
        y = y.to(device)

        with torch.no_grad():
            y_hat = model.forward(x, y) # (batchsize * seq len, vocab size)

            # loss = criterion(y_hat, gold)
            alpha = 0.1
            n_class = y_hat.size(1)

            y_hat_hot = torch.zeros_like(y_hat).scatter(1, gold.view(-1, 1), 1)
            y_hat_smooth = (1 - alpha) * y_hat_hot + (1 - y_hat_hot) * alpha / (n_class - 1) # y_ls = (1 - α) * y_hot + α / K
            log_prb = F.log_softmax(y_hat, dim=1)

            non_pad_mask = gold.ne(full_word_idx_dict[PAD_CHAR])
            loss = -(y_hat_smooth * log_prb).sum(dim=1)
            loss = loss.masked_select(non_pad_mask).sum()
            pred = y_hat.max(1)[1]
            temp_loss = loss.item()/BATCH_SIZE
            eval_loss += temp_loss

    model.train()
    eval_loss /=  NUM_EVAL_BATCHES
    return eval_loss

def train(model: Transformer, optim, criterion, model_name):
    print("Starting to train")
    model.train()
    ep_loss = []
    ep_eval_loss = []
    temp_loss = []
    eps = []
    for ep in range(1, EP + 1):
        np.random.shuffle(train_data_indices)
        train_loader = get_batch(train_data_indices)

        data = train_loader.__next__()
        x, y = np.array(data[:, 0]), np.array(data[:, 1])
        x = torch.tensor(np.array([x[i] for i in range(len(x))])).long()
        y = torch.tensor(np.array([y[i] for i in range(len(y))])).long()

        x = x.to(device)
        y, gold = map(lambda x: x.to(device), patch_trg(y))
        y = y.to(device)

        loss, true_summary, pred_summary = one_pass(x, y, gold, model, optim, criterion)

        temp_loss.append(loss.item() / BATCH_SIZE)

        if ep % PRINT_EVERY_EP == 0 and ep > 0:
            eval_loss = one_pass_eval(model, criterion)
            ep_loss.append(sum(temp_loss)/len(temp_loss))
            ep_eval_loss.append(eval_loss)
            eps.append(ep)
            temp_loss = []
            # print(ep, eval_loss, ep_loss[-1])
            # print(f'\nEP:{ep} | Train Loss: {np.array(ep_loss[-PRINT_EVERY_EP:]).mean()} | Val Loss: {np.array(eval_loss).mean()}')
            # print('Training')
            # print(f'True: {true_summary[:100]}')
            # print(f'Pred: {pred_summary[:100]}')
            # print('Validation')
            # print(f'True: {eval_true_summary[0][:100]}')
            # print(f'Pred: {eval_pred_summary[0][:100]}')

        if ep % SAVE_MODEL_EVERY_EP == 0 and ep > 0:
            torch.save({
            'epoch': ep,
            'model': model.state_dict(),
            'loss': loss
            }, f'{drive_prefix}models/{model_name}/checkpoint_{ep}')

    return model, ep_loss, ep_eval_loss, eps

def predict(model, X):
    model.eval()

    pred_summaries = []
    true_summaries = []

    for i in range(0, X.shape[0]):
      x, y = np.array(X[i:i+1, 0]), np.array(X[i:i+1, 1])
      x = torch.tensor(np.array([x[i] for i in range(len(x))])).long()
      y = torch.tensor(np.array([y[i] for i in range(len(y))])).long()

      x = x.to(device)
      y, gold = map(lambda x: x.to(device), patch_trg(y))
      y = y.to(device)

      with torch.no_grad():
          y_hat = model.forward(x, y)

          pred = y_hat.max(1)[1]
          pred_summary = ' '.join(_construct_summary(pred, full_idx_word_dict)[1:-1])
          true_summary = ' '.join(_construct_summary(y.contiguous().view(-1), full_idx_word_dict)[1:-1])
          pred_summaries.append(pred_summary)
          true_summaries.append(true_summary)

    return pred_summaries, true_summaries

# # Evaluation Metrics

# In[ ]:

def compute_rouge(pred_summaries, true_summaries):
  from rouge_score import rouge_scorer
  scorer = rouge_scorer.RougeScorer(['rouge1', 'rougeL'], use_stemmer=True)
  rogue1_precision = []
  rogue1_recall = []
  rogue1_fmeasure = []
  rogueL_precision = []
  rogueL_recall = []
  rogueL_fmeasure = []
  for i in range(len(pred_summaries)):
    score = scorer.score(pred_summaries[i], true_summaries[i])
    rogue1_precision.append(score['rouge1'].precision)
    rogue1_recall.append(score['rouge1'].recall)
    rogue1_fmeasure.append(score['rouge1'].fmeasure)
    rogueL_precision.append(score['rougeL'].precision)
    rogueL_recall.append(score['rougeL'].recall)
    rogueL_fmeasure.append(score['rougeL'].fmeasure)

  scores = {'rogue1_precision':rogue1_precision, 'rogue1_recall':rogue1_recall, 'rogue1_fmeasure': rogue1_fmeasure, 'rogueL_precision':rogueL_precision, 'rogueL_recall':rogueL_recall, 'rogueL_fmeasure': rogueL_fmeasure }
  return scores

# In[ ]:

def plot_rogue(scores, label_text):
  from matplotlib import pyplot as plt
  import pandas as pd
  import seaborn as sns
  for measure in scores.keys():
    df = pd.DataFrame({"score":scores['rogue1_precision'] , "dummy": range(len(scores['rogue1_precision']))})
    sns.set_style("whitegrid")
    f, (ax_box, ax_hist) = plt.subplots(2, sharex=True, gridspec_kw= {"height_ratios": (0.2, 1)})
    f.set_size_inches(11.7, 8.27)
    mean=df['score'].mean()
    median=df['score'].median()
    mode=df['score'].mode()[0]

    sns.boxplot(df["score"], ax=ax_box)
    ax_box.axvline(mean, color='r', linestyle='--')
    ax_box.axvline(median, color='g', linestyle='-')
    ax_box.axvline(mode, color='b', linestyle='-')

    sns.distplot(df["score"], ax=ax_hist)
    ax_hist.axvline(mean, color='r', linestyle='--')
    ax_hist.axvline(median, color='g', linestyle='-')
    ax_hist.axvline(mode, color='b', linestyle='-')

    plt.legend({'Mean':mean,'Median':median,'Mode':mode})
    plt.title(measure + f'{measure} {label_text}')
    ax_box.set(xlabel='')
    plt.show()

# In[ ]:

def compute_rouge_summary_stats(scores):
  def mean_confidence_interval(data, confidence=0.95):
    import scipy.stats
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1)
    return m, m-h, m+h
  stats = {}
  for key in scores.keys():
    stats[key] = mean_confidence_interval(scores[key])
  return stats

# In[ ]:

def plot_losses(ep_loss, ep_eval_loss, eps):
  import matplotlib.pyplot as plt
  import matplotlib.ticker as ticker
  plt.plot(eps, ep_loss, label = 'train')
  plt.plot(eps, ep_eval_loss, label = 'val')
  plt.legend()
  plt.xlabel('Epoch')
  plt.ylabel('Train Losses')
  plt.title('Training Loss')
  plt.show()

# ## Run
# - **make sure to create a folder in models/called transformers** (i.e. ese546/models/transformers must exist)

# In[ ]:

# MAIN DRIVER
model_name = 'transformers'
transformer = Transformer(num_heads=NUM_HEADS,
                          word_vector=full_word_vector,
                          word_idx_dict=full_word_idx_dict,
                          idx_word_dict=full_idx_word_dict).to(device)

# model_optim = optim.Adam(transformer.parameters(), lr=LR)

model_optim = ScheduledOptim(
        optim.Adam(transformer.parameters(), betas=(0.9, 0.98), eps=1e-09),
        2.0, EMB_SIZE, 3000)

criterion = nn.CrossEntropyLoss(ignore_index=full_word_idx_dict[PAD_CHAR], reduction='sum')

model, ep_loss, ep_eval_loss, eps = train(transformer, model_optim, criterion, model_name)

# ## TODO
#
# - validation
# - metrics
# - try to preprocess data better (e.g. there are lots of "t" in the label)
# - hyperparameter tweaking

# # Evaluations

# ## Summary of Hyperparamters
#

# In[ ]:

print(f'EMB_SIZE : {EMB_SIZE}')
print(f'BATCH_SIZE : {BATCH_SIZE}')
print(f'NUM_EVAL_BATCHES : {NUM_EVAL_BATCHES}')
print(f'NUM_HEADS : {NUM_HEADS}')
print(f'DROPOUT_RATE : {DROPOUT_RATE}')
print(f'MAX_ARTICLE_LEN : {MAX_ARTICLE_LEN}')
print(f'MAX_LABEL_LEN : {MAX_LABEL_LEN}')
print(f'E_HIDDEN_DIM : {E_HIDDEN_DIM}')
print(f'D_HIDDEN_DIM : {D_HIDDEN_DIM}')
print(f'KEY_DIM : {KEY_DIM}')
print(f'VALUE_DIM : {VALUE_DIM}')
print(f'FORCE_CREATE_DICT : {FORCE_CREATE_DICT}')

# ## Plot training vs validation loss

# In[ ]:

plot_losses(ep_loss, ep_eval_loss, eps)

# ## Compute training vs validation rouge scores

# In[ ]:

pred_summaries_train, true_summaries_train = predict(model, train_data_indices)
pred_summaries_val, true_summaries_val = predict(model, val_data_indices)

# In[ ]:

scores_train =  compute_rouge(pred_summaries_train, true_summaries_train)
scores_val  = compute_rouge(pred_summaries_val, true_summaries_val)

# ##  Rouge scores Training

# In[ ]:

compute_rouge_summary_stats(scores_train)

# ## Rouge Scores Validation

# In[ ]:

compute_rouge_summary_stats(scores_val)

# ## Rouge Scores Histograms (Training)

# In[ ]:

plot_rogue(scores_train, 'Training')

# In[ ]:

plot_rogue(scores_train, '')


# _*_ coding :utf-8 _*_
__author__ = 'du'
__blog__ = 'www.cnblogs.com/anmutu;www.zmfei4.com;'
__date__ = '2020/1/6 2:01'

from collections import defaultdict
from datetime import datetime

# 效率最高的其实就是这个记忆搜索法。

total = defaultdict(int)
total_memory = defaultdict(int)

def fib_recursion(k):
    assert k > 0, "k的值必须大于0"
    if k in [1, 2]:
        return 1
    else:
        global total
        total[k] += 1
        return fib_recursion(k - 2) + fib_recursion(k - 1)

# 将值存到total里,在递归前判断是否在这个total里,如果在就取total里的值,如果不在就将新值存到total里.
def fib_recursion_memory(k):
    assert k > 0, "k的值必须大于0"
    if k in [1, 2]:
        return 1
    global total_memory
    if k in total_memory:
        result = total_memory[k]
    else:
        result = fib_recursion_memory(k - 2) + fib_recursion_memory(k - 1)
        total_memory[k] = result
    return result

if __name__ == "__main__":
    start_time = datetime.now()
    print("普通递归方式。第{0}位的数的值是{1},此次递归耗时{2}".format(35, fib_recursion(35), (datetime.now()-start_time).total_seconds()))
    print("递归的次数也是一个斐波拉契数列,比如你看:{0}".format(total))

    start_time_memory = datetime.now()
    print("采用记忆搜索的递归方式。第{0}位的数的值是{1},递归耗时{2}".format(35, fib_recursion_memory(35), (datetime.now()-start_time_memory).total_seconds()))
    print("递归的次数也是一个斐波拉契数列,比如你看:{0}".format(total_memory))


__author__ = 'Justin Finkle'
__email__ = 'jfinkle@u.northwestern.edu'

import sys
import os
import pandas as pd
from matplotlib.backends.backend_pdf import PdfPages
import numpy as np
import networkx as nx
from collections import deque
from Swing.util.Evaluator import Evaluator
from Swing.util.lag_identification import get_experiment_list, xcorr_experiments, calc_edge_lag
import matplotlib.pyplot as plt
import matplotlib as mpl
from matplotlib import rcParams, gridspec
from matplotlib.path import Path
import matplotlib.patches as patches
from matplotlib.patches import Polygon
from scipy.stats import fisher_exact, linregress, ttest_rel, mannwhitneyu, ttest_ind, pearsonr, zscore, wilcoxon
import palettable

mpl.rcParams['pdf.fonttype'] = 42
mpl.rcParams['font.sans-serif'] = 'Arial'

def get_swing_list(pkl):
    x = pd.read_pickle(pkl)
    edges = x.edge_list
    full_edges = list(set([e for sub in edges for e in sub]))
    ranking = np.array([[sub.index(e) for e in full_edges] for sub in edges])
    edge_rankings = pd.DataFrame(ranking.T, index=full_edges)
    return edge_rankings

def get_true_edges(gold_filename):
    evaluator = Evaluator(gold_filename, '\t')
    edges = evaluator.gs_flat.tolist()
    return edges, evaluator

if __name__ == '__main__':
    swing_rankings = get_swing_list('gardner_RF_w7_results.pkl')
    mean_swing = swing_rankings.mean(axis=1).sort_values().to_frame().reset_index()
    # mean_swing = swing_rankings[15].sort_values().to_frame().reset_index()
    mean_swing.columns = ['regulator-target', 'rank_importance']
    mean_swing['rank'] = np.arange(1, len(mean_swing)+1)

    params = {'axes.labelsize': 16, 'font.size': 14, 'legend.fontsize': 14,
              'xtick.labelsize': 14, 'ytick.labelsize': 14, 'text.usetex': False}
    rcParams.update(params)

    colors1 = palettable.colorbrewer.qualitative.Set1_8.mpl_colors
    colors2 = palettable.colorbrewer.qualitative.Set2_8.mpl_colors
    colors3 = palettable.colorbrewer.qualitative.Set3_8.mpl_colors
    split_c2 = [colors2[i] for i in [0, 2]]
    paired = palettable.colorbrewer.qualitative.Paired_10.mpl_colors

    gs = "../../data/invitro/gardner_goldstandard.tsv"
    true_edges, ee = get_true_edges(gs)

    _, _, swing_roc = ee.calc_roc(mean_swing)
    swing_roc = swing_roc.values[-1]

    _, _, swing_pr = ee.calc_pr(mean_swing)
    swing_pr = swing_pr.values[-1]

    data = get_experiment_list(gs.replace('goldstandard', 'timeseries'), 14, 1.0)
    gene_list = data[0].columns.values.tolist()
    xcorr_array = xcorr_experiments(data)
    edge_lags = calc_edge_lag(xcorr_array, gene_list, 1, 0.3, timestep=1)
    edge_lags = edge_lags.set_index('Edge').iloc[:, 2]
    true_lags = edge_lags[edge_lags.index.isin(true_edges)]

    roc_list = pd.read_pickle('gardner_rf_roc.pkl')
    pr_list = pd.read_pickle('gardner_rf_pr.pkl')
    rankings = pd.read_pickle('gardner_rf_rankings.pkl')
    # s_roc_list = pd.read_pickle('gardner_swing_rf_roc.pkl')
    # s_pr_list = pd.read_pickle('gardner_swing_rf_pr.pkl')
    # s_rankings = pd.read_pickle('gardner_swing_rf_rankings.pkl')
    avg_rank = pd.DataFrame()
    # s_avg_rank = pd.DataFrame()
    avg_rank['regulator-target'] = rankings[0].sort_values('regulator-target').values[:, 0]
    # s_avg_rank['regulator-target'] = s_rankings[0].sort_values('regulator-target').values[:, 0]
    rank = np.array([rankings[ii].sort_values('regulator-target')['Rank'].values for ii in range(len(rankings))]).T
    avg_rank['rank'] = np.mean(rank, axis=1)
    # s_rank = np.array([s_rankings[ii].sort_values('regulator-target')['Rank'].values for ii in range(len(s_rankings))]).T
    # s_avg_rank['rank'] = np.mean(s_rank, axis=1)

    s_avg_rank = mean_swing.copy()
    avg_rank.set_index('regulator-target', drop=True, inplace=True)
    s_avg_rank.set_index('regulator-target', drop=True, inplace=True)
    avg_rank['s_rank'] = s_avg_rank['rank']
    avg_rank.sort_values('rank', inplace=True)
    avg_rank['rank'] = range(len(avg_rank))
    avg_rank.sort_values('s_rank', inplace=True)
    avg_rank['s_rank'] = range(len(avg_rank))

    avg_rank = pd.concat([edge_lags, avg_rank], axis=1, join='inner')
    te_ranks = avg_rank[avg_rank.index.isin(true_edges)]
    # avg_rank.reset_index(inplace=True)
    # auroc = ee.calc_roc(avg_rank.sort_values('rank'))[2].values[-1]
    # s_auroc = ee.calc_roc(avg_rank.sort_values('s_rank'))[2].values[-1]
    # avg_rank.set_index('regulator-target', drop=True, inplace=True)

    # Promoted
    te_p = te_ranks[te_ranks['s_rank'] < te_ranks['rank']]

    # Demoted
    te_d = te_ranks[(te_ranks['s_rank'] > te_ranks['rank'])]

    #Same
    te_s = te_ranks[(te_ranks['s_rank'] == te_ranks['rank'])]
    fe_ranks = avg_rank[~avg_rank.index.isin(true_edges)]

    fig = plt.figure(figsize=(10, 9))
    left_int = 2
    right_int = 2
    cols = left_int + right_int
    ax1 = plt.subplot2grid((2, cols), (0, 0), rowspan=2, colspan=left_int)
    ax2 = plt.subplot2grid((2, cols), (0, left_int), colspan=right_int)
    ax3 = plt.subplot2grid((2, cols), (1, left_int), colspan=right_int)
    lw = 3
    shift_down = ['ruvA', 'polB', 'recA']
    shift_up = ['umuDC']
    shift_pad = 0.2

    for row in fe_ranks.iterrows():
        edge = row[0]
        lag = row[1]["Lag"]
        start = row[1].values[1]
        end = row[1].values[2]
        verts = [(0, start),
                 (0.5, start),
                 (0.5, end),
                 (1, end)]
        codes = [Path.MOVETO, Path.CURVE4, Path.CURVE4, Path.CURVE4]

        path = Path(verts, codes)
        patch = patches.PathPatch(path, facecolor='none', lw=2, ec='0.75', zorder=0)
        ax1.add_patch(patch)

    for row in te_d.iterrows():
        edge = row[0]
        lag = row[1]["Lag"]
        start = row[1].values[1]
        end = row[1].values[2]
        verts = [(0, start),
                 (0.5, start),
                 (0.5, end),
                 (1, end)]
        codes = [Path.MOVETO, Path.CURVE4, Path.CURVE4, Path.CURVE4]

        path = Path(verts, codes)
        ec = paired[5]
        patch = patches.PathPatch(path, facecolor='none', lw=lw, ec=ec)
        ax1.add_patch(patch)
        va = 'center'
        if edge[1] in shift_down:
            va = 'top'
            start -= shift_pad
        elif edge[1] in shift_up:
            va = 'bottom'
            start += shift_pad
        if lag > 0:
            edge_str = edge[0] + u"\u2192" + edge[1] + "\nLag = " + str(int(lag))
        else:
            edge_str = edge[0] + u"\u2192" + edge[1]
        ax1.text(-0.02, start, edge_str, ha='right', va=va)

    for row in te_s.iterrows():
        edge = row[0]
        lag = row[1]["Lag"]
        start = row[1].values[1]
        end = row[1].values[2]
        verts = [(0, start),
                 (0.5, start),
                 (0.5, end),
                 (1, end)]
        codes = [Path.MOVETO, Path.CURVE4, Path.CURVE4, Path.CURVE4]

        path = Path(verts, codes)
        ec = 'k'
        patch = patches.PathPatch(path, facecolor='none', lw=lw, ec=ec, zorder=0)
        ax1.add_patch(patch)
        va = 'center'
        if edge[1] in shift_down:
            va = 'top'
            start -= shift_pad
        elif edge[1] in shift_up:
            va = 'bottom'
            start += shift_pad
        if lag > 0:
            edge_str = edge[0] + u"\u2192" + edge[1] + "\nLag = " + str(int(lag))
        else:
            edge_str = edge[0] + u"\u2192" + edge[1]
        ax1.text(-0.02, start, edge_str, ha='right', va=va)

    for row in te_p.iterrows():
        edge = row[0]
        lag = row[1]["Lag"]
        start = row[1].values[1]
        end = row[1].values[2]
        verts = [(0, start),
                 (0.5, start),
                 (0.5, end),
                 (1, end)]
        codes = [Path.MOVETO, Path.CURVE4, Path.CURVE4, Path.CURVE4]

        path = Path(verts, codes)
        ec = paired[1]
        if edge == ('lexA', 'umuDC'):
            ec = paired[3]
        patch = patches.PathPatch(path, facecolor='none', lw=lw, ec=ec)
        ax1.add_patch(patch)
        if lag > 0:
            edge_str = edge[0] + u"\u2192" + edge[1] + "\nLag = " + str(int(lag))
        else:
            edge_str = edge[0] + u"\u2192" + edge[1]
        va = 'center'
        if edge[1] in shift_down:
            va = 'top'
            start -= shift_pad
        elif edge[1] in shift_up:
            va = 'bottom'
            start += shift_pad
        ax1.text(-0.02, start, edge_str, ha='right', va=va)

    ax1.set_ylim([57, -1])
    ax1.plot([0, 0], [-1, 57], '-', c='k', lw=1)
    # ax1.spines['right'].set_visible = False
    ax1.spines['top'].set_bounds(0, 1)
    ax1.spines['bottom'].set_bounds(0, 1)
    ax1.spines['left'].set_position(('data', 0))
    ax1.set_xlim([-0.6, 1])
    ax1.yaxis.tick_right()
    ax1.set_ylabel('Rank', rotation=0, ha='center')
    ax1.yaxis.set_label_coords(1.005, 1.005)
    ax1.tick_params(axis='x', which='both', bottom='off', top='off')
    ax1.set_xticks([0, 1])
    ax1.set_xticklabels(['RF', 'SWING-RF'])

    # In experiment 4, G2 and nothing else upstream is perturbed, so the relation between G2, G1 is clearer
    # The apparent lag is 2, so shift it that much
    # Plot unlagged time series and correlation
    time = data[0].index.values
    lexa = zscore(data[0]['lexA'].values, ddof=1)
    umudc = zscore(data[0]['umuDC'].values, ddof=1)

    ax2.plot(time, lexa, '.-', label='lexA', c=paired[3], lw=3, ms=15)
    ax2.plot(time, umudc, '.--', dashes=(3, 2), label='umuDC', c='k', lw=3, ms=15, mfc='w', mew=2)
    ax2.plot(time[:-1], umudc[1:], '.-', label='umuDC-shifted', c='k', lw=3, ms=15)
    arrow_nums = [1, 2, 3, 7, 10, 12, 13]
    pad = 0.2
    for idx, point in enumerate(time):
        if idx > 0 and idx in arrow_nums:
            ax2.annotate("", xy=(time[idx-1]+pad, umudc[idx]+.001),
                         xytext=(point-pad, umudc[idx]+.001),
                         arrowprops=dict(facecolor='k', headlength=8, headwidth=7, width=1.45))
    # ax2.set_yticks(np.arange(0.0, 0.6, 0.1))
    # ax2.set_xticks(np.arange(0.0, 1200, 200))
    ax2.set_ylabel('Normalized expression')
    ax2.set_xlabel('Time')
    ax2.legend(loc='best', numpoints=2)

    reg_fit = linregress(lexa, umudc)
    shift_fit = linregress(lexa[:-1], umudc[1:])

    ax3.plot(lexa, umudc, '.', c='k', mfc='w', mew=2, ms=15, label='umuDC')
    ax3.plot(lexa[:-1], umudc[1:], '.', c='k', ms=15, label='umuDC-shifted')
    xvals = np.array([np.min(lexa), np.max(lexa)])
    ax3.plot(xvals, xvals * reg_fit.slope + reg_fit.intercept,
             c='k', ls='--', dashes=(5, 5), lw=1, zorder=0, label=('$\mathregular{R^2 = %0.3f}$' % reg_fit.rvalue))
    xvals = np.array([np.min(lexa[:-1]), np.max(lexa[:-1])])
    ax3.plot(xvals, xvals * shift_fit.slope + shift_fit.intercept,
             c='k', zorder=1, label=('$\mathregular{R^2 = %0.3f}$' % shift_fit.rvalue))
    # ax3.set_yticks(np.arange(0.0, 0.4, 0.1))
    # ax3.set_xticks(np.arange(0.05, 0.5, 0.1))
    ax3.set_xlabel('lexA normalized expression')
    ax3.set_ylabel('Normalized expression')
    ax3.legend(loc='best', numpoints=1, ncol=2, handletextpad=0.5, handlelength=1, columnspacing=0.5)

    plt.tight_layout()
    plt.savefig('gardner_sos_edge_promotion_w7.pdf', fmt='pdf')


from configuration import listOfCountries
from configuration import listOfColors
from configuration import randomDataMax
from configuration import randomDataMin
import pandas as pd
import matplotlib.pyplot as plt
import random
import numpy as np
from matplotlib import cm
from matplotlib.colors import ListedColormap, LinearSegmentedColormap
from PIL import Image
#funkcja do generowania losowych wykresów kołowych

def isfloat(value):
    try:
        float(value)
        return True
    except ValueError:
        return False

def randomPieChart(title):
    #wybór indeksów
    selectedIndexes = []

    indexDataChoice = input("If you want to randomly select countries to be indexes - press 1 and enter, if you want manually enter indexes press 2 and enter: ")
    if str.isdigit(indexDataChoice):
        indexDataChoice = int(indexDataChoice)
        if indexDataChoice == 1:
            randomisedCountryNumberChoice = input("If you want to number of countries to be randomised enter 1, if you want the specify how many countries you want to have in the graph - enter 2: ")
            if str.isdigit(randomisedCountryNumberChoice):
                randomisedCountryNumberChoice = int(randomisedCountryNumberChoice)
                if randomisedCountryNumberChoice == 1:
                    print("przypisanie")
                    recordsNum = random.randint(1, len(listOfCountries))
                    selectedIndexes = listOfCountries
                    random.shuffle(selectedIndexes)
                    selectedIndexes = selectedIndexes[:recordsNum]
                elif randomisedCountryNumberChoice == 2:
                    isNumOfCountriesSet = False
                    while isNumOfCountriesSet == False:
                        recordsNumInput = input("Enter number of countries you want to chose: ")
                        if str.isdigit(recordsNumInput):
                            recordsNum = int(recordsNumInput)
                            selectedIndexes = listOfCountries
                            random.shuffle(selectedIndexes)
                            selectedIndexes = selectedIndexes[:recordsNum]
                            isNumOfCountriesSet = True
                        else:
                            print("Invalid input")
                else:
                    print("There is no such option you know")
            else:
                print("There is no such option you know")
        #opcja aby użytkownik wpisał se indexy ręcznie
        elif indexDataChoice == 2:
            indexDataInputChoice = ""
            print('When you are done entering data write "Done"')
            while indexDataInputChoice != "Done":
                indexDataInputChoice = input('Enter your custom index or enter "Done": ')
                if(indexDataInputChoice != "Done"):
                    selectedIndexes.append(indexDataInputChoice)
            recordsNum = len(selectedIndexes)
        else:
            print("There is no such option you know")
    else:
        print("There is no such option you know")

    print(title, "\n")

    data = []
    for y in range(0, recordsNum):
        print(selectedIndexes[y])
        newDataRecord = random.random() + 1
        data.append(newDataRecord)
    print(selectedIndexes)
    print(data)

    df = pd.DataFrame({title: data})
    plot = df.plot.pie(y=title, figsize=(11, 11), labels=selectedIndexes, fontsize=12, legend=False, autopct='%1.1f%%')
    fig = plot.get_figure()
    fig.savefig("myplot.png")

    im = Image.open(r"myplot.png")
    im.show()

#funkcja do generowania losowych wykresów słupkowych
def randomBarChart(title):
    selectedIndexes = []
    colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k']
    indexDataChoice = input("If you want to randomly select countries to be indexes - press 1 and enter, if you want manually enter indexes press 2 and enter: ")
    if str.isdigit(indexDataChoice):
        indexDataChoice = int(indexDataChoice)
        if indexDataChoice == 1:
            randomisedCountryNumberChoice = input("If you want to number of countries to be randomised enter 1, if you want the specify how many countries you want to have in the graph - enter 2: ")
            if str.isdigit(randomisedCountryNumberChoice):
                randomisedCountryNumberChoice = int(randomisedCountryNumberChoice)
                if randomisedCountryNumberChoice == 1:
                    print("przypisanie")
                    recordsNum = random.randint(1, len(listOfCountries))
                    selectedIndexes = listOfCountries
                    random.shuffle(selectedIndexes)
                    selectedIndexes = selectedIndexes[:recordsNum]
                elif randomisedCountryNumberChoice == 2:
                    isNumOfCountriesSet = False
                    while isNumOfCountriesSet == False:
                        recordsNumInput = input("Enter number of countries you want to chose: ")
                        if str.isdigit(recordsNumInput):
                            recordsNum = int(recordsNumInput)
                            selectedIndexes = listOfCountries
                            random.shuffle(selectedIndexes)
                            selectedIndexes = selectedIndexes[:recordsNum]
                            isNumOfCountriesSet = True
                        else:
                            print("Invalid input")
                else:
                    print("There is no such option you know")
            else:
                print("There is no such option you know")
        elif indexDataChoice == 2:
            indexDataInputChoice = ""
            print('When you are done entering data write "Done"')
            while indexDataInputChoice != "Done":
                indexDataInputChoice = input('Enter your custom index or enter "Done": ')
                if(indexDataInputChoice != "Done"):
                    selectedIndexes.append(indexDataInputChoice)
            recordsNum = len(selectedIndexes)
        else:
            print("There is no such option you know")
    else:
        print("There is no such option you know")

    print(title, "\n")
    print("Please now specife minimum and maximum value for data: ")

    inputMin = input("Specify the minimum value: ")
    while isfloat(inputMin) == False:
        inputMin = input("This is incorrect format of number. Specify the maximum value once again: ")
    minBarPlotValue = float(inputMin)

    inputMax = input("Specify the maximum value: ")
    while isfloat(inputMax) == False:
        inputMax = input("This is incorrect format of number. Specify the maximum value once again: ")
    maxBarPlotValue = float(inputMax)

    if(minBarPlotValue > maxBarPlotValue):
        temp = minBarPlotValue
        minBarPlotValue = maxBarPlotValue
        maxBarPlotValue = temp

    data = []
    for y in range(0, recordsNum):
        print(selectedIndexes[y])
        newDataRecord = random.uniform(minBarPlotValue, maxBarPlotValue)
        data.append(newDataRecord)
    print(selectedIndexes)
    print(data)

    #color selection
    colorToUse = random.choice(colors)

    df = pd.DataFrame({title: data}, index=selectedIndexes)
    fig = df.plot.bar(rot=0, color = colorToUse)
    fig.figure.savefig("myplot.png")

    im = Image.open(r"myplot.png")
    im.show()

def __main__():
    while True:
        print("""
        RandomChartGenerator
        Menu
        0 - Exit Program
        1 - Generate random pie chart
        #in production
        2 - Generate random bar chart (with one bar per index)
        #to be done
        3 - Generate random bar chart (with more than one bar per index)
        4 - generate random scatter plot
        2137 - Generate random COVID stats
        2138 - Generate random voting poll
        """)
        menuChoice = input("Select positon from menu: ")
        if str.isdigit(menuChoice):
            menuChoice = int(menuChoice)
            if menuChoice == 0:
                break
            elif (menuChoice >= 1 and menuChoice <= 2 or menuChoice == 2137):
                title = input("Add some title to your chart: ")
                if menuChoice == 1:
                    randomPieChart(title)
                elif menuChoice == 2:
                    randomBarChart(title)
                if menuChoice == 2137:
                    print("I am sorry, I didn't add this option yet, please be patient")
                    #plan na ten: bierzemy datę, generujemy liczbę, w dni tygodnia do soboty włącznie zwiększa się statystyka, niedzielę i poniedziałek - zmniejsza
                    #jeszcze można dodać generowanie na podstawie liczby wpisanej przez użytkownika jako dzisiejsza
            else:
                print("There is no such option you know")
        else:
                print("There is no such option you know")

__main__()


from ...util import config

import logging

def logLoopProgress(count: int, last_check: int, N: int,
                    name: str) -> int:
    """Function to log the progress of loops through the a of size N.

    Arguments:
        count {int} -- Current iteration count.
        last_check {int} -- Last time logging was done.
        N {int} -- Number of elements being iterated over.
        name {str} -- Name of the iteration to be used in log.

    Returns:
        int -- Updated last_check.
    """

    # Computing log increment
    log_increment = (N / 100) * config.compute['log_freq']

    # Check if the current count has exceeded the increment
    if (count - last_check) > log_increment:
        # Compute completed percentage, log
        percent_complete = count / N * 100

        # Converting to string with 1 decimal point
        percent_complete = ('%.1f' % percent_complete)

        # regulate percentage length (i'm anal)
        if len(percent_complete) < 4:
            percent_complete = ' ' + percent_complete

        logging.info('{0} iteration is {1}% complete'
                     .format(name, percent_complete))

        return count  # Return this to update last_check to current count

    # If increment has not passed, return same last_check
    return last_check


def hanoi(n,d,a,t):
    if n == 1:
        print(d,'-->',a)
    else:
        hanoi(n-1,d,t,a)
        print(d,'-->',a)
        hanoi(n-1,t,a,d)

# hanoi(45, 'D', 'A', 'T')

def fusion(tab1, tab2):
    tab3 = []
    i, j, k = 0, 0, 0
    while i < len(tab1) and j < len(tab2):
        if tab1[i] < tab2[j]:
            tab3.append(tab1[i])
            i += 1
        else:
            tab3.append(tab2[j])
            j += 1
        k += 1

    return tab3

def tri_fusion(tab):
    if len(tab) <= 1:
        return tab
    else:
        tab1 = tri_fusion(tab[:len(tab)//2])
        tab2 = tri_fusion(tab[len(tab)//2:])
        return fusion(tab1, tab2)

print(fusion('ABCDEFGHIJKLMNOPQRSTUVWXYZ','ABCDEFGHIJKLMNOPQRSTUVWXYZ'))
print(tri_fusion([1,3,4,4,3,0]))


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import matplotlib.pyplot as plt
import Advanced_1.confusionMatrix as cm
import pickle

from tensorflow.examples.tutorials.mnist import input_data
from Advanced_1.dataBatcher import DataBatcher
from Advanced_1.learningRateScheduler import LearningRateScheduler
from Advanced_1.network import Network
from Advanced_1.layer import LinearLayer
from Advanced_1.layer import ReLULayer
from Advanced_1.layer import ConvLayer
from Advanced_1.layer import ConvLayerColumns
from Advanced_1.layer import MaxPoolLayer
from Advanced_1.layer import FlattenLayer
from Advanced_1.layer import CrossEntropyLogits

from Advanced_1.convergenceTester import ConvergenceTester

from scipy.misc import toimage
from sklearn.metrics import confusion_matrix

root_dir = 'C:/Users/Dave/Documents/GI13-Advanced/Assignment1';
summaries_dir = root_dir + '/Summaries';
save_dir = root_dir + '/SavedVariables';

def print_confusion_matrix(network, X, y, model_type):
    y_pred = network.get_predictions(X)
    y_true = y.argmax(axis=1)

    cnf_matrix = confusion_matrix(y_true, y_pred)
    cm_filename = model_type.replace(' ','_') + '.p'
    pickle.dump( cnf_matrix, open( cm_filename, "wb" ) )

    class_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
    np.set_printoptions(precision=2)

    # Plot non-normalized confusion matrix
    plt.figure()
    cm.plot_confusion_matrix(cnf_matrix, classes=class_names,
                          title=('Confusion matrix - ' + model_type))
    plt.show()

def part_a(network):
    network.addLayer( LinearLayer(10,(10,784),10) )
    return 'P2_a'

def part_b(network):
    network.addLayer( ReLULayer(128,(128,784),128) )
    network.addLayer( LinearLayer(10,(10,128),10) )
    return 'P2_b'

def part_c(network):
    network.addLayer( ReLULayer(256,(256,784),256) )
    network.addLayer( ReLULayer(256,(256,256),256) )
    network.addLayer( LinearLayer(10,(10,256),10) )
    return 'P2_c'

def part_d(network, X_train, X_test, use_saved=True):

    cl1 = ConvLayerColumns(depth=16, filter_size=3, width=28, height=28)

    if use_saved:
        X_train_cols = np.load('X_train_cols.npy')
        X_test_cols = np.load('X_test_cols.npy')
    else:
        X_train_pad = cl1.pre_pad_all_images(X_train)
        X_train_cols = cl1.im2col(X_train_pad)
        np.save('X_train_cols', X_train_cols)

        X_test_pad = cl1.pre_pad_all_images(X_test)
        X_test_cols = cl1.im2col(X_test_pad)
        np.save('X_test_cols', X_test_cols)

    network.addLayer( cl1 )
    network.addLayer( MaxPoolLayer(2, 14, 14, (4,14,14,16), 16 ))
    network.addLayer( ConvLayer(depth=16, filter_size=3, width=14, height=14) )
    network.addLayer( MaxPoolLayer(2, 7, 7, (4,7,7,16), 16) )
    network.addLayer( FlattenLayer((16,7,7)) )
    network.addLayer( ReLULayer(256, (256,784), 256) )
    network.addLayer( LinearLayer(10,(10,256),10) )

    return 'P2_d', X_train_cols, X_test_cols

def run_part2_models(FLAGS):

    # Import data
    mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)

    X_train = mnist.train.images
    y_train = mnist.train.labels
    X_test = mnist.test.images
    y_test = mnist.test.labels

    network = Network()
    network.addLossFunction( CrossEntropyLogits() )

    if FLAGS.model=='P2_a':
        model_type = part_a(network)
        data_batcher = mnist.train
    elif FLAGS.model=='P2_b':
        model_type = part_b(network)
        data_batcher = mnist.train
    elif FLAGS.model=='P2_c':
        model_type = part_c(network)
        data_batcher = mnist.train
    elif FLAGS.model=='P2_d':
        print('P2:d not implemented')
        exit
        model_type, X_train_cols, X_test_cols = part_d(network, X_train, X_test, use_saved=True)
        data_batcher = DataBatcher(X_train_cols, y_train)

#     rs = np.reshape(mnist.test.images[0], (28,28))
#     toimage(rs).show()

    train_accuracy_hist = []
    test_accuracy_hist = []
    conv_tester = ConvergenceTester(0.0005, lookback_window=25) #stop if converged to within 0.01%
    learning_rate = 0.005
    decay = learning_rate / 50000
#     learning_rate = 0.1
#     decay = learning_rate / 2e6

    model_filename = FLAGS.saved_model_dir + '/' + "trained_model_" + model_type + ".p"

    if FLAGS.use_saved:
        network = pickle.load( open( model_filename, "rb" ) )
        train_accuracy = network.get_accuracy(X_train, y_train)
        test_accuracy = network.get_accuracy(X_test, y_test)
    else:
        lrs = LearningRateScheduler(decay)
        for i in range(50000):
            learning_rate = lrs.get_learning_rate(i, learning_rate)
            batch_xs, batch_ys = data_batcher.next_batch(50)
            train_error = network.run_one_train_epoch(batch_xs, batch_ys, learning_rate)

            if i % 100 == 0:
                train_accuracy = network.get_accuracy(X_train, y_train)
                test_accuracy = network.append([i,train_accuracy])
                test_accuracy_hist.append([i,test_accuracy])

                print('{0:d} accuracy train test: {1:0.5f} : {2:0.5f} learning rate {3:0.8f}'.
                      format( i, train_accuracy, test_accuracy, learning_rate))

                if conv_tester.has_converged(test_accuracy):
                    print('converged after ', i, ' epochs')
                    break

        np.savetxt(model_type + "_train_accuracy_hist.csv", train_accuracy_hist, delimiter=",", fmt='%f')
        np.savetxt(model_type + "_test_accuracy_hist.csv", test_accuracy_hist, delimiter=",", fmt='%f')

        pickle.dump( network, open( model_filename, "wb" ) )

    print('Final accuracy train, test: {0:0.5f}, {1:0.5f}'.format( train_accuracy, test_accuracy))
    print_confusion_matrix(network, X_train, y_train, model_type + ' train data')
    print_confusion_matrix(network, X_test, y_test, model_type + ' test data')


import json

import requests
if __name__ == '__main__':
    post_url='https://fanyi.baidu.com/sug'
    word=input('key:')

    data={'kw':word}
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36'
    }
    #确认响应数据是json时调用.json方法
    response=requests.post(url=post_url,data=data,headers=headers).json()
    print(response)
    fp=open('./'+word+'.json','w',encoding='utf-8')
    json.dump(response,fp=fp)


import numpy as np
import tensorflow as tf

class TrajectoryOptimizer(object):
  """
  Trajectory optimizer.
  """

  def __init__(self, options={}):
    """
    Initialize a trajectory optimizer.
    """

    super(TrajectoryOptimizer, self).__init__()

    _options = {
      'x_dim': None, # dimensions of state
      'u_dim': None, # dimensions of control
      'dt':    0.05, # time step interval
      'f':     None, # dynamics model, x_next = f(x, u, dt)
      'c_t':   None, # running cost, c = c_t(x, u, dt)
      'c_f':   None, # final cost, c = c_f(x_f)
      'T':     None  # horizon
    }

    # load options
    for o in _options:
      if o in options:
        if _options[o] is None and o not in options:
          raise Error('Missing required option %s.' % (o))
        _options[o] = options[o]

    self.options = _options

    # initialize optimizer
    self.optimizer = tf.train.AdamOptimizer()

    # construct the computation graph
    print('constructing computation graph..')
    self.interfaces = self._construct_computation_graph()
    print('computation graph constructed..')

  def _construct_computation_graph(self):
    """
    Construct computation graph.
    """

    x_dim = self.options['x_dim']
    u_dim = self.options['u_dim']
    dt    = self.options['dt']
    f     = self.options['f']
    c_t   = self.options['c_t']
    c_f   = self.options['c_f']
    T     = self.options['T']

    # initialize placeholders
    x_0 = tf.placeholder(tf.float32, [x_dim])
    U_init = tf.placeholder(tf.float32, [T, u_dim])
    U = tf.Variable([ [ 0.0 for i in range(u_dim) ] for j in range(T) ], tf.float32)

    # initialize control trajectory
    initialize_ctrls = U.assign(U_init)

    # simulate forward
    J = 0
    X = [x_0]
    running_costs = []
    for t in range(T):
      x_next = f(X[t], U[t], dt)
      c_next = c_t(X[t], U[t], dt, t)
      J = J + c_next
      X.append(x_next)
      running_costs.append(c_next)
    final_cost = c_f(X[-1])
    J = J + final_cost

    # define operations
    minimize = self.optimizer.minimize(J)

    # construct interfaces
    interfaces = {
      'x_0': x_0,
      'U_init': U_init,
      'U': U,
      'X': X,
      'C_t': running_costs,
      'c_f': final_cost,
      'J': J,
      'initialize': initialize_ctrls,
      'minimize': minimize
    }

    return interfaces

  def optimize(self, x_0_val, U_init_val, epsilon=1e-4, max_epochs=100, evaluate=False, verbose=False):
    """
    Optimize the action trajectory.
    """

    interfaces = self.interfaces

    # log
    if verbose:
      print('initializing trajectory optimization..')

    # initialize session
    sess = tf.Session()

    # initialize all variables
    sess.run(tf.global_variables_initializer())

    # initialize the control trajectory
    sess.run(interfaces['initialize'], {
      interfaces['U_init']: U_init_val
    })

    # evaluate the initial trajectory
    X_init_val = None
    if evaluate:
      X_init_val = []
      for t, x in enumerate(interfaces['X']):
        x_val = sess.run(x, {
          interfaces['x_0']: x_0_val
        })
        X_init_val.append(x_val)

    J_init_val = sess.run(interfaces['J'], {
      interfaces['x_0']: x_0_val
    })

    # log
    if verbose:
      print('x_0 = %s' % (str(np.array(x_0_val).reshape(self.options['x_dim']))))

      for t in range(self.options['T']):
        print('U_init[%d] = %s' % (t, str(np.array(U_init_val[t]).reshape(self.options['u_dim']))))

    if evaluate and verbose:
      for t in range(self.options['T'] + 1):
        print('X_init[%d] = %s' % (t, str(np.array(X_init_val[t]).reshape(self.options['x_dim']))))

    if verbose:
      print('J_init = %.4f' % (J_init_val))

    # log
    if verbose:
      print('optimizing trajectory..')

    # run epochs
    last_J_val = None
    for ep in range(max_epochs):
      # optimize the action trajectory
      sess.run(interfaces['minimize'], {
        interfaces['x_0']: x_0_val
      })

      # evaluate
      J_val = sess.run(interfaces['J'], {
        interfaces['x_0']: x_0_val
      })

      # log
      if verbose:
        print('[%d/%d] J_val=%.4f' % (ep + 1, max_epochs, J_val))

      # check optimization termination conditions
      if last_J_val is not None:
        if abs(J_val - last_J_val) < epsilon:
          break

      # update last J value
      last_J_val = J_val

    # log
    if verbose:
      print('trajectory optimization finished..')

    # evaluate the optimized trajectory
    U_opt_val = sess.run(interfaces['U'], {
      interfaces['x_0']: x_0_val
    })

    X_opt_val = None
    if evaluate:
      X_opt_val = []
      for t, x in enumerate(interfaces['X']):
        x_val = sess.run(x, {
          interfaces['x_0']: x_0_val
        })
        X_opt_val.append(x_val)

    J_opt_val = sess.run(interfaces['J'], {
      interfaces['x_0']: x_0_val
    })

    # log
    if verbose:
      print('x_0 = %s' % (str(np.array(x_0_val).reshape(self.options['x_dim']))))

      for t in range(self.options['T']):
        print('U_opt[%d] = %s' % (t, str(np.array(U_opt_val[t]).reshape(self.options['u_dim']))))

    if evaluate and verbose:
      for t in range(self.options['T'] + 1):
        print('X_opt[%d] = %s' % (t, str(np.array(X_opt_val[t]).reshape(self.options['x_dim']))))

    if verbose:
      print('J_opt = %.4f' % (J_opt_val))

    # construct result
    res = {
      'x_0': x_0_val,
      'U_init': U_init_val,
      'X_init': X_init_val,
      'J_init': J_init_val,
      'U_opt': U_opt_val,
      'X_opt': X_opt_val,
      'J_opt': J_opt_val
    }

    return res

def _test():
  x_dim = 4
  u_dim = 2
  dt = 1.0
  T = 10

  def f(x, u, dt):
    A = [[1, 0, dt,  0],
         [0, 1,  0, dt],
         [0, 0,  1,  0],
         [0, 0,  0,  1]]
    B = [[ 0,  0],
         [ 0,  0],
         [dt,  0],
         [ 0, dt]]

    A = tf.constant(A, tf.float32)
    B = tf.constant(B, tf.float32)

    x_next = tf.reshape(tf.matmul(A, tf.reshape(x, [x_dim, 1])) + tf.matmul(B, tf.reshape(u, [u_dim, 1])), [x_dim])

    return x_next

  def c_t(x, u, dt, t):
    u_lambda = 1e-6
    c = u_lambda * tf.reduce_sum(u * u) * dt
    return c

  def c_f_loc_speed(x_f):
    x_g = tf.constant(np.array([5., 5., 0., 0.]), tf.float32)
    c = tf.reduce_sum((x_g - x_f) * (x_g - x_f))
    return c

  def c_f_loc(x_f):
    loc_diff = tf.convert_to_tensor([x_f[0] - 5.0, x_f[1] - 5.0])
    c = tf.reduce_sum(loc_diff * loc_diff)
    return c

  # initialize trajectory optimizer
  options = {
    'x_dim': x_dim,
    'u_dim': u_dim,
    'dt':    dt,
    'f':     f,
    'c_t':   c_t,
    'c_f':   c_f_loc_speed,
    'T':     T
  }

  trajopt = TrajectoryOptimizer(options)

  # optimize trajectory
  x_0 = np.array([0, 0, 0, 0])
  U_init = [np.array([0, 0]) for t in range(T)]

  trajopt.optimize(x_0, U_init, epsilon=1e-4, max_epochs=1000, evaluate=True, verbose=True)

if __name__ == '__main__':
  _test()


# Data-guided multi-granularity selector for attribute reduction
# Jiang Z, Dou H, Song J, et al. Data-guided multi-granularity selector for attribute reduction[J]. Applied Intelligence, 2021, 51: 876-888.

import math
import time
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from scipy.spatial.distance import squareform, pdist

# 现在要做的事情是在给定一个半径下得到全属性集合下各个样本的邻域
# 参数情况: 固定的delta = 0.2, w = 0.1, s = 10
# 写一个函数 可以计算出数据集中每两个样本 在某一个属性子集上的距离
def generateDisMatrixUnderAttrSet(X: np.ndarray, attrSet: list[int]) -> np.array:
    '''
    :param X: 传入的原始数据集
    :param cols: 纳入计算距离的那些列
    :return: 对象之间的距离矩阵
    '''
    distMatrix = squareform(pdist(X[:, attrSet], metric='euclidean'))
    return distMatrix

def generateNeighbor(X:np.ndarray, attrSet:list[int], radius: float) -> dict:
    '''
    :param disMatrix: 距离矩阵
    :param radius:  用来生成邻域的半径
    :return:  字典 键:对象索引 值:该对象所生成邻域中的对象
    '''
    disMatrix = generateDisMatrixUnderAttrSet(X, attrSet) # 生成该属性集之下的各个样本之间的距离矩阵
    neighbors = dict()
    n, _ = disMatrix.shape
    for i in range(n):
        # neighbors[i] = set(np.where(disMatrix[i, :] < radius)[0])
        neighbors[i] = set(np.where(disMatrix[i, :] < radius)[0]) - {i}
    return neighbors

def calDiffBetTwoGran(neighbors1:dict, neighbors2:dict)->float:
    sampleNum = len(neighbors1)
    diffVal = 0
    for i in range(sampleNum):
        s1 = neighbors1[i]
        s2 = neighbors2[i]
        symDif = s1.symmetric_difference(s2)
        diffVal += len(symDif)/sampleNum
    return diffVal/sampleNum

def dataGuidedParameterSelector(X:np.ndarray, s:int=10, w:int=0.01):
    '''
    :param X: 数据集的条件属性部分
    :param s: 默认要选出10个半径
    :param w: 差异值设置为0.1
    :return: 返回选出来具有差异的一些半径
    '''
    selectedRadius = []
    prePartRadius = [0.2-0.02*i for i in range(1, 10)]
    postPartRadius = [0.2+0.02*i for i in range(1, 11)]
    sampleNum, attrNum = X.shape
    AT = list(range(attrNum))
    neighborsWithDelta = generateNeighbor(X, AT, 0.2)

    preDiffAndRadArr = []
    for rad in prePartRadius:
        neighborsCompare = generateNeighbor(X, AT, rad)
        dif = calDiffBetTwoGran(neighborsCompare, neighborsWithDelta)
        preDiffAndRadArr.append([dif, rad])
    preDiffAndRadArr.sort(key=lambda pair: pair[0], reverse=True)
    preSelectedRadius = [e[1] for e in preDiffAndRadArr[2:7]]

    postDiffAndRadArr = []
    for rad in postPartRadius:
        neighborsCompare = generateNeighbor(X, AT, rad)
        dif = calDiffBetTwoGran(neighborsCompare, neighborsWithDelta)
        postDiffAndRadArr.append([dif, rad])
    postDiffAndRadArr.sort(key=lambda pair: pair[0], reverse=True)
    postSelectedRadius = [e[1] for e in postDiffAndRadArr[2:7]]
    selectedRadius = preSelectedRadius+postSelectedRadius

    return selectedRadius

def generateDecisionClasses(Y:np.ndarray):
    '''
    :param Y: 数据集的决策属性部分
    :return: 各个决策类的集合
    '''
    decClasses = dict()
    decValues = np.unique(Y)
    for decValue in decValues:
        decClasses[decValue] = set(np.where(Y==decValue)[0])
    return decClasses

def getPos(decisionClasses: dict, neighbors: dict) -> list:
    '''
    :param decisionClasses: 决策类
    :param neighbors:  生成的邻域
    :return: D的B正域
    '''
    posBD = []
    for decisionClass in decisionClasses.values():
        for k in neighbors.keys():
            neighbor = neighbors[k]
            if neighbor.issubset(decisionClass):
                # print("对象{} 邻域{},被包含在决策类{}中\n".format(k, neighbor, decisionClass))
                posBD.append(k)
        # break
    # 去除重复项
    posBD = list(set(posBD))
    # print("D的B正域为{},其中元素个数为{}".format(posBD, len(posBD)))
    # print("条件属性集相对于决策属性集的重要度为:",len(posBD)/conditionData.shape[0])
    return posBD

def getAttrSetScoreDataGuidedSelector(X:np.ndarray, selectedRadius:list[float], decClasses:dict, attrSet:list[int]):
        score = 0
        for rad in selectedRadius:
            neighbors = generateNeighbor(X, attrSet, rad)
            tmpScore = len(getPos(decClasses, neighbors))
            score += tmpScore
        score = score / len(selectedRadius)
        return score

def reductionUseDataGuidedSelector(X:np.ndarray, Y:np.ndarray):
    sampleNum, attrNum = X.shape
    decClasses = generateDecisionClasses(Y)
    selectedRadius = dataGuidedParameterSelector(X)
    AT = set(range(attrNum))
    A = set() # 用于存储最后的约简结果
    preScore = -100

    fullAttrSetScore = getAttrSetScoreDataGuidedSelector(X, selectedRadius, decClasses, list(AT))
    # print("全属性集的得分为:{}".format(fullAttrSetScore))

    start_time = time.time()  # 程序开始时间
    while True:
        curScore = -100
        selectedAttr = 0
        for a in AT - A:
            tmpScore = getAttrSetScoreDataGuidedSelector(X, selectedRadius, decClasses, list(A | set([a])))
            if tmpScore > curScore:
                curScore = tmpScore
                selectedAttr = a
        A = A | set([selectedAttr])
        if curScore >= fullAttrSetScore:
            break

    # 删除属性的过程 如果从选出的属性集A中去除属性之后仍然满足约束就可以将一个属性从A中删除
    while True:
        nextA = A
        for a in A:
            tmpScore = getAttrSetScoreDataGuidedSelector(X, selectedRadius, decClasses, list(A - set([a])))
            if tmpScore >= fullAttrSetScore:
                nextA = nextA - set([a])
                break  # 每次只删除一个属性
        if len(A) == 1 or nextA == A:
            break
        A = nextA

    end_time = time.time()  # 程序结束时间
    run_time_sec = end_time - start_time  # 程序的运行时间，单位为秒

    return A, curScore/sampleNum, run_time_sec

if __name__ == "__main__":
    path = '../DataSet_TEST/ori/{}.csv'.format("movement")
    data = np.loadtxt(path, delimiter=",", skiprows=1)

    X = data[:, :-1]
    X = MinMaxScaler().fit_transform(X)  # 归一化取值均归为0-1之间
    Y = data[:, -1]

    red = reductionUseDataGuidedSelector(X, Y)
    print(red)

    # sampleNum, attrNum = X.shape
    # selectedRadius = [0.12+0.02*i for i in range(10)]
    # decClasses = generateDecisionClasses(Y)
    #
    # AT = set(range(attrNum))
    # score1 = getAttrSetScoreDataGuidedSelector(X, selectedRadius, decClasses, list(AT))
    # print(score1)
    #
    # score2 = getAttrSetScoreDataGuidedSelector(X, selectedRadius, decClasses, list(AT-{3}))
    # print(score2)


"""library_system URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path,include
from bookmanagement.views import *

urlpatterns = [
    path('admin/', admin.site.urls),
    path('home/', homepage, name = "homepage"),
    path('', homepage, name = "homepage"),
    path('show-all-books/', show_all_books, name = "show_all_books"),
    path('show_active_books/', show_active_books, name = "show_active_books"),
    path('show_inactive_books/', show_inactive_books, name = "show_inactive_books"),
    path('edit/<int:id>/', edit_data, name = "edit_data"),
    path('delete/<int:id>/', delete_book, name = "delete_book"),
    path('soft_delete/<int:id>/', soft_delete_book, name = "soft_delete_or_reco_book"),

]
    # path('__debug__/', include('debug_toolbar.urls')),

# show_active_books


# -*- coding: utf-8 -*-
"""
Code to compile Tulsa football hit data into a DataFrame
"""
from __future__ import print_function, division
import pandas as pd
import numpy as np
import toolz
import copy

#------------#
# FUNCTIONS  #
#------------#
@toolz.curry
def during_practice(practice, hit_date):
    """
    during_practice takes a hit_date that is a datetime object and a pandas
    dataframe called practices. practices should have a start and end time for
    the practice session that occurs on a given day.
    """
    # Find day of hit and format as a string
    hit_day = hit_date.date().strftime('%Y-%m-%d')
    # Find the start and end time of the practice of the hit day
    start_time = practice[hit_day].loc[practice.period == 'start'].date
    end_time = practice[hit_day].loc[practice.period == 'end'].date
    if start_time.empty or end_time.empty:
        return False
    # determine if the hit occurred during practice
    return np.logical_and(start_time <= hit_date, hit_date <= end_time).values[0]

@toolz.curry
def during_event(practices, hit):
    return during_practice(practices[hit.ptype], hit.date)

@toolz.curry
def before_break(practice, hit_date):
    # Find day of hit and format as a string
    hit_day = hit_date.date().strftime('%Y-%m-%d')
    # Find the start time of the break period
    break_time = practice[hit_day].loc[practice.period == 'break'].date
    if break_time.empty:
        # If the practice session didn't have a break, by definition all events
        # for that session occurred before the break.
        return True
    # determine if the hit occurred during practice
    return (hit_date <= break_time).values[0]

def read_hit_data(filename, skiprows=0, sheet='FINAL- bad hits and 1 out',
                  date_column={'date': [1,2]}):
    df = pd.read_excel(filename, skiprows=skiprows, sheetname=sheet,
                       parse_dates=date_column)
    df.columns = [col.strip() for col in df.columns]
    df.set_index('date', inplace=True)
    return df

def read_attendance_data(filename, date_column='date'):
    df = pd.read_excel(filename)
    df.columns = [str(col).strip() for col in df.columns]
    df.set_index(date_column, inplace=True)
    return df

def read_practice_data(filename, skiprows=None, date_column='date'):
    df = pd.read_excel(filename, skiprows=skiprows, sheetname=None)
    df['offensive']['day'] = df['offensive']['day'].ffill()
    df['offensive']['name'] = df['offensive']['name'].ffill()
    df['offensive']['type'] = df['offensive']['type'].ffill()
    df['offensive'].index = df['offensive'][date_column]
    df['defensive']['day'] = df['defensive']['day'].ffill()
    df['defensive']['name'] = df['defensive']['name'].ffill()
    df['defensive']['type'] = df['defensive']['type'].ffill()
    df['defensive'].index = df['defensive'][date_column]
    return df

def read_participation_data(filename, skiprows=None, date_column='date'):
    df = pd.read_excel(filename, skiprows=skiprows, sheetname=None)
    df['offensive']['day'] = df['offensive']['day'].ffill()
    df['offensive']['name'] = df['offensive']['name'].ffill()
    df['offensive'].set_index(date_column, inplace=True)
    df['defensive']['day'] = df['defensive']['day'].ffill()
    df['defensive']['name'] = df['defensive']['name'].ffill()
    df['defensive'].set_index(date_column, inplace=True)
    return df

def read_serial_num_to_position(filename, index_col='number'):
    return pd.read_excel(filename).set_index(index_col)

def read_player_types(filename, index_col='type'):
    return pd.read_excel(filename).set_index(index_col)

def read_event_types(filename, index_col='date'):
    return pd.read_excel(filename).set_index(index_col)

def read_game_durations(filename, index_col=0):
    return pd.read_excel(filename, index_col=index_col, header=[0,1])

@toolz.curry
def get_player_type(position_type_df, player_code):
    if not player_code in position_type_df.columns:
        print('Player code not in position type table. Call Seth.')
        raise KeyError
    try:
        return position_type_df[position_type_df[player_code]==True].index[0]
    except:
        print('Something went wrong while determining player type (offensive or defensive). Call Seth.')
        raise RuntimeError

@toolz.curry
def get_player_position(positions, player_number):
    return positions.loc[player_number].position

@toolz.curry
def get_player_position_code(positions, player_number):
    return positions.loc[player_number]['code']

@toolz.curry
def get_practice(practices, ptype):
    return practices[ptype]

def get_closest_activity_idx(practice, hit_date):
    # Find the index of the practice activity during which the hit occurred
    return practice.index.get_loc(hit_date, method='pad')

def get_activity_idx(practice, hit_date):
    # Return the activity during which the hit occurred
    activity_idx = get_closest_activity_idx(practice, hit_date)
    act_row = practice.iloc[activity_idx]
    ## Hits shouldn't occur during breaks or the all up activities. Assign hits
    ## during these activities to the activity immediately prior.
    if act_row.period == 'break' or act_row.period == 'all up':
        activity_idx -= 1
        ## NOTE: I could do the above recursively, which may be more robust but
        ##       not strictly necessary since I'm basically looking for the
        ##       above result.
        #return get_activity(practice, player_code, hit_date-1)
    return activity_idx

@toolz.curry
def get_activity(practice, pcode, hit_date):
    # Return the activity during which the hit occurred
    activity_idx = get_activity_idx(practice, hit_date)
    return practice.loc[practice.index[activity_idx], pcode]

def get_parti(practice, pcode, hit_date):
    # Return the pcodes of the players that were participating in the event
    # in which the hit occurred
    activity_idx = get_activity_idx(practice, hit_date)
    activity = get_activity(practice, pcode, hit_date)
    return list(practice.columns[practice.iloc[activity_idx] == activity])

@toolz.curry
def get_practice_name(practice, hit_date):
    activity_idx = get_activity_idx(practice, hit_date)
    return practice.loc[practice.index[activity_idx], 'name']

@toolz.curry
def get_event_type(event_df, hit_date):
    return event_df.loc[hit_date.date()].type

def get_num_players(method='by_position', data={}):
    att_df = data['attendance_df']
    pcode = data['pcode']
    hit_date = data['hit_date']
    # Look up number of players
    if method == 'by_position':
        # How many people were playing this position on the hit date?
        return att_df.loc[hit_date.date()][pcode]
    elif method == 'total':
        # How many total people were participating on the hit date?
        att_df = data['df']
        return att_df.loc[hit_date.date()]['total']
    elif method == 'by_activity':
        # How many people were participating in the activity on the hit date?
        # What player codes participated in the activity?
        parti_codes = get_parti(data['practice_df'], pcode, hit_date)
        return att_df.loc[hit_date.date()][parti_codes].sum()
    else:
        raise NotImplementedError

def get_duration(data={}):
    if data['event_type'] == 'game':
        hit_date = data['hit_date']
        game_dur_df = data['game_dur_df']
        ptype = data['ptype']
        activity = data['activity']
        date_mask = [dd.date() == hit_date.date() for dd in game_dur_df.index]
        return game_dur_df.loc[date_mask, (ptype, activity)].iloc[0]
    else:
        durs = data['durations']
        event = data['event']
        prebrk = data['prebreak']
        return durs[np.logical_and(durs.prebreak == prebrk,
                                   durs.name == event)].length.values[0]

def normalize_hits(hit, player_method=None, time_method=None, data={}):
    hit = copy.deepcopy(hit).astype(np.float64)
    if player_method:
        hit = norm_by_players(hit, method=player_method, data=data)
    if time_method:
        hit = norm_by_time(hit, data=data)
    return hit

def norm_by_players(hit, **kwargs):
    num_players = get_num_players(**kwargs)
    if num_players == 0:
        #hit['sum 1'] = np.NAN
        hit['sum 2'] = np.NAN
        hit['sum 3'] = np.NAN
        hit['sum 4'] = np.NAN
        hit['sum 5'] = np.NAN
    else:
        #hit['sum 1'] /= num_players
        hit['sum 2'] /= num_players
        hit['sum 3'] /= num_players
        hit['sum 4'] /= num_players
        hit['sum 5'] /= num_players
    return hit

def norm_by_time(hit, **kwargs):
    duration = get_duration(**kwargs)
    if duration == 0:
        #hit['sum 1'] = np.NAN
        hit['sum 2'] = np.NAN
        hit['sum 3'] = np.NAN
        hit['sum 4'] = np.NAN
        hit['sum 5'] = np.NAN
    else:
        #hit['sum 1'] /= duration
        hit['sum 2'] /= duration
        hit['sum 3'] /= duration
        hit['sum 4'] /= duration
        hit['sum 5'] /= duration
    return hit

def get_hits_out_of_event(input_files):
    practices = read_practice_data(input_files['practices'])
    psn_to_pcode = read_serial_num_to_position(input_files['psn_to_pcode'])
    pcode_to_ptype = read_player_types(input_files['pcode_to_ptype'])

    de = during_event(practices)
    psn_to_ptype = toolz.compose(get_player_type(pcode_to_ptype),
                                 get_player_position_code(psn_to_pcode))

    hits = (read_hit_data(input_files['hits'])
             .reset_index()
             .assign(ptype=lambda x: x.Number.apply(psn_to_ptype))
             .assign(in_event=lambda x: x.apply(de, axis=1)))
    hits = hits.drop(hits.columns[range(2, 22)], axis=1)
    hits[hits.in_event == False].to_csv('hits_not_during_events.csv')

def get_practice_times_pre_post(practices):
    # Just select the offensive practices - they should be the same
    for p in practices:
        practice = practices[p]
        practice['day'] = [pd.to_datetime(x).date() for x in practice['date']]
        practice['prebreak'] = ['pre' if before_break(practice, x['date'])
                                else 'post'
                                for idx, x in practice.iterrows()]
        (practice.groupby(['name', 'prebreak', 'day'], as_index=False)
            .sum()
            .sort_values(by=['day', 'name', 'prebreak'])
            .to_csv('{}_practice_lengths.csv'.format(p), index=False))

def compile_data(input_files, norm_methods=(None, None)):
    hits = read_hit_data(input_files['hits'])
    practices = read_practice_data(input_files['practices'])
    att = read_attendance_data(input_files['attendance'])
    #game_dur = read_game_durations(input_files['game_duration'])
    #duration_data = pd.read_csv('practice_lengths.csv')
    #parti = read_participation_data(input_files['participation'])
    event_df = read_event_types(input_files['event_types'])
    psn_to_pcode = read_serial_num_to_position(input_files['psn_to_pcode'])
    pcode_to_ptype = read_player_types(input_files['pcode_to_ptype'])

    results = []
    for hit_date, hit in hits.iterrows():
        # Look up the hit player's position code
        pcode = get_player_position_code(psn_to_pcode, hit.Number)
        # Determine the player type
        player_type = get_player_type(pcode_to_ptype, pcode)
        # Load the correct set of practices
        practice = practices[player_type]
        # Filter the hit out if it doesn't occur during a practice period
        if not during_practice(practice, hit_date):
            continue
        # Look up the hit player's position
        player_position = get_player_position(psn_to_pcode, hit.Number)
        # Look up the event
        event = get_practice_name(practice, hit_date)
        # Look up the event type
        event_type = get_event_type(event_df, hit_date)
        # Look up the activity
        activity = get_activity(practice, pcode, hit_date)
        # Determine pre/post break
        prebreak = 'pre' if before_break(practice, hit_date) else 'post'
        # Normalize the hit data
        data = {'attendance_df': att,
                'practice_df': practice,
                #'game_dur_df': game_dur,
                'pcode': pcode,
                'ptype': player_type,
                'event': event,
                'event_type': event_type,
                'activity': activity,
                'hit': hit,
                'hit_date': hit_date,
                #'durations': duration_data,
                'prebreak': prebreak}
        hit = normalize_hits(hit, *norm_methods, data)

        print('Processing hit...\n'+
              '\tHit Date: {}\n'.format(hit_date))

        # Add results to list
        results.append({'event': event,
                        'type': event_type,
                        'date': hit_date,
                        'activity': activity,
                        'before_break': prebreak,
                        'player': player_position,
                        'pcode': pcode,
                        'ptype': player_type,
                        #'h1': hit['sum 1'],
                        'h2': hit['sum 2'],
                        'h3': hit['sum 3'],
                        'h4': hit['sum 4'],
                        'h5': hit['sum 5']})
    return (pd.DataFrame(results)
              .assign(day=lambda x: [pd.to_datetime(y).date() for y in x.date])
              .sort_values(by=['date']))

#--------------#
# MAIN PROGRAM #
#--------------#
if __name__ == '__main__':
    input_files = {'hits': 'Tulsa_HIE_5 minute increments final- corrected 7-10.xlsx',
                   'practices': 'practices.xlsx',
                   'attendance': 'attendance.xlsx',
                   'event_types': 'event_type.xlsx',
                    #'game_duration': 'minutes_per_game.xlsx',
                   'pcode_to_ptype': 'player_code_to_type.xlsx',
                   'psn_to_pcode': 'serial_number_to_position.xlsx'}

    compile_data(input_files, norm_methods=(None, None)).to_csv('compiled_raw.csv')
    #compile_data(input_files, norm_methods=('by_position', None)).to_csv('compiled_by_position.csv')
    #compile_data(input_files, norm_methods=('by_activity', None)).to_csv('compiled_by_activity.csv')
    #compile_data(input_files, norm_methods=('by_activity', 'by_time')).to_csv('compiled_by_time_activity_test.csv')


import logging
logger = logging.getLogger('server_logger')

class ListenPort:
    def __set__(self, instance, value):
        if value < 1024 or value > 65535:
            logger.critical('Попытка запуска сервера с неверным номером порта. '
                            'Номер порта должен быть из диапазона [1024;65535]')
            exit(1)
        instance.__dict__[self.name] = value

    def __set_name__(self, owner, name):
        self.name = name