import os
import time
import math
import json
import codecs
import argparse          ##python自带的命令行参数解析包
import numpy as np
from tqdm import tqdm
from numpy import finfo

from sklearn.metrics import accuracy_score

from transformers import BertTokenizer
from pytorch_pretrained_bert import BertModel

import torch
import torch.nn as nn
import torch.distributed as dist
from torch.autograd import Variable
from torch.nn import functional as F
from torch.utils.data import DataLoader
from torch.utils.data.distributed import DistributedSampler
from distributed import apply_gradient_allreduce

import parse_nk
from models import G2PTransformerMask, poly_tonesandhi, Cascaded_Tacotron2
from data_utils import TextMelLoader, TextMelCollate, G2PDatasetMask, get_dataloader, polyTTS_get_dataloader
from loss_function import Tacotron2Loss
from logger import Tacotron2Logger
from hparams import create_hparams

def reduce_tensor(tensor, n_gpus):   ##？
    rt = tensor.clone()   ## 返回一个张量的副本，其与原张量的尺寸和数据类型相同
    dist.all_reduce(rt, op=dist.reduce_op.SUM)  ## 在所有机器上减少张量数据，通过获得最终的结果。在调用之后张量在所有过程中都是按位相同的。
    rt /= n_gpus   ## /=是除法赋值运算符  rt=rt/n_gpus
    return rt

def init_distributed(hparams, n_gpus, rank, group_name):
    assert torch.cuda.is_available(), "Distributed mode requires CUDA."
    ## cuda是否可用，cuda(compute unified device architecture)是显卡厂商NVIDIA推出的运算平台
    print("Initializing Distributed")

    # Set cuda device so everything is done on the right GPU.
    torch.cuda.set_device(rank % torch.cuda.device_count())
    ## torhc.cuda.set_device(device)设置当前设备。不鼓励使用此函数来设置，在大多数情况下，最好使用CUDA_VISIBLE_DEVICES环境变量。参数device(int)-所选设备，如果此参数为负，则此函数是无效操作。
    ## torch.cuda.device_count() 返回可得到的GPU数量

    # Initialize distributed communication
    dist.init_process_group(
        backend=hparams.dist_backend, init_method=hparams.dist_url,
        world_size=n_gpus, rank=rank, group_name=group_name)
    ## pytorch分布式训练
    ## backend str/Backend 是通信所用的后端，可以是'nccl''gloo'或者是一个torch.distributed.Backend类（Backend.GLOO）
    ## init_method str 这个URL指定了如何初始化互相通信的进程
    ## world_size init 执行训练的所有进程数
    ## rank int 这个进程的编号，也是其优先级
    ## group_name str 进程所在group的name

    print("Done initializing distributed")

def prepare_dataloaders(hparams):
    # Get data, data loaders and collate function ready
    ## if not 用法：   if true才执行  （即 if not false）
    if not hparams.load_mel_from_disk:
        trainset = TextMelLoader(hparams.training_files, hparams.polyphone_dict_files, hparams.mask_dict_files, hparams)
        valset = TextMelLoader(hparams.validation_files, hparams.polyphone_dict_files, hparams.mask_dict_files, hparams)
    else:
        trainset = TextMelLoader(hparams.mel_training_files, hparams.polyphone_dict_files, hparams.mask_dict_files, hparams)
        valset = TextMelLoader(hparams.mel_validation_files, hparams.polyphone_dict_files, hparams.mask_dict_files, hparams)
    collate_fn = TextMelCollate(hparams.n_frames_per_step, hparams.num_classes)

    if hparams.distributed_run:   ##False
        train_sampler = DistributedSampler(trainset)
        ## 在多机多卡情况下分布式训练数据的读取，不同的卡读到的数据应该是不同的，利用sampler确保dataloader只会load到整个数据集的一个特定子集
        ## 它为每个子进程划分出一部分数据集，以避免不同进程之间的数据重复。
        shuffle = False
    else:
        train_sampler = None
        shuffle = True

    ## 定义一个可迭代的数据加载器
    train_loader = DataLoader(trainset, num_workers=0, shuffle=shuffle,
                              sampler=train_sampler,
                              batch_size=hparams.batch_size, pin_memory=False,
                              drop_last=True, collate_fn=collate_fn)
    ## dataset（Dataset类，决定数据从哪里读取及如何读取）  batch_size(每个batch的大小，批大小)  shuffle(是否进行shuffle操作，每个epoch是否乱序)
    ## num_workers(加载数据时使用几个子进程)  drop_last(当样本数不能被batchsize整除时，是否舍弃最后一批数据)
    return train_loader, valset, collate_fn

def prepare_directories_and_logger(output_directory, log_directory, rank):
    if rank == 0:
        if not os.path.isdir(output_directory):
            os.makedirs(output_directory)
            os.chmod(output_directory, 0o775)
        logger = Tacotron2Logger(os.path.join(output_directory, log_directory))
    else:
        logger = None
    return logger

def load_model(hparams):
    model = Cascaded_Tacotron2(hparams).cuda()## 参数是hparams,因为继承了nn.module,所以有.cuda()
    if hparams.fp16_run:   ## False
        model.decoder.attention_layer.score_mask_value = finfo('float16').min ##？

    if hparams.distributed_run:   ## False
        model = apply_gradient_allreduce(model)

    return model

def warm_start_model(checkpoint_path, model, ignore_layers):
    assert os.path.isfile(checkpoint_path)
    print("Warm starting model from checkpoint '{}'".format(checkpoint_path))
    checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
    model_dict = checkpoint_dict['state_dict']
    if len(ignore_layers) > 0:
        model_dict = {k: v for k, v in model_dict.items()
                      if k not in ignore_layers}
        dummy_dict = model.state_dict()
        dummy_dict.update(model_dict)
        model_dict = dummy_dict
    model.load_state_dict(model_dict)
    return model

def load_checkpoint(checkpoint_path, model, optimizer):
    assert os.path.isfile(checkpoint_path)
    print("Loading checkpoint '{}'".format(checkpoint_path))
    checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
    model.load_state_dict(checkpoint_dict['state_dict'])
    optimizer.load_state_dict(checkpoint_dict['optimizer'])
    learning_rate = checkpoint_dict['learning_rate']
    iteration = checkpoint_dict['iteration']
    print("Loaded checkpoint '{}' from iteration {}" .format(
        checkpoint_path, iteration))
    return model, optimizer, learning_rate, iteration

def save_checkpoint(model, optimizer, learning_rate, iteration, filepath):
    print("Saving model and optimizer state at iteration {} to {}".format(
        iteration, filepath))
    torch.save({'iteration': iteration,
                'state_dict': model.state_dict(),
                'optimizer': optimizer.state_dict(),
                'learning_rate': learning_rate}, filepath)

def validate(model, criterion, valset, iteration, batch_size, n_gpus,
             collate_fn, logger, distributed_run, rank):
    """Handles all the validation scoring and printing"""
    model.eval()
    with torch.no_grad():
        val_sampler = DistributedSampler(valset) if distributed_run else None
        val_loader = DataLoader(valset, sampler=val_sampler, num_workers=0,
                                shuffle=False, batch_size=batch_size,
                                pin_memory=False, collate_fn=collate_fn)

        val_loss = 0.0
        val_mel_loss = 0.0
        val_gate_loss = 0.0
        val_select_loss = 0.0
        for i, batch in enumerate(val_loader):
            x, y = model.parse_batch(batch)
            y_pred = model(x)

            # original_words = x[3]
            # # print('CHECK original_words IN validate:', original_words)

            # _, _, select_target = y
            # select_target = np.array(select_target.cpu())
            # # print('CHECK select_target IN validate:', select_target)
            # np.savetxt('select_target.txt',select_target)

            # _, _, _, _, select_pred = y_pred
            # select_pred = np.array(select_pred.cpu())
            # select_pred = np.argmax(select_pred, axis=2)
            # # print('CHECK select_pred IN validate:', select_pred)
            # np.savetxt('select_pred.txt',select_pred)

            # mask_padded_to_show = np.array(mask_padded.cpu())
            # mask_padded_to_show = np.sum(mask_padded_to_show, axis=2)
            # # print('CHECK mask_padded_to_show IN validate:', mask_padded_to_show)
            # np.savetxt('select_mask.txt',mask_padded_to_show)

            mask_padded = x[3]
            loss, mel_loss, gate_loss, select_loss = criterion(y_pred, y, mask_padded)
            if distributed_run:
                reduced_val_loss = reduce_tensor(loss.data, n_gpus).item()
                reduced_val_mel_loss = reduce_tensor(mel_loss.data, n_gpus).item()
                reduced_val_gate_loss = reduce_tensor(gate_loss.data, n_gpus).item()
                reduced_val_select_loss = reduce_tensor(select_loss.data, n_gpus).item()
            else:
                reduced_val_loss = loss.item()
                reduced_val_mel_loss = mel_loss.item()
                reduced_val_gate_loss = gate_loss.item()
                reduced_val_select_loss = select_loss.item()
            val_loss += reduced_val_loss
            val_mel_loss += reduced_val_mel_loss
            val_gate_loss += reduced_val_gate_loss
            val_select_loss += reduced_val_select_loss
        val_loss = val_loss / (i + 1)
        val_mel_loss = val_mel_loss / (i + 1)
        val_gate_loss = val_gate_loss / (i + 1)
        val_select_loss = val_select_loss / (i + 1)

    model.train()
    if rank == 0:
        print("Validation loss {}: {:9f}  ".format(iteration, val_loss))
        logger.log_validation(val_loss, val_mel_loss, val_gate_loss, val_select_loss, model, y, y_pred, iteration)

def train_tts(output_directory, log_directory, checkpoint_path, warm_start, n_gpus,
          rank, group_name, hparams):
    """Training and validation logging results to tensorboard and stdout

    Params
    ------
    output_directory (string): directory to save checkpoints
    log_directory (string) directory to save tensorboard logs
    checkpoint_path(string): checkpoint path
    n_gpus (int): number of gpus
    rank (int): rank of current gpu
    hparams (object): comma separated list of "name=value" pairs.
    """
    if hparams.distributed_run:
        init_distributed(hparams, n_gpus, rank, group_name)

    torch.manual_seed(hparams.seed)  ##设置(CPU)生成随机数的种子，在每次重新运行程序时，同样的随机数生成代码得到的是同样的结果。
    torch.cuda.manual_seed(hparams.seed)## 设置当前GPU的随机数生成种子  torch.cuda.manual_seed_all(seed)设置所有GPU的随机数生成种子
    ## 手动设置种子一般可用于固定随机初始化的权重值，这样就可以让每次重新从头训练网络时的权重的初始值虽然是随机生成的但却是固定的。

    model = load_model(hparams)
    learning_rate = hparams.learning_rate
    # optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate,
    #                              weight_decay=hparams.weight_decay)

    for name, param in model.named_parameters():
        # frozen except tts
        # if name.split('.')[0] == 'poly_phoneme_classifier':
        #     param.requires_grad = False

        # frozen poly module except tone sandhi & tts
        # if name.split('.')[0] == 'poly_phoneme_classifier':
        #     if name.split('.')[1] != 'linear_pre' and name.split('.')[1] != 'conv_layers' and name.split('.')[1] != 'linear_aft':
        #         param.requires_grad = False

        # frozen except structure CNN & tonesandhi & tts
        if name.split('.')[0] == 'poly_phoneme_classifier':
            if name.split('.')[1] == 'g2ptransformermask':
                if name.split('.')[2] != 'structure_cnn_tts':
                    param.requires_grad = False
            elif name.split('.')[1] != 'linear_pre' and name.split('.')[1] != 'conv_layers' and name.split('.')[1] != 'linear_aft':
                param.requires_grad = False
            # else:
            #    param.requires_grad = False

    training_parameters_list = [p for p in model.parameters() if p.requires_grad]
    optimizer = torch.optim.Adam(training_parameters_list, lr=learning_rate,
                                 weight_decay=hparams.weight_decay)

    if hparams.fp16_run:
        from apex import amp
        model, optimizer = amp.initialize(
            model, optimizer, opt_level='O2')
    ## apex是一款由Nvidia开发的基于PyTorch的混合精度训练加速神奇，用短短几行代码就能实现不同程度的混合精度加速，训练时间直接缩小一半。
    ## fp16:半精度浮点数，是一种计算机使用的二进制浮点数数据类型，使用2字节（16位）存储。
    ## fp16优点：减少显存占用；加快训练和推断的计算；张量核心的普及。缺点：量化误差。

    if hparams.distributed_run:
        model = apply_gradient_allreduce(model)

    criterion = Tacotron2Loss()

    logger = prepare_directories_and_logger(
        output_directory, log_directory, rank)

    train_loader, valset, collate_fn = prepare_dataloaders(hparams)

    # Load checkpoint if one exists
    iteration = 0
    epoch_offset = 0
    if checkpoint_path is not None:
        if warm_start:
            model = warm_start_model(
                checkpoint_path, model, hparams.ignore_layers)
        else:
            model, optimizer, _learning_rate, iteration = load_checkpoint(
                checkpoint_path, model, optimizer)
            if hparams.use_saved_learning_rate:
                learning_rate = _learning_rate
            iteration += 1  # next iteration is iteration + 1
            epoch_offset = max(0, int(iteration / len(train_loader)))

    model.train()
    is_overflow = False
    # ================ MAIN TRAINNIG LOOP! ===================
    for epoch in range(epoch_offset, hparams.epochs):
        print("Epoch: {}".format(epoch))
        for i, batch in enumerate(train_loader):
            start = time.perf_counter()   ## 返回当前的计算机系统时间
            for param_group in optimizer.param_groups:
                param_group['lr'] = learning_rate

            # print('CHECK batch:', batch)

            model.zero_grad()
            x, y = model.parse_batch(batch)
            y_pred = model(x)

            mask_padded = x[3]
            loss, mel_loss, gate_loss, select_loss = criterion(y_pred, y, mask_padded)  ## Tacotron2Loss(model_output,targets,mask_padded)
            ## 区分几种loss

            if hparams.distributed_run:
                reduced_loss = reduce_tensor(loss.data, n_gpus).item()
                reduced_val_mel_loss = reduce_tensor(mel_loss.data, n_gpus).item()
                reduced_val_gate_loss = reduce_tensor(gate_loss.data, n_gpus).item()
                reduced_val_select_loss = reduce_tensor(select_loss.data, n_gpus).item()
            else:
                reduced_loss = loss.item()
                reduced_val_mel_loss = mel_loss.item()
                reduced_val_gate_loss = gate_loss.item()
                reduced_val_select_loss = select_loss.item()
            if hparams.fp16_run:
                with amp.scale_loss(loss, optimizer) as scaled_loss:
                    scaled_loss.backward()
            else:
                loss.backward()

            # print('CHECK  structure_cnn.convs.0.weight IS CHANGE:', model.structure_cnn.convolutions[0][0].conv.weight)

            if hparams.fp16_run:
                grad_norm = torch.nn.utils.clip_grad_norm_(
                    amp.master_params(optimizer), hparams.grad_clip_thresh)
                is_overflow = math.isnan(grad_norm)
            else:
                grad_norm = torch.nn.utils.clip_grad_norm_(
                    model.parameters(), hparams.grad_clip_thresh)

            optimizer.step()
            ## 在用pytorch训练模型时，通常会在遍历epochs的过程中依次用到optimizer.zero_grad(),loss.backward(),optimizer.step()三个函数，总的来说，这三个函数的作用是先将梯度归零(optimizer.zero_grad()),
            ## 然后反向传播计算得到每个参数的梯度值(loss.backward()),最后通过梯度下降执行一步参数更新(optimizer.step())

            if not is_overflow and rank == 0:
                duration = time.perf_counter() - start  ## time.perf_counter()返回当前的计算机系统时间，只有连续两次perf_counter()进行差值才能有意义，一般用于计算程序运行时间。
                print("Train loss {} {:.6f} Grad Norm {:.6f} {:.2f}s/it".format(
                    iteration, reduced_loss, grad_norm, duration))
                logger.log_training(
                    reduced_loss, reduced_val_mel_loss, reduced_val_gate_loss, reduced_val_select_loss, grad_norm, learning_rate, duration, iteration)

            if not is_overflow and (iteration % hparams.iters_per_checkpoint == 0):
                validate(model, criterion, valset, iteration,
                         hparams.batch_size, n_gpus, collate_fn, logger,
                         hparams.distributed_run, rank)
                if rank == 0:
                    checkpoint_path = os.path.join(
                        output_directory, "checkpoint_{}".format(iteration))
                    save_checkpoint(model, optimizer, learning_rate, iteration,
                                    checkpoint_path)

            iteration += 1

class Mask_Softmax(nn.Module):
    def __init__(self, plus=1.0):
        super(Mask_Softmax, self).__init__()
        self.plus = plus
    def forward(self, logits, output_mask):
        logits = logits + (output_mask + 1e-45).log()
        return torch.nn.functional.log_softmax(logits, dim=-1)

class Gumbel_Softmax(nn.Module):
    def __init__(self, temperature=1):
        super(Gumbel_Softmax, self).__init__()
        self.softmax = nn.Softmax(dim=-1)    ## dim=-1 最后一维取softmax
        # initial temperature for gumbel softmax (default: 1)
        self.temperature = temperature
        self.mask_softmax = Mask_Softmax()
        # self.mask_softmax = nn.LogSoftmax()

    def forward(self, logits, output_mask, hard=False):
        y = self._gumbel_softmax_sample(logits, output_mask, hard)
        return y

    def _sample_gumbel(self, shape, eps=1e-20):
        U = torch.rand(shape)
        return -torch.log(-torch.log(U + eps) + eps)

    def _gumbel_softmax_sample(self, logits, output_mask, hard=False):
        sample = Variable(self._sample_gumbel(logits.size()[-1]), requires_grad=True)
        if logits.is_cuda:
            sample = sample.cuda()
        y = logits + sample
        # return self.softmax(y / self.temperature)
        y_soft = self.mask_softmax(y / self.temperature, output_mask)
        # y_soft = self.mask_softmax(y / self.temperature)

        if hard:
            # Straight through.
            index = y_soft.max(-1, keepdim=True)[1]
            y_hard = torch.zeros_like(logits).scatter_(-1, index, 1.0)
            ret = y_hard - y_soft.detach() + y_soft
        else:
            # Reparametrization trick.
            ret = y_soft
        return ret

def masked_augmax(logits, mask, dim, min_val=-1e7):
    logits = logits.exp()
    logits = logits.mul(mask)
    # one_minus_mask = (1.0 - mask).byte()
    # replaced_vector = vector.masked_fill(one_minus_mask, min_val)
    # max_value, _ = replaced_vector.max(dim=dim)
    max_value = torch.argmax(logits, dim=1)
    return max_value

def train_poly(args, hparams):
    torch.manual_seed(42)
    torch.cuda.manual_seed(42)
    np.random.seed(42)
    print('CHECK HERE train poly ONLY')
    train_dataloader = get_dataloader(hparams.use_output_mask, hparams.train_file, hparams.train_label,
                                      hparams, hparams.poly_batch_size,
                                      hparams.poly_max_length, shuffle=True)

    val_dataloader = get_dataloader(hparams.use_output_mask, hparams.val_file, hparams.val_label,
                                    hparams, hparams.poly_batch_size,
                                    hparams.poly_max_length, shuffle=True)

    # test_dataloader = get_dataloader(args.use_output_mask, args.test_file, args.test_label,
                                     # args.class2idx, args.merge_cedict, args.poly_batch_size,
                                     # args.max_length, shuffle=True)

    with codecs.open(hparams.class2idx, 'r', 'utf-8') as usernames:
        class2idx = json.load(usernames)
    print("num classes: {}".format(len(class2idx)))
    num_classes = len(class2idx)
    model = G2PTransformerMask(num_classes, hparams)
    device = torch.cuda.current_device()   ## 查看当前使用的gpu序号
    model = model.to(device)   ## 将模型加载到指定设备上
    for name, param in model.named_parameters():
        # frozen syntax module
        if name.split('.')[0] != 'tree_shared_linear' and name.split('.')[0] != 'structure_cnn_poly' \
                and name.split('.')[0] != 'linear_pre' and name.split('.')[0] != 'poly_phoneme_classifier' \
                and name.split('.')[0] != 'linear_aft':
            param.requires_grad = False
    training_parameters_list = [p for p in model.parameters() if p.requires_grad]
    optimizer = torch.optim.Adam(training_parameters_list, lr=hparams.poly_lr)

    criterion = nn.NLLLoss()
    # mask_criterion = Mask_Softmax()
    mask_criterion = Gumbel_Softmax()
    model_dir = "./save/poly_only_syntax_frozen"

    if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    best_acc = 0
    for epoch in range(hparams.poly_epochs):
        model.train()
        for idx, batch in enumerate(train_dataloader, start=1):
            # print('CEHCK batch:', batch)
            # if idx > 200:
            #     break
            batch = tuple(t.to(device) for t in batch)
            if hparams.use_output_mask:
                input_ids, poly_ids, labels, output_mask = batch
                mask = torch.sign(input_ids)
                inputs = {"input_ids": input_ids,
                          "poly_ids": poly_ids,
                          "attention_mask": mask}
            else:
                input_ids, poly_ids, labels = batch
                mask = torch.sign(input_ids)  ## torch.sign(input,out=None) 符号函数，返回一个新张量，包含输入input张量每个元素的正负（大于0的元素对应1，小于0的元素对应-1，0还是0）
                inputs = {"input_ids": input_ids,
                          "poly_ids": poly_ids,
                          "attention_mask": mask}

            # inputs = {"input_ids": input_ids,
                      # "poly_ids": poly_ids,
                      # "attention_mask": mask}
            logits, _ = model(**inputs)

            batch_size = logits.size(0)
            logits = logits[torch.arange(batch_size), poly_ids]

            # logits = mask_criterion(logits, output_mask, True)
            logits = mask_criterion(logits, output_mask)
            loss = criterion(logits, labels)
            loss.backward()
            # nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            optimizer.step()
            model.zero_grad()

            if idx % 100 == 0:    ## %取余
                print("loss : {:.4f}".format(loss.item()))
        all_preds = []
        all_mask_preds = []
        all_labels = []
        model.eval()
        for batch in tqdm(val_dataloader, total=len(val_dataloader)):
            batch = tuple(t.to(device) for t in batch)
            # input_ids, poly_ids,  labels = batch
            # mask = torch.sign(input_ids)

            # inputs = {"input_ids": input_ids,
                      # "poly_ids": poly_ids,
                      # "attention_mask": mask}
            if hparams.use_output_mask:
                input_ids, poly_ids, labels, output_mask = batch
                mask = torch.sign(input_ids)
                inputs = {"input_ids": input_ids,
                          "poly_ids": poly_ids,
                          "attention_mask": mask}
            else:
                input_ids, poly_ids, labels = batch
                mask = torch.sign(input_ids)
                inputs = {"input_ids": input_ids,
                          "poly_ids": poly_ids,
                          "attention_mask": mask}
            with torch.no_grad():
                logits, _ = model(**inputs)

            batch_size = logits.size(0)
            logits = logits[torch.arange(batch_size), poly_ids]
            # logits = logits.exp()
            # output_mask_false = 1.0 - output_mask
            # logits = logits - output_mask_false
            # logits = mask_criterion(logits, output_mask, True)
            logits = mask_criterion(logits, output_mask)
            preds = torch.argmax(logits, dim=1).cpu().numpy()
            mask_preds = masked_augmax(logits, output_mask, dim=1).cpu().numpy()
            if not (preds == mask_preds).all():
                print('CHECK preds:', preds)
                print('CHECK mask_preds:', mask_preds)
                print('CHECK labels:', labels)
                print('CHECK output_mask:', np.where(output_mask.cpu().numpy()==1.0))

            all_preds.append(preds)
            all_mask_preds.append(mask_preds)
            all_labels.append(labels.cpu().numpy())
        preds = np.concatenate(all_preds, axis=0)
        mask_preds = np.concatenate(all_mask_preds, axis=0)
        labels = np.concatenate(all_labels, axis=0)
        # print('CHECK preds:', preds)
        # print('CHECK mask_preds:', mask_preds)
        # print('CHECK labels:', labels)

        val_acc = accuracy_score(labels, preds)
        mask_val_acc = accuracy_score(labels, mask_preds)
        pred_diff_acc = accuracy_score(preds, mask_preds)
        print("epoch :{}, acc: {:.2f}, mask acc: {:.2f}, pred_diff_acc: {:.2f}".format(epoch, val_acc*100, mask_val_acc*100, pred_diff_acc*100))
        if val_acc > best_acc:
            best_acc = val_acc
            state_dict = model.state_dict()
            save_file = os.path.join(
                model_dir, "{:.2f}_model.pt".format(val_acc*100))
            torch.save(state_dict, save_file)

def train_poly_tts(args, hparams):
    torch.manual_seed(42)
    torch.cuda.manual_seed(42)
    np.random.seed(42)
    print('CHECK HERE train TTS poly')
    train_dataloader = polyTTS_get_dataloader(hparams.use_output_mask, hparams.training_files,
                                      hparams, hparams.poly_batch_size,
                                      hparams.poly_max_length, shuffle=True)

    val_dataloader = polyTTS_get_dataloader(hparams.use_output_mask, hparams.validation_files,
                                      hparams, hparams.poly_batch_size,
                                      hparams.poly_max_length, shuffle=True)

    with codecs.open(hparams.class2idx, 'r', 'utf-8') as usernames:
        class2idx = json.load(usernames)
    print("num classes: {}".format(len(class2idx)))
    num_classes = len(class2idx)
    model = poly_tonesandhi(num_classes, hparams)
    device = torch.cuda.current_device()
    model = model.to(device)

    for name, param in model.named_parameters():
        # frozen syntax module
        if name.split('.')[0] == 'g2ptransformermask':
            if name.split('.')[1] != 'tree_shared_linear' and name.split('.')[1] != 'structure_cnn_poly' \
                    and name.split('.')[1] != 'linear_pre' and name.split('.')[1] != 'poly_phoneme_classifier' \
                    and name.split('.')[1] != 'linear_aft':
                param.requires_grad = False

    training_parameters_list = [p for p in model.parameters() if p.requires_grad]
    optimizer = torch.optim.Adam(training_parameters_list, lr=hparams.poly_lr)

    # criterion = nn.CrossEntropyLoss()
    criterion = nn.NLLLoss()
    # mask_criterion = Mask_Softmax()
    mask_criterion = Gumbel_Softmax()
    model_dir = "./save/poly_tts_CNN_syntax_frozen"

    if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    best_acc = 0
    for epoch in range(hparams.poly_epochs):
        model.train()
        for idx, batch in enumerate(train_dataloader, start=1):
            # print('CEHCK batch:', batch)
            # if idx > 200:
            #     break
            batch = tuple(t.to(device) for t in batch)
            if hparams.use_output_mask:
                input_ids, poly_ids, labels, output_mask = batch
                mask = torch.sign(input_ids)
                inputs = {"input_ids": input_ids,
                          "poly_ids": poly_ids,
                          "attention_mask": mask}
            else:
                input_ids, poly_ids, labels = batch
                mask = torch.sign(input_ids)
                inputs = {"input_ids": input_ids,
                          "poly_ids": poly_ids,
                          "attention_mask": mask}

            logits, _ = model(**inputs)
            # logits = mask_criterion(logits, output_mask, True)
            labels, logits, output_mask = model.select_poly(labels, logits, output_mask, poly_ids)

            logits = mask_criterion(logits, output_mask)

            loss = criterion(logits, labels)
            loss.backward()
            # nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            optimizer.step()
            model.zero_grad()

            if idx % 100 == 0:
                print("loss : {:.4f}".format(loss.item()))

        all_preds = []
        all_labels = []
        model.eval()
        for batch in tqdm(train_dataloader, total=len(train_dataloader)):
            batch = tuple(t.to(device) for t in batch)
            if hparams.use_output_mask:
                input_ids, poly_ids, labels, output_mask = batch
                mask = torch.sign(input_ids)
                inputs = {"input_ids": input_ids,
                          "poly_ids": poly_ids,
                          "attention_mask": mask}
            else:
                input_ids, poly_ids, labels = batch
                mask = torch.sign(input_ids)
                inputs = {"input_ids": input_ids,
                          "poly_ids": poly_ids,
                          "attention_mask": mask}
            with torch.no_grad():
                logits, _ = model(**inputs)

            labels, logits, output_mask = model.select_poly(labels, logits, output_mask, poly_ids)
            logits = mask_criterion(logits, output_mask)
            # labels, logits = model.select_poly(labels, logits, poly_ids)
            preds = torch.argmax(logits, dim=1).cpu().numpy()

            all_preds.append(preds)
            all_labels.append(labels.cpu().numpy())
        preds = np.concatenate(all_preds, axis=0)
        labels = np.concatenate(all_labels, axis=0)

        train_acc = accuracy_score(labels, preds)
        print("epoch :{}, Trainging acc: {:.2f}".format(epoch, train_acc*100))

        all_preds = []
        # all_mask_preds = []
        all_labels = []
        model.eval()
        for batch in tqdm(val_dataloader, total=len(val_dataloader)):
            batch = tuple(t.to(device) for t in batch)
            if hparams.use_output_mask:
                input_ids, poly_ids, labels, output_mask = batch
                mask = torch.sign(input_ids)
                inputs = {"input_ids": input_ids,
                          "poly_ids": poly_ids,
                          "attention_mask": mask}
            else:
                input_ids, poly_ids, labels = batch
                mask = torch.sign(input_ids)
                inputs = {"input_ids": input_ids,
                          "poly_ids": poly_ids,
                          "attention_mask": mask}
            with torch.no_grad():
                logits, _ = model(**inputs)

            labels, logits, output_mask = model.select_poly(labels, logits, output_mask, poly_ids)
            logits = mask_criterion(logits, output_mask)
            # labels, logits = model.select_poly(labels, logits, poly_ids)
            preds = torch.argmax(logits, dim=1).cpu().numpy()

            all_preds.append(preds)
            all_labels.append(labels.cpu().numpy())
        preds = np.concatenate(all_preds, axis=0)
        labels = np.concatenate(all_labels, axis=0)

        val_acc = accuracy_score(labels, preds)
        print("epoch :{}, acc: {:.2f}".format(epoch, val_acc*100))
        if val_acc > best_acc:
            best_acc = val_acc
            state_dict = model.state_dict()
            save_file = os.path.join(
                model_dir, "{:.2f}_model.pt".format(val_acc*100))
            torch.save(state_dict, save_file)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-o', '--output_directory', type=str,
                        help='directory to save checkpoints')
    parser.add_argument('-l', '--log_directory', type=str,
                        help='directory to save tensorboard logs')
    parser.add_argument('-c', '--checkpoint_path', type=str, default=None,
                        required=False, help='checkpoint path')
    parser.add_argument('--warm_start', action='store_true',
                        help='load model weights only, ignore specified layers')
    parser.add_argument('--n_gpus', type=int, default=1,
                        required=False, help='number of gpus')
    parser.add_argument('--rank', type=int, default=0,
                        required=False, help='rank of current gpu')
    parser.add_argument('--group_name', type=str, default='group_name',
                        required=False, help='Distributed group name')
    parser.add_argument('--hparams', type=str,
                        required=False, help='comma separated name=value pairs')

    parser.add_argument('--train_poly', dest='train_poly', action='store_true', default=False, required=False)
    parser.add_argument('--train_tts', dest='train_tts', action='store_true', default=False, required=False)

    args = parser.parse_args()
    hparams = create_hparams()

    torch.backends.cudnn.enabled = hparams.cudnn_enabled
    torch.backends.cudnn.benchmark = hparams.cudnn_benchmark

    print("FP16 Run:", hparams.fp16_run)
    print("Dynamic Loss Scaling:", hparams.dynamic_loss_scaling)
    print("Distributed Run:", hparams.distributed_run)
    print("cuDNN Enabled:", hparams.cudnn_enabled)
    print("cuDNN Benchmark:", hparams.cudnn_benchmark)

    if args.train_poly:
        # train_poly(args, hparams)
        train_poly_tts(args, hparams)
    if args.train_tts:
        train_tts(args.output_directory, args.log_directory, args.checkpoint_path,
            args.warm_start, args.n_gpus, args.rank, args.group_name, hparams)


import logging
import re

# Setup logging
log = logging.getLogger(__name__)

def retrieve_hc_dpd_words():
    """Extracts all words from the hc_dpd application"""
    from hc_dpd.models import (
        ActiveIngredient, Company, DrugProduct, SubBrand, SubCompanyName,
        SubIngredient
    )

    # Setup the words dictionary to hold all the retrieved words
    words = {
        "English": {
            "company_names": {
                "disinfectant": {
                    "word_set": set(),
                    "word_list": []
                },
                "human": {
                    "word_set": set(),
                    "word_list": []
                },
                "veterinary": {
                    "word_set": set(),
                    "word_list": []
                },
            },
            "trade_names": {
                "disinfectant": {
                    "word_set": set(),
                    "word_list": []
                },
                "human": {
                    "word_set": set(),
                    "word_list": []
                },
                "veterinary": {
                    "word_set": set(),
                    "word_list": []
                },
            },
            "ingredients": {
                "disinfectant": {
                    "word_set": set(),
                    "word_list": []
                },
                "human": {
                    "word_set": set(),
                    "word_list": []
                },
                "veterinary": {
                    "word_set": set(),
                    "word_list": []
                },
            }
        },
        "French": {
            "company_names": {
                "disinfectant": {
                    "word_set": set(),
                    "word_list": []
                },
                "human": {
                    "word_set": set(),
                    "word_list": []
                },
                "veterinary": {
                    "word_set": set(),
                    "word_list": []
                },
            },
            "trade_names": {
                "disinfectant": {
                    "word_set": set(),
                    "word_list": []
                },
                "human": {
                    "word_set": set(),
                    "word_list": []
                },
                "veterinary": {
                    "word_set": set(),
                    "word_list": []
                },
            },
            "ingredients": {
                "disinfectant": {
                    "word_set": set(),
                    "word_list": []
                },
                "human": {
                    "word_set": set(),
                    "word_list": []
                },
                "veterinary": {
                    "word_set": set(),
                    "word_list": []
                },
            }
        }
    }

    # Extract the trade name words
    log.debug("Retrieving trade names")

    trade_names = SubBrand.objects.all().values_list("substitution", flat=True)

    for trade_name in trade_names:
        log.debug("Extracting/processing words from '{}'".format(trade_name))

        # Get references to any entry this substitution applies to
        drug_products = DrugProduct.objects.filter(
            brand_name=trade_name
        )

        for product in drug_products:
            class_name = product.class_e.lower()
            word_dict = words["English"]["trade_names"][class_name]

            word_list = re.findall(r"[\w']+", trade_name)

            for word in word_list:
                # Check if this word is already in the set
                if word not in word_dict["word_set"]:
                    # Add word to set
                    word_dict["word_set"].add(word)

                    # Add diciontary to word_list
                    word_dict["word_list"].append({
                        "original_string": trade_name,
                        "word": word,
                    })

    # Extract the ingredient words
    log.debug("Retrieving ingredients")

    ingredients = SubIngredient.objects.all().values_list(
        "substitution", flat=True
    )

    for ingredient in ingredients:
        log.debug("Extracting/processing words from '{}'".format(ingredient))

        # Get references to any entry this substitution applies to
        active_ingredients = ActiveIngredient.objects.filter(
            ingredient=ingredient
        )

        for active_ingredient in active_ingredients:
            # Get the proper class name for this active_ingredient
            try:
                class_name = DrugProduct.objects.get(
                    drug_code=active_ingredient.drug_code
                ).class_e.lower()
            except MultipleObjectsReturned:
                log.warn(
                    "Multiple DrugProducts with same drug code",
                    exc_info=True
                )
                class_name = DrugProduct.objects.filter(
                    drug_code=active_ingredient.drug_code
                )[0].class_e.lower()
            except DoesNotExist:
                class_name = ""

            # Setup reference to word_dictioanry & create word_list
            try:
                word_dict = words["English"]["ingredients"][class_name]
                word_list = re.findall(r"[\w']+", ingredient)
            except IndexError:
                log.warn(
                    "Invalid class name provided: {}".format(class_name),
                    exc_info=True
                )

            # Add each unique word to the dictionary
            for word in word_list:
                # Check if this word is already in the set
                if word not in word_dict["word_set"]:
                    # Add word to set
                    word_dict["word_set"].add(word)

                    # Add diciontary to word_list
                    word_dict["word_list"].append({
                        "original_string": ingredient,
                        "word": word,
                    })

    # Extract the company words
    log.debug("Retrieving company names")

    company_names = SubCompanyName.objects.all().values_list(
        "substitution", flat=True
    )

    for name in company_names:
        log.debug("Extracting/processing words from '{}'".format(name))

        # Get references to any entry this substitution applies to
        companies = Company.objects.filter(
            company_name=name
        )

        for company in companies:
            # Get the proper class name for this active_ingredient
            try:
                class_name = DrugProduct.objects.get(
                    drug_code=company.drug_code
                ).class_e.lower()
            except MultipleObjectsReturned:
                log.warn(
                    "Multiple DrugProducts with same drug code",
                    exc_info=True
                )
                class_name = DrugProduct.objects.filter(
                    drug_code=company.drug_code
                )[0].class_e.lower()
            except DoesNotExist:
                class_name = ""

            # Setup reference to word_dictioanry & create word_list
            try:
                word_dict = words["English"]["company_names"][class_name]
                word_list = re.findall(r"[\w']+", name)
            except IndexError:
                log.warn(
                    "Invalid class name provided: {}".format(class_name),
                    exc_info=True
                )

            # Add each unique word to the dictionary
            for word in word_list:
                # Check if this word is already in the set
                if word not in word_dict["word_set"]:
                    # Add word to set
                    word_dict["word_set"].add(word)

                    # Add diciontary to word_list
                    word_dict["word_list"].append({
                        "original_string": name,
                        "word": word,
                    })

    return words

def retrieve_words(application):
    """Uses the provided application to return a word list"""
    if application.upper() == "HC_DPD":
        log.info("Retrieving HC DPD application words")
        return retrieve_hc_dpd_words()


import time
import matplotlib.pyplot as plt
import numpy as np

def draw_features(width,height,x,savename):
    tic=time.time()
    fig = plt.figure(figsize=(16, 16))
    fig.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.95, wspace=0.05, hspace=0.05)
    for i in range(width*height):
        plt.subplot(height,width, i + 1)
        plt.axis('off')
        # plt.tight_layout()
        img = x[0, i, :, :]
        pmin = np.min(img)
        pmax = np.max(img)
        img = (img - pmin) / (pmax - pmin + 0.000001)
        plt.imshow(img, cmap='gray')
        print("{}/{}".format(i,width*height))
    fig.savefig(savename, dpi=100)
    fig.clf()
    plt.close()
    print("time:{}".format(time.time()-tic))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import sys
import csv
from datetime import datetime
import locale

# set czech locale for float number formating
locale.setlocale(locale.LC_ALL, 'cs_CZ.UTF-8')

GPC_TYPE_REPORT = '074'
GPC_TYPE_ITEM   = '075'
INPUT_ENCODING = 'windows-1250'

if (sys.version_info < (3,0)):
    print("The script needs to run under Python3")
    print("Usage: python3 %s input_file [output_file]" % sys.argv[0])
    sys.exit(1)

class GPC_Reader:

    def parse_gpc_record(self, line):
        rec_type = line[0:3]
        result = {
            'type': rec_type
        }

        if rec_type == GPC_TYPE_REPORT:
            #print line
            result['account_number'] = line[3:19]
            result['account_name'] = line[19:39].strip()
            result['old_balance_date'] = datetime.strptime(line[39:45],'%d%m%y').strftime('%d.%m.%Y')
            result['old_balance_value'] = int(line[59:60] + line[45:59])/100.0
            result['new_balance_value'] = int(line[74:75] + line[60:74])/100.0
            result['debit_value'] = int(line[89:90] + line[75:89])/100.0
            result['credit_value'] = int(line[104:105] + line[90:104])/100.0
            result['statement_number'] = line[105:108]
            result['date'] = datetime.strptime(line[108:114],'%d%m%y').strftime('%d.%m.%Y')
        elif rec_type == GPC_TYPE_ITEM:
            #print line
            result['own_account'] = line[3:19]
            result['external_account'] = line[19:35] + '/' + line[73:77]
            result['record_number'] = line[35:48]
            result['value'] = int(line[48:60])/100.0
            result['value_code'] = int(line[60:61])
            result['variable_symbol'] = int(line[61:71])
            result['constant_symbol'] = line[77:81]
            result['specific_symbol'] = int(line[81:91])
            result['client_name'] = line[97:117].strip()
            result['currency_code'] = line[117:122]
            result['due_date'] = datetime.strptime(line[122:128],'%d%m%y').strftime('%d.%m.%Y')
            if ((result['value_code']==1) or (result['value_code']==4)): result['value'] = -result['value']

            # convert value to string
            result['value'] = locale.format("%.2f", result['value'])
        else:
            raise Exception('Invalid GPC record type (%s)' % (rec_type))

        return result
    #end def

    def csv_output(self, output_file):
        # czech names for the gpc fields
        field_names_cz = {
            'type': 'typ záznamu',
            'own_account': 'vlastní účet',
            'external_account': 'účet protistrany',
            'record_number': 'pořadové číslo',
            'value': 'částka',
            'value_code': 'kód účtování',
            'variable_symbol': 'VS',
            'constant_symbol': 'KS',
            'specific_symbol': 'SS',
            'client_name': 'název účtu prostistrany',
            'currency_code': 'kód měny',
            'due_date': 'datum zaúčtování'
        }

        # fields to export
        fields = list(self.gpc_data[0].keys())
        fields.remove('type')

        # czech translation of export fields
        fields_cz = [field_names_cz.get(n,n) for n in fields]

        fp = open(output_file, 'w') if (output_file) else sys.stdout
        w = csv.DictWriter(fp, fieldnames=fields, extrasaction='ignore', dialect='excel')
        w.writerow(field_names_cz) #print header with translated field names
        w.writerows(self.gpc_data)
        if (output_file): fp.close()
    #end def

    def read(self, filename):
        self.gpc_data = []
        with open(filename, 'r', encoding=INPUT_ENCODING) as fp:
            for line in fp:
                r = self.parse_gpc_record(line)
                # only include transaction records to the output
                if (r['type'] == GPC_TYPE_ITEM): self.gpc_data.append(r)
    #end def

#end class

if (len(sys.argv) <= 1):
    print("Missing input file argument.")
    print("Usage: python3 %s input_file [output_file]" % sys.argv[0])
    sys.exit(1)

input_file = sys.argv[1]
output_file = sys.argv[2] if (len(sys.argv) >= 3) else None

if (not os.path.isfile(input_file)):
    print("Input file does not exits.")
    print("Usage: python3 %s input_file [output_file]" % sys.argv[0])
    sys.exit(1)

gpc = GPC_Reader()
gpc.read(sys.argv[1])
gpc.csv_output(output_file)


from tkinter import *

def yaz(x):
    s = len(giriş.get())
    giriş.insert(s,str(x))
    #print(x)

s2 = 0
def hesapla():
    global s2
    s2 = float(giriş.get())
    print(s2)
    global  hesap
    sonuc=0
    if(hesap==0):
        sonuc = s1 + s2
    elif(hesap==1):
        sonuc = s1 - s2
    elif(hesap==2):
        sonuc = s1 * s2
    elif(hesap==3):
        sonuc = s1 / s2
    giriş.delete(0,'end')
    giriş.insert(0,str(sonuc))

hesap = 5
s1 = 0

def işlemler(x):
    global  hesap
    hesap = x
    global s1
    s1 = float(giriş.get())
    print(hesap)
    print(s1)
    giriş.delete(0,'end')

pencere= Tk()
pencere.geometry("250x250")

giriş = Entry(font="verdana 14 bold",fg="RED" ,width=15,justify=RIGHT)
giriş.place(x=20,y=20)

b = []

for i in range(1,10):
    b.append(Button(text=str(i),fg= "GREEN" ,font="Verdana 14 bold",command=lambda x=i:yaz(x)))

sayac=0
for i in range(0,3):
    for j in range(0,3):
        b[sayac].place(x=20+j*50,y=50+i*50)
        sayac += 1

işlem = []
for i in range(0,4):
    işlem.append(Button(font="Verdana 14 bold",width=2,command=lambda x=i:işlemler(x)))

işlem[0]['text'] = "+"
işlem[1]['text'] = "-"
işlem[2]['text'] = "*"
işlem[3]['text'] = "/"

for i in range(0,4):
    işlem[i].place(x=170,y=50+50*i)

sb = Button(text=0,font="Verdana 14 bold",command=lambda x=0:yaz(x))
sb.place(x=20,y=200)

nb = Button(text=".",font="Verdana 14 bold",width=2,command=lambda x=".":yaz(x))
nb.place(x=70,y=200)

eb = Button(text='=',fg= "BLUE",font="Verdana 14 bold",width=2,command=hesapla)
eb.place(x=120,y=200)
pencere.mainloop()


from base import *
def cov_pathway(cov_3d,cutoff):
    import networkx as nx
    g1=nx.Graph()
    g1.add_nodes_from(cov_3d.index-1)
    for i in cov_3d.index-1:
        for j in cov_3d.index-1:
            if i<=j:
                continue
            elif cov_3d.iat[i,j]>cutoff:
                g1.add_edge(i,j,weight=cov_3d.iat[i,j],capacity=cov_3d.iat[i,j])
    return g1
def residual_network_domain(similarities,dist_av_std,rs,rt,zbool,figname):
    import pandas as pd
    import numpy as np
    import multiprocessing as mp
    from sklearn.cluster import AffinityPropagation
    import matplotlib.pyplot as plt
    from sklearn import cluster, covariance, manifold
    from sklearn.decomposition import PCA
    from matplotlib.collections import LineCollection
    import networkx as nx
    from networkx.algorithms.flow import shortest_augmenting_path
    import matplotlib.colors as colors
    import matplotlib.cm as cm
    domains={'Nterm':list(range(0,131))+list(range(337,370)),'Exon':list(range(131,337)),\
             'Palm':list(range(370,446))+list(range(498,606)),'Finger':list(range(446,498)),\
             'Thumb':list(range(606,756)),'Pdna':list(range(756,768)),'Tdna':list(range(768,782))}
    dom_colors={'Nterm':'blue','Exon':'olive','Palm':'cyan','Finger':'orange','Thumb':'green','Pdna':'purple','Tdna':'brown'}
    seed = np.random.RandomState(seed=3)
    mds = manifold.MDS(n_components=2, max_iter=3000, eps=1e-9, random_state=seed,
                       dissimilarity="precomputed", n_jobs=1).fit(1/(1+similarities/(similarities.max().max()-similarities.min().min()))**2)
    pos = mds.embedding_
    print('mds is ok')
    nmds = manifold.MDS(n_components=2, metric=False, max_iter=3000, eps=1e-12,
                        dissimilarity="precomputed", random_state=seed, n_jobs=1,
                        n_init=1).fit_transform(1/(1+similarities/(similarities.max().max()-similarities.min().min()))**2, init=pos)
    print('nmds is ok')
    npos = nmds
    clf = PCA(n_components=2)
    tnpos = clf.fit_transform(npos)
    tnpos[:,1]*=zbool
    print('pca is ok')
    edge_cut=0.5
    pymol_color='red,raspberry,darksalmon,brown,salmon,warmpink,ruby,green,chartreuse,splitpea,smudge,palegreen,\
                lime,limon,forest,blue,tv_blue,marine,slate,lightblue,skyblue,purplebule,deepblue,density,\
                yellow,paleyellow,yelloworange,wheat,sand,magenta,lightmagenta,hotpink,pink,lightpink,violet,\
                violetpurple,purple,deeppurple,cyan,palecyan,aquamarine,greencyan,teal,deepteal,lighteal,\
                orange,brightorange,lightorange,yelloworange,olive,deepolive,wheat,palegreen,lightblue,\
                paleyellow,palecyan,bluewhite,white,gray90,gray80,gray70,gray60,gray50,gray40,gray30,gray20,gray10,black'.split(',')

    #'''
    G1=cov_pathway(similarities,0)
    mgr=mp.Manager()
    maxf_nets=mgr.dict()
    maxf_netsv=mgr.dict()
    def get_resnet(G1,s,t,nets,netsv):
        net_temp=nx.flow.maxflow.maximum_flow(G1,s,t,capacity='capacity')
        nets[str(s)+'_'+str(t)]=pd.concat([pd.DataFrame(net_temp[1][i],index=[i]) for i in net_temp[1].keys()]).fillna(0)
        netsv[str(s)+'_'+str(t)]=net_temp[0]
        #print(s,t,'is done')
    jobs={}
    for i in range(len(rs)):
        for j in range(len(rt)):
            keyij=str(rs[i]+1)+'_'+str(rt[j]+1)
            jobs[keyij]=mp.Process(target=get_resnet,args=(G1,rs[i],rt[j],maxf_nets,maxf_netsv))
            jobs[keyij].start()
        for j in range(len(rt)):
            keyij=str(rs[i]+1)+'_'+str(rt[j]+1)
            jobs[keyij].join()
    keyijs=[k for k in maxf_nets.keys()]
    print(keyijs)
    maxf_resnet_pd=pd.DataFrame(np.zeros(similarities.shape),index=G1.nodes,columns=G1.nodes)
    maxf_netsvalue={}
    for k in maxf_nets.keys():
        maxf_resnet_pd+=maxf_nets[k]
        maxf_netsvalue[k]=maxf_netsv[k]
    maxf_resnet_pd/=len(maxf_nets)
    plt.imshow(maxf_resnet_pd)
    plt.colorbar()
    plt.show()
    fig = plt.figure(1,figsize=(12,12))
    ax = plt.axes([0., 0., 1., 1.])
    lines_r=[]
    lines_g=[]
    for i in range(maxf_resnet_pd.shape[0]):
        if i%100==0:
            print('running:',i)
        #if i>40:
        #    continue
        for j in range(maxf_resnet_pd.shape[1]):
            if maxf_resnet_pd.iat[i,j]<=0.25*edge_cut:
                continue
            if i!=j and maxf_resnet_pd.iat[i,j]<edge_cut:
                deltxy=tnpos[j]-tnpos[i]
                plt.arrow(tnpos[i][0],tnpos[i][1],deltxy[0],deltxy[1],head_width=8*0.0025*maxf_resnet_pd.iat[i,j],\
                          head_length=8*0.0025*maxf_resnet_pd.iat[i,j],linewidth=maxf_resnet_pd.iat[i,j],\
                          length_includes_head=True,color='grey',alpha=0.3)
                lines_r.append([tnpos.T[0][[i,j]].T,tnpos.T[1][[i,j]].T])
        for j in range(maxf_resnet_pd.shape[1]):
            if maxf_resnet_pd.iat[i,j]<=0.25*edge_cut:
                continue
            if i!=j and maxf_resnet_pd.iat[i,j]>=edge_cut:
                #plt.plot(tnpos.T[0][[i,j]].T,tnpos.T[1][[i,j]].T,linestyle='-',color='red',linewidth=maxf_resnet_pd.iat[i,j])
                deltxy=tnpos[j]-tnpos[i]
                plt.arrow(tnpos[i][0],tnpos[i][1],deltxy[0],deltxy[1],head_width=8*0.0025*maxf_resnet_pd.iat[i,j],\
                          head_length=8*0.0025*maxf_resnet_pd.iat[i,j],linewidth=maxf_resnet_pd.iat[i,j],\
                          length_includes_head=True,color='red',alpha=0.4)
                lines_r.append([tnpos.T[0][[i,j]].T,tnpos.T[1][[i,j]].T])
    #'''
    for k in domains.keys():
        plt.plot(tnpos.T[0][domains[k]].T, tnpos.T[1][domains[k]].T,'.',color=dom_colors[k],markersize=15)
        #plt.plot(tnpos.T[0][data_ap_xyz.cluster_centers_indices_[k]], tnpos.T[1][[data_ap_xyz.cluster_centers_indices_[k]]],'*',color=pycolor[k],markersize=10)
        #plt.text(tnpos.T[0][data_ap_xyz.cluster_centers_indices_[k]], tnpos.T[1][[data_ap_xyz.cluster_centers_indices_[k]]],similarities.index[data_ap_xyz.cluster_centers_indices_[k]],size=8,alpha=0.8)
    plt.plot(tnpos.T[0][rs].T, tnpos.T[1][rs].T,'o',color='red',alpha=0.6,markersize=20)
    #plt.plot(tnpos.T[0][list(range(756,782))].T, tnpos.T[1][list(range(756,782))].T,'s',color='grey',alpha=0.3,markersize=20)
    plt.plot(tnpos.T[0][rt].T, tnpos.T[1][rt].T,'o',color='white',alpha=0.5,markersize=20)
    for s in rs:
        plt.text(tnpos.T[0][s].T, tnpos.T[1][s].T,s+1,color='black',size=22)
    for t in rt:
        plt.text(tnpos.T[0][t].T, tnpos.T[1][t].T,t+1,color='black',size=22)
    plt.savefig(figname+'_domain_net.png',dpi=300,transparent=True)
    plt.show()
    resnet1=maxf_resnet_pd[maxf_resnet_pd>0].fillna(-1)
    resnet2=resnet1[resnet1==-1].fillna(1)
    resnet2[resnet2!=-1].sum().sum()
    print(len(lines_r),resnet2[resnet2!=-1].sum().sum(),'MaxF_sum:',pd.DataFrame(maxf_netsvalue,index=['MaxF']).sum().sum(),'MaxF:',maxf_netsv)
    print('Color Code:',dom_colors)
    pd.DataFrame(maxf_netsvalue,index=['MaxF']).T.sort_values('MaxF',ascending=False).plot.bar(figsize=(14,3),ylim=(0,220))
    return lines_r,lines_g,maxf_nets,maxf_netsvalue,tnpos


import sys

strFromFile = sys.argv[1]
strToFile = sys.argv[2]

print ("Read from " + strFromFile)
print ("Save to " + strToFile)

fp = open(strFromFile, "r")
lines = fp.readlines()
fp.close()

lines.sort()

fp = open(strToFile, "w")
fp.writelines(lines)
fp.close()


dog = dict()  # Create an empty dictionary:dog
dog["Name"] = "Jarvis"  # To add name to the dog
dog["Color"] = "Black"  # To add color to the dog
dog["Breed"] = "Bulldog"  # To add breed name to the dog
dog["Legs"] = 4  # To add no of legs to the dog
dog["Age"] = 5  # To add age to the dog
print(dog)  # To print dog

student = dict()  # Create an empty dictionary student
student["First_name"] = "Thala"  # To add first name to the student
student["Last_name"] = "Ajith"  # To add last name to the student
student["Gender"] = "Male"  # To add gender to the student
student["Marital status"] = "Married"  # To add marital status to the student
student["Skills"] = ["Actor", "Pilot", "Car Racer"]  # To add skills to the student
student["Country"] = "India"  # To add country to the student
student["Address"] = "Chennai"  # To add address to the student
print(student)  # To print student
print(len(student))  # To print length of student
print(student["Skills"])  # To get skills of student
print(type(student["Skills"]))  # To get type of skills of student
student["Skills"].extend(["Bike Racer", "Chef"])  # To modify skills of student
print(student["Skills"])  # To print student skills after modifying
print(student.keys())  # To get keys of student dictionary as list
print(student.values())  # To get values of student dictionary as list


# Name : Abhishek Dethe
# Roll No: 2201037
# Div  A

# 5.Write Python program to handle multiple exceptions in single except block

try:
    # Code block where an exception might occur
    numerator = int(input("Enter the numerator: "))
    denominator = int(input("Enter the denominator: "))

    result = numerator / denominator
    print("Result:", result)

except (ValueError, ZeroDivisionError):
    # Exception handler for multiple exceptions
    print("An error occurred. Please enter valid numeric values and avoid division by zero.")

finally:
    # Code block that will always execute, regardless of exceptions
    print("Program execution completed.")


# -*- coding: utf-8 -*-
import scrapy
import json

class QuoteScrollSpider(scrapy.Spider):
    """
        This spider fetches the data from the infinite scroll page.
    """
    name = 'quote_scroll'
    allowed_domains = ['quotes.toscrape.com']
    api_url = 'http://quotes.toscrape.com/api/quotes?page={}'
    start_urls = [api_url.format(1)]

    def parse(self, response):
        #dict_keys(['has_next', 'page', 'quotes', 'tag', 'top_ten_tags'])
        quotes_dict = json.loads(response.text)

        for quotes in quotes_dict["quotes"]:
            yield {
                'author': quotes['author']['name'],
                'text': quotes['text'],
                'tags': quotes['tags'],
            }

        if quotes_dict['has_next']:
            next_page = quotes_dict['page']+1
            yield scrapy.Request(url=self.api_url.format(next_page), callback=self.parse)


from audioop import avg
import statistics
listacal=[]
listanombre=[]
alum=int(input("Dime cuántos alumnos tiene la clase: "))
for i in range (alum):
    nombre=input("Dime el nombre del alumno: ")
    nota=int(input("dime cuantas notas tiene ese alumno: "))
    listanombre.append(nombre)
    for n in range(nota):
        calificacion=int(input("Ingesa la nota: "))
        listacal.append(calificacion)
        listacal[nota:]
print(listacal)
 #print(sum(listacal)/nota)


import networkx as nx
import pytest

from dvc.cli import parse_args
from dvc.command.dag import CmdDAG, _build, _show_ascii, _show_dot

@pytest.mark.parametrize("fmt", [None, "--dot"])
def test_dag(tmp_dir, dvc, mocker, fmt):
    tmp_dir.dvc_gen("foo", "foo")

    args = ["dag", "--full", "foo.dvc"]
    if fmt:
        args.append(fmt)
    cli_args = parse_args(args)
    assert cli_args.func == CmdDAG

    cmd = cli_args.func(cli_args)

    mocker.patch("dvc.command.dag._build", return_value=dvc.graph)

    assert cmd.run() == 0

@pytest.fixture
def repo(tmp_dir, dvc):
    tmp_dir.dvc_gen("a", "a")
    tmp_dir.dvc_gen("b", "b")

    dvc.run(
        no_exec=True, deps=["a", "c"], outs=["d", "e"], cmd="cmd1", name="1"
    )
    dvc.run(
        no_exec=True, deps=["b", "c"], outs=["f", "g"], cmd="cmd2", name="2"
    )
    dvc.run(
        no_exec=True,
        deps=["a", "b", "c"],
        outs=["h", "i"],
        cmd="cmd3",
        name="3",
    )
    dvc.run(no_exec=True, deps=["a", "h"], outs=["j"], cmd="cmd4", name="4")

    return dvc

def test_build(repo):
    assert nx.is_isomorphic(_build(repo), repo.graph)

def test_build_target(repo):
    G = _build(repo, target="3")
    assert set(G.nodes()) == {"3", "b.dvc", "a.dvc"}
    assert set(G.edges()) == {("3", "a.dvc"), ("3", "b.dvc")}

def test_build_target_with_outs(repo):
    G = _build(repo, target="3", outs=True)
    assert set(G.nodes()) == {"a", "b", "h", "i"}
    assert set(G.edges()) == {("i", "a"), ("i", "b"), ("h", "a"), ("h", "b")}

def test_build_granular_target_with_outs(repo):
    G = _build(repo, target="h", outs=True)
    assert set(G.nodes()) == {"a", "b", "h"}
    assert set(G.edges()) == {("h", "a"), ("h", "b")}

def test_build_full(repo):
    G = _build(repo, target="3", full=True)
    assert nx.is_isomorphic(G, repo.graph)

# NOTE: granular or not, full outs DAG should be the same
@pytest.mark.parametrize("granular", [True, False])
def test_build_full_outs(repo, granular):
    target = "h" if granular else "3"
    G = _build(repo, target=target, outs=True, full=True)
    assert set(G.nodes()) == {"j", "i", "d", "b", "g", "f", "e", "a", "h"}
    assert set(G.edges()) == {
        ("d", "a"),
        ("e", "a"),
        ("f", "b"),
        ("g", "b"),
        ("h", "a"),
        ("h", "b"),
        ("i", "a"),
        ("i", "b"),
        ("j", "a"),
        ("j", "h"),
    }

def test_show_ascii(repo):
    assert [
        line.rstrip() for line in _show_ascii(repo.graph).splitlines()
    ] == [
        "                        +----------------+                          +----------------+",  # noqa: E501
        "                        | stage: 'a.dvc' |                          | stage: 'b.dvc' |",  # noqa: E501
        "                       *+----------------+****                      +----------------+",  # noqa: E501
        "                  *****           *           *****                  ***           ***",  # noqa: E501
        "              ****                *                *****           **                 **",  # noqa: E501
        "           ***                     *                    ***      **                     **",  # noqa: E501
        "+------------+                     **                   +------------+              +------------+",  # noqa: E501
        "| stage: '1' |                       **                 | stage: '3' |              | stage: '2' |",  # noqa: E501
        "+------------+                         ***              +------------+              +------------+",  # noqa: E501
        "                                          **           ***",
        "                                            **       **",
        "                                              **   **",
        "                                          +------------+",
        "                                          | stage: '4' |",
        "                                          +------------+",
    ]

def test_show_dot(repo):
    assert _show_dot(repo.graph) == (
        "strict digraph  {\n"
        "stage;\n"
        "stage;\n"
        "stage;\n"
        "stage;\n"
        "stage;\n"
        "stage;\n"
        "\"stage: '1'\" -> \"stage: 'a.dvc'\";\n"
        "\"stage: '2'\" -> \"stage: 'b.dvc'\";\n"
        "\"stage: '3'\" -> \"stage: 'a.dvc'\";\n"
        "\"stage: '3'\" -> \"stage: 'b.dvc'\";\n"
        "\"stage: '4'\" -> \"stage: 'a.dvc'\";\n"
        "\"stage: '4'\" -> \"stage: '3'\";\n"
        "}\n"
    )


import json
import os

import numpy as np
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow.keras.optimizers.schedules import ExponentialDecay

from ..layers.frcnn_layers import RoIDelta
from ..metrics import mAP
from ..utils import train_utils, log_utils

class Trainer:
    """
    Class to train DAROD model.
    """
    def __init__(self, config, model, labels, experiment_name, backbone,
                 backup_dir="/home/nxf67149/Documents/codes/DAROD/saves/"):
        """

        :param config: configuration dictionary with training settings
        :param model: the model to train
        :param labels (list): class labels
        :param experiment_name: name of the experiment
        :param backbone: backbone to use (DAROD or vision based one)
        :param backup_dir: directory to save logs
        """
        self.config = config
        self.eval_every = self.config["training"]["eval_every"]
        self.lr = self.config["training"]["lr"]
        optimizer = self.config["training"]['optimizer']
        self.scheduler = self.config["training"]["scheduler"]
        momentum = self.config["training"]["momentum"]
        self.variances_boxes = self.config["fastrcnn"]["variances_boxes"]
        self.total_labels = self.config["data"]["total_labels"]
        self.frcnn_num_pred = self.config["fastrcnn"]["frcnn_num_pred"]
        self.box_nms_score = self.config["fastrcnn"]['box_nms_score']
        self.box_nms_iou = self.config["fastrcnn"]['box_nms_iou']
        #
        decay_step = self.config["training"]["scheduler_step"]
        if self.scheduler:
            self.lr = ExponentialDecay(self.lr, decay_rate=0.9, staircase=True,
                                       decay_steps=(config["training"]["num_steps_epoch"] / config["training"][
                                           "batch_size"]) * decay_step)

        if optimizer == "SGD":
            self.optimizer = tf.optimizers.SGD(learning_rate=self.lr, momentum=momentum)
        elif optimizer == "WSGD":
            self.optimizer = tfa.optimizers.SGDW(learning_rate=self.lr, momentum=momentum, weight_decay=0.0005)
        elif optimizer == "adam":
            self.optimizer = tf.optimizers.Adam(learning_rate=self.lr)
        elif optimizer == "adad":
            self.optimizer = tf.optimizers.Adadelta(learning_rate=1.0)
        elif optimizer == "adag":
            self.optimizer = tf.optimizers.Adagrad(learning_rate=self.lr)
        else:
            raise NotImplemented("Not supported optimizer {}".format(optimizer))
        #
        self.experience_name = experiment_name
        self.backbone = backbone
        #
        self.labels = labels
        self.n_epochs = self.config["training"]["epochs"]
        self.model = model

        self.global_step = tf.Variable(0, trainable=False, dtype=tf.int64)

        self.backup_dir = os.path.join(backup_dir, experiment_name)
        if not os.path.exists(self.backup_dir):
            os.makedirs(self.backup_dir)
        with open(os.path.join(self.backup_dir, 'config.json'), 'w') as file:
            json.dump(config, file, indent=2)
        # Tensorboard setting
        self.summary_writer = tf.summary.create_file_writer(self.backup_dir)

        # Checkpoint manager
        self.ckpt = tf.train.Checkpoint(optimizer=self.optimizer, model=self.model, step=self.global_step)
        self.manager = tf.train.CheckpointManager(self.ckpt, self.backup_dir, max_to_keep=5)
        self.ckpt.restore(self.manager.latest_checkpoint)

        if self.manager.latest_checkpoint:
            print("Restored from {}".format(self.manager.latest_checkpoint))
            self.global_step.assign(self.ckpt.step.numpy())

    def init_metrics(self):
        """
        Initialise metrics for training
        :return:
        """
        self.train_loss = tf.keras.metrics.Mean(name="train_loss")
        self.train_rpn_reg_loss = tf.keras.metrics.Mean(name="train_rpn_reg_loss")
        self.train_rpn_cls_loss = tf.keras.metrics.Mean(name="train_rpn_cls_loss")
        self.train_frcnn_reg_loss = tf.keras.metrics.Mean(name="train_frcnn_reg_loss")
        self.train_frcnn_cls_loss = tf.keras.metrics.Mean(name="train_frcnn_cls_loss")

        self.val_loss = tf.keras.metrics.Mean(name="val_loss")
        self.val_rpn_reg_loss = tf.keras.metrics.Mean(name="val_rpn_reg_loss")
        self.val_rpn_cls_loss = tf.keras.metrics.Mean(name="val_rpn_cls_loss")
        self.val_frcnn_reg_loss = tf.keras.metrics.Mean(name="val_frcnn_reg_loss")
        self.val_frcnn_cls_loss = tf.keras.metrics.Mean(name="val_frcnn_cls_loss")

    def reset_all_metrics(self):
        """
        Reset all metrics
        :return:
        """
        self.train_loss.reset_states()
        self.train_rpn_reg_loss.reset_states()
        self.train_rpn_cls_loss.reset_states()
        self.train_frcnn_reg_loss.reset_states()
        self.train_frcnn_cls_loss.reset_states()
        self.val_loss.reset_states()
        self.val_rpn_reg_loss.reset_states()
        self.val_rpn_cls_loss.reset_states()
        self.val_frcnn_reg_loss.reset_states()
        self.val_frcnn_cls_loss.reset_states()

    def update_train_metrics(self, loss, rpn_reg_loss, rpn_cls_loss, frcnn_reg_loss, frcnn_cls_loss):
        """
        Update train metrics
        :param loss: total loss
        :param rpn_reg_loss: rpn regression loss
        :param rpn_cls_loss: rpn classification loss
        :param frcnn_reg_loss: fast rcnn regression loss
        :param frcnn_cls_loss: fast rcnn classification loss
        :return:
        """
        self.train_loss(loss)
        self.train_rpn_reg_loss(rpn_reg_loss)
        self.train_rpn_cls_loss(rpn_cls_loss)
        self.train_frcnn_reg_loss(frcnn_reg_loss)
        self.train_frcnn_cls_loss(frcnn_cls_loss)

    def update_val_metrics(self, loss, rpn_reg_loss, rpn_cls_loss, frcnn_reg_loss, frcnn_cls_loss):
        """
        Update val metrics
        :param loss: total loss
        :param rpn_reg_loss: rpn regression loss
        :param rpn_cls_loss: rpn classification loss
        :param frcnn_reg_loss: fast rcnn regression loss
        :param frcnn_cls_loss: fast rcnn classification loss
        :return:
        """
        self.val_loss(loss)
        self.val_rpn_reg_loss(rpn_reg_loss)
        self.val_rpn_cls_loss(rpn_cls_loss)
        self.val_frcnn_reg_loss(frcnn_reg_loss)
        self.val_frcnn_cls_loss(frcnn_cls_loss)

    def train_step(self, input_data, anchors, epoch):
        """
        Performs one training step (forward pass + optimisation)
        :param input_data: RD spectrum
        :param anchors: anchors
        :param epoch: epoch number
        :return:
        """
        spectrums, gt_boxes, gt_labels, is_same_seq, _, _ = input_data

        # Take only valid idxs
        valid_idxs = tf.where(is_same_seq == 1)
        spectrums, gt_boxes, gt_labels = tf.gather_nd(spectrums, valid_idxs), tf.gather_nd(gt_boxes,
                                                                                valid_idxs), tf.gather_nd(
            gt_labels, valid_idxs)

        if spectrums.shape[0] != 0:

            # Get RPN labels
            bbox_deltas, bbox_labels = train_utils.calculate_rpn_actual_outputs(anchors, gt_boxes, gt_labels,
                                                                                self.config)

            # Train step
            with tf.GradientTape() as tape:
                rpn_cls_pred, rpn_delta_pred, frcnn_cls_pred, frcnn_reg_pred, roi_bboxes, _ = self.model(spectrums,
                                                                                training=True)

                frcnn_reg_actuals, frcnn_cls_actuals = RoIDelta(self.config)([roi_bboxes, gt_boxes, gt_labels])
                rpn_reg_loss, rpn_cls_loss, frcnn_reg_loss, frcnn_cls_loss = train_utils.darod_loss(rpn_cls_pred,
                                                                                rpn_delta_pred,
                                                                                frcnn_cls_pred,
                                                                                frcnn_reg_pred,
                                                                                bbox_labels,
                                                                                bbox_deltas,
                                                                                frcnn_reg_actuals,
                                                                                frcnn_cls_actuals)

            losses = rpn_reg_loss + rpn_cls_loss + frcnn_reg_loss + frcnn_cls_loss
            self.update_train_metrics(losses, rpn_reg_loss, rpn_cls_loss, frcnn_reg_loss, frcnn_cls_loss)

            gradients = tape.gradient([rpn_reg_loss, rpn_cls_loss, frcnn_reg_loss, frcnn_cls_loss],
                                      self.model.trainable_variables)
            self.optimizer.apply_gradients(zip(gradients, self.model.trainable_variables))
            if self.scheduler:
                cur_lr = self.optimizer.lr(self.global_step)

            print("===> ite(epoch) {}({})/{}({})  <==> train loss {} <==> rpn_reg_loss {} <==> rpn_cls_loss {} \ "
                  "<==> frcnn_reg_loss {} <==> frcnn_cls_loss {}".format(self.global_step.value().numpy(), epoch,
                                                                         self.total_ite, self.n_epochs, losses.numpy(),
                                                                         rpn_reg_loss.numpy(), rpn_cls_loss.numpy(),
                                                                         frcnn_reg_loss.numpy(),
                                                                         frcnn_cls_loss.numpy()))

            with self.summary_writer.as_default():
                tf.summary.scalar("train_loss/total_loss", losses, step=self.global_step)
                tf.summary.scalar("train_loss/rpn_reg_loss", rpn_reg_loss, step=self.global_step)
                tf.summary.scalar("train_loss/rpn_cls_loss", rpn_cls_loss, step=self.global_step)
                tf.summary.scalar("train_loss/frcnn_reg_loss", frcnn_reg_loss, step=self.global_step)
                tf.summary.scalar("train_loss/frcnn_cls_loss", frcnn_cls_loss, step=self.global_step)
                if not self.scheduler:
                    tf.summary.scalar("lr", self.optimizer.learning_rate.numpy(), step=self.global_step)
                else:
                    tf.summary.scalar("lr", cur_lr.numpy(), step=self.global_step)

            if np.isnan(losses):
                raise ValueError("Get NaN at iteration ", self.ite)
            self.global_step.assign_add(1)

    def val_step(self, input_data, anchors):
        """
        Performs one validation step
        :param input_data: RD spectrum
        :param anchors: anchors
        :return: ground truth boxes, labels and predictions
        """
        spectrums, gt_boxes, gt_labels, is_same_seq, _, _ = input_data

        # Take only valid idxs
        valid_idxs = tf.where(is_same_seq == 1)
        spectrums, gt_boxes, gt_labels = tf.gather_nd(spectrums, valid_idxs), tf.gather_nd(gt_boxes,
                                                                                valid_idxs), tf.gather_nd(
            gt_labels, valid_idxs)

        if spectrums.shape[0] != 0:
            # Get RPN labels
            bbox_deltas, bbox_labels = train_utils.calculate_rpn_actual_outputs(anchors, gt_boxes, gt_labels,
                                                                                self.config)
            rpn_cls_pred, rpn_delta_pred, frcnn_cls_pred, frcnn_reg_pred, roi_bboxes, decoder_output = self.model(
                spectrums, training=True)

            if self.config["fastrcnn"]["reg_loss"] == "sl1":
                # Smooth L1-loss
                frcnn_reg_actuals, frcnn_cls_actuals = RoIDelta(self.config)([roi_bboxes, gt_boxes, gt_labels])
                rpn_reg_loss, rpn_cls_loss, frcnn_reg_loss, frcnn_cls_loss = train_utils.darod_loss(rpn_cls_pred,
                                                                                rpn_delta_pred,
                                                                                frcnn_cls_pred,
                                                                                frcnn_reg_pred,
                                                                                bbox_labels,
                                                                                bbox_deltas,
                                                                                frcnn_reg_actuals,
                                                                                frcnn_cls_actuals)
            elif self.config["fastrcnn"]["reg_loss"] == "giou":
                # Generalized IoU loss
                expanded_roi_bboxes, expanded_gt_boxes, frcnn_cls_actuals = RoIDelta(self.config)(
                    [roi_bboxes, gt_boxes, gt_labels])
                rpn_reg_loss, rpn_cls_loss, frcnn_reg_loss, frcnn_cls_loss = train_utils.darod_loss_giou(rpn_cls_pred,
                                                                                rpn_delta_pred,
                                                                                frcnn_cls_pred,
                                                                                frcnn_reg_pred,
                                                                                bbox_labels,
                                                                                bbox_deltas,
                                                                                expanded_gt_boxes,
                                                                                frcnn_cls_actuals,
                                                                                expanded_roi_bboxes)

            losses = rpn_reg_loss + rpn_cls_loss + frcnn_reg_loss + frcnn_cls_loss
            self.update_val_metrics(losses, rpn_reg_loss, rpn_cls_loss, frcnn_reg_loss, frcnn_cls_loss)
            return gt_boxes, gt_labels, decoder_output
        else:
            return gt_boxes, gt_labels, None

    def train(self, anchors, train_dataset, val_dataset):
        """
        Train the model
        :param anchors: anchors
        :param train_dataset: train dataset
        :param val_dataset: validation dataset
        :return:
        """
        best_loss = np.inf
        self.ite = 0
        self.total_ite = int(
            (self.config["training"]["num_steps_epoch"] / self.config["training"]["batch_size"]) * self.n_epochs)
        self.init_metrics()

        start_epoch = int(self.global_step.numpy() / (
                    self.config["training"]["num_steps_epoch"] / self.config["training"]["batch_size"]))

        for epoch in range(start_epoch, self.n_epochs):

            # Reset metrics
            self.reset_all_metrics()
            # Perform one train step
            for input_data in train_dataset:
                self.train_step(input_data, anchors, epoch)

            # Testing phase
            if (epoch % self.eval_every == 0 and epoch != 0) or epoch == self.n_epochs - 1:
                tp_dict = mAP.init_tp_dict(len(self.labels) - 1, iou_threshold=[0.5])

            # Validation phase
            for input_data in val_dataset:
                gt_boxes, gt_labels, decoder_output = self.val_step(input_data, anchors)

                if decoder_output is not None and (
                        epoch % self.eval_every == 0 and epoch != 0) or epoch == self.n_epochs - 1:
                    pred_boxes, pred_labels, pred_scores = decoder_output
                    pred_labels = pred_labels - 1
                    gt_labels = gt_labels - 1
                    for batch_id in range(pred_boxes.shape[0]):
                        tp_dict = mAP.accumulate_tp_fp(pred_boxes.numpy()[batch_id], pred_labels.numpy()[batch_id],
                                                       pred_scores.numpy()[batch_id], gt_boxes.numpy()[batch_id],
                                                       gt_labels.numpy()[batch_id], tp_dict,
                                                       iou_thresholds=[0.5])

            if (epoch % self.eval_every == 0 and epoch != 0) or epoch == self.n_epochs - 1:
                ap_dict = mAP.AP(tp_dict, n_classes=len(self.labels) - 1)
                log_utils.tensorboard_val_stats(self.summary_writer, ap_dict, self.labels[1:], self.global_step)
                if self.val_loss.result().numpy() < best_loss:
                    best_loss = self.val_loss.result().numpy()
                    self.model.save_weights(os.path.join(self.backup_dir, "best-model.h5"))
                    self.config["training"]["best_epoch"] = epoch

            # Log metrics
            with self.summary_writer.as_default():
                tf.summary.scalar("val_loss/total_loss", self.val_loss.result().numpy(), step=self.global_step)
                tf.summary.scalar("val_loss/rpn_reg_loss", self.val_rpn_reg_loss.result().numpy(),
                                  step=self.global_step)
                tf.summary.scalar("val_loss/rpn_cls_loss", self.val_rpn_cls_loss.result().numpy(),
                                  step=self.global_step)
                tf.summary.scalar("val_loss/frcnn_reg_loss", self.val_frcnn_reg_loss.result().numpy(),
                                  step=self.global_step)
                tf.summary.scalar("val_loss/frcnn_cls_loss", self.val_frcnn_cls_loss.result().numpy(),
                                  step=self.global_step)

            print("<=============== Validation ===============>")
            print("===> epoch {}/{}  <==> val loss {} <==> val rpn_reg_loss {} <==> val rpn_cls_loss {}"
                  " <==> val frcnn_reg_loss {} <==> val frcnn_cls_loss {}".format(
                epoch, self.n_epochs, self.val_loss.result().numpy(), self.val_rpn_reg_loss.result().numpy(),
                self.val_rpn_cls_loss.result().numpy(), self.val_frcnn_reg_loss.result().numpy(),
                self.val_frcnn_cls_loss.result().numpy()))
            print("<==========================================>")

            save_path = self.manager.save()
            print("Saved checkpoint for step {}: {}".format(int(self.ckpt.step), save_path))


'''
Contains tests for `CardSet` and `CardSetCollection` objects.
'''

import datetime

from fab import CardSet, CardSetCollection

from . import (
    C1,
    C2,
    C3,
    CS1,
    CSC1
)

def test_set_dict_rep():
    '''
    Tests the dictionary representation of `CardSet` objects.
    '''
    assert CS1['name'] == 'Monarch'
    assert set(CS1.keys()) == set([
        'editions',
        'identifier',
        'name',
        'release_date'
    ])
    assert dict(CS1) == {
        'editions': ['F', 'U'],
        'identifier': 'MON',
        'name': 'Monarch',
        'release_date': datetime.date(2021, 5, 7)
    }

def test_set_json_conversion():
    '''
    Tests the `CardSet.to_json()` and `CardSet.from_json()` methods.
    '''
    CS1_json = CS1.to_json()
    CS1_str = str(CS1)
    CS1_from_json = CardSet.from_json(CS1_json)
    assert CS1_json == CS1_str
    assert CS1_from_json == CS1

def test_set_values():
    '''
    Tests card set value via "dot" syntax.
    '''
    assert CS1.editions     == ['F', 'U']
    assert CS1.identifier   == 'MON'
    assert CS1.name         == 'Monarch'
    assert CS1.release_date == datetime.date(2021, 5, 7)

def test_set_collection_collections():
    '''
    Tests collection methods on `CardSetCollection` objects.
    '''
    assert set(CSC1.editions())      == set(['F', 'U', 'A', 'N'])
    assert set(CSC1.identifiers())   == set(['MON', 'WTR', 'UPR'])
    assert set(CSC1.names())         == set(['Monarch', 'Welcome to Rathe', 'Uprising'])
    assert set(CSC1.release_dates()) == set([
        datetime.date(2021, 5, 7), datetime.date(2019, 10, 11), datetime.date(2022, 6, 24)
    ])

def test_set_collection_get_release_date():
    '''
    Tests `CardSetCollection.get_release_date()`
    '''
    assert CSC1.get_release_date(C1) == datetime.date(2019, 10, 11)
    assert CSC1.get_release_date(C2) == datetime.date(2019, 10, 11)
    assert CSC1.get_release_date(C3) == datetime.date(2021, 5, 7)

def test_set_collection_json_conversion():
    '''
    Tests the `CardSetCollection.to_json()` and `CardSetCollection.from_json()`
    methods.
    '''
    CSC1_json = CSC1.to_json()
    CSC1_from_json = CardSetCollection.from_json(CSC1_json)
    assert CSC1_from_json == CSC1


"""Utility methods for manipulating python-type collections of data."""

# Utility function(s)
import json
from typing import Any, Dict, Iterable, List, Tuple, Union

import pandas as pd

def flatten_nested_dictionary(
    obj: Union[Dict, Any],
    parent_key: str = "",
    separator: str = "__",
) -> Union[Dict, Any]:
    """Flatten a nested dictionary to a dictionary with non-dict values.

    Example:
        d = {"a": {"b": 1}, "c": 2}
        flatten_nested_dictionary(d)
        >>> {"a__b": 1, "c": 2}

    Args:
        obj: The object that should be flattened, if applicable.
        parent_key: The combined name of the parent key(s) containing `obj`.
        separator: The string used to concatenate nester parent keys.
    """
    # Dict-like
    if isinstance(obj, dict):
        items: List[Tuple[str, Any]] = []
        for key, value in obj.items():
            new_key = parent_key + separator + key if parent_key else key
            result = flatten_nested_dictionary(value, new_key, separator)
            items.extend(result.items())

        return dict(items)

    # Other non-dict like (list, str, float, int, etc.)
    else:
        return {parent_key: obj}

def serialise(obj: Union[Dict, Any]) -> Union[Dict, Any]:
    """Serialise the necessary keys in `obj` to JSON for saving to file.

    It is typically not possible to save nested collections (lists, dicts) to
    file. Therefore, if `obj` is, e.g., a list of lists, we need to serialise
    each element in the outer list to JSON (i.e., convert it to a JSON-
    formatted string) in order to be able to save `obj` to file. It will then
    be possible to de-serialise corresponding elements when reading them from
    file.
    """
    if isinstance(obj, dict):
        for key in list(obj.keys()):
            value = obj[key]
            if (
                isinstance(value, (list, tuple))
                and len(value)
                and isinstance(value[0], Iterable)
            ):
                obj[key] = [json.dumps(element) for element in value]

    elif isinstance(obj, (list, tuple)):
        obj = [json.dumps(element) for element in obj]

    return obj

def transpose_list_of_dicts(
    array: List[Dict[str, Any]]
) -> Dict[str, List[Any]]:
    """Transpose a list of dicts to a dict of lists."""
    if len(array) == 0:
        return {}

    all_keys = [key for d in array for key in d.keys()]
    keys = pd.unique(all_keys)  # Returns keys in order of appearance

    transposed_dict = {
        key: [element.get(key, None) for element in array] for key in keys
    }
    return transposed_dict


from flask import render_template, redirect, url_for, Blueprint
from flask_login import current_user, login_required
from app.forms.coins import AddForm, DeleteForm
from app.models import db
from app.models.coin import Coin
from app.models.daily_coins import DailyCoins
from app.views_class.Create import Create
from app.views_class.Delete import Delete
from app.views_class.Read import Read
from app.views_class.Show import Show

coins = Blueprint('coin', __name__)

@coins.route('/')
def list_coins():
    """
    The list_coins function is used to display the list of coins that are available for
    the user to choose from. It also displays the amount of money that is currently in their account if he is authenticated.
    :return: The list of coins that the user has saved in his account if he is authenticated or a liste of the coins save in the last dy if he is not
    """
    if current_user.is_authenticated:
        coins, data, all_benef = Read(current_user).user_coins()
        return render_template('coins/list_coins.html', data=data, coins=coins, all_benef=all_benef)
    else:
        data = DailyCoins().get_last_save()
        return render_template('home/index.html', data=data)

@coins.route('/coin/add', methods=['GET', 'POST'])
@login_required
def add_coin():
    """
    The add_coin function allows the user to add a new coin in the database.
    It uses an AddForm object which is defined in forms.py and contains all fields needed for adding a new coin.
    :return: A render_template object that contains the html page add_coins
    """
    form = AddForm()
    if form.validate_on_submit():
        return Create(db, current_user, form).new_coins()
    return render_template('coins/add_coins.html', form=form, title='Ajouter une monnaie')

@coins.route('/coin-delete-value', methods=['GET', 'POST'])
@login_required
def delete_value_coin():
    """
    The delete_value_coin function is used to delete a coin from the database.
    It takes no parameters
    :return: The template delete_coin_value or a redirect if the form is valid and submited
    """
    form = DeleteForm()
    delete = Delete(current_user, db, form)
    delete.set_up()
    if form.validate_on_submit():
        errors, coins_quantity, success_redirect = delete.delete_coins()
        if success_redirect == True:
            return redirect(url_for('coin.list_coins'))
        elif success_redirect == False:
            return render_template('coins/delete_coin_value.html', coins_quantity=coins_quantity,
                                   title='Mes monnaies', form=form, errors=errors)
    coins_quantity = delete.view_delete_coins()
    return render_template('coins/delete_coin_value.html', coins_quantity=coins_quantity,
                           title='Mes monnaies', form=form)

@coins.route('/coin/<int:coin_id>/detail', methods=['GET'])
@login_required
def detail_coin(coin_id):
    """
    The detail_coin function is used to display the details of a coin.
    It takes as input the id of a coin and returns its graph with the help of matplotlip Figure .
    :param coin_id: Get the coin_id of the coin selected in the list_coins
    :return: The template coins/detail_coins
    """
    coin = Coin.query.get_or_404(coin_id)
    daily = DailyCoins().get_week_save()
    if coin.user_id == current_user.id:
        show = Show(coin, daily)
        dataurl = show.figure_gain_value()
        return render_template('coins/detail_coins.html', coin=coin, title='Détails de la monnaie', img=dataurl)
    else:
        return redirect(url_for('coin.list_coins'))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Nov  4 13:50:55 2019

@author: joel
En sammenligning av pygfunction, den uendelige linjesluk-løsningen og den
den uendelige sylindersluk-løsningen for et borefelt bestående av ett borehull
er gjort. Det er antatt uniform og kontinuerlig varmeuttak langs hele borehullet.

Den uendelige linjeslukløsningen er hentet fra Carslaw og Jaeger sin løsning
fra 1947.

Koden retunerer:
    1) Temperaturprofilen ved borehullveggen for de tre modellene beskrevet
       ovenfor, for et borefelt bestående av ett borehull.
    2) Forskjellen i temperatur ved borehullveggen for de ulike modellene.
    3) En graf som viser hvilket år den dimensjonsløse logaritmiske tidsskalaen
       viser til.
"""

from __future__ import absolute_import, division, print_function

from matplotlib.legend_handler import HandlerLine2D
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.ticker import AutoMinorLocator
import pygfunction as gt
from scipy import integrate
import math as m
import scipy.special as sp
import timeit

# tidtaker
start = timeit.default_timer()

#------------------------------------------------------------------------------
# Parametere i simuleringen
#------------------------------------------------------------------------------

# Antall plottpunkter
n = 100

# Dimensjonering av borehull
H = 160         # Borehulets lengde (m)
D = 5           # Borehullets avstand fra overflaten (m)
r_b = 0.1       # Borehullets radius (m)
x = 0           # Borehullhodets posisjon langs x - aksen
y = 0           # Borehullhodets posisjon langs y - aksen

#

# egenskaper til grunnen
T_g = 6                         # uforstyrret grunntemperatur['C]
k_g = 1                         # konduktivitet i grunnen [W/mK]
a_g = 10**-6                    # diffusivitet i grunnen [m^2/s]
q = 2*np.pi*k_g                 # uniform varmeuttømming fra grunnen [W/m]
r = 20                          # radiell størrelse til grunnen [m]

# Geometrisk ekspanderende tidsvektor
dt = 3600                       # Tidssteg [sec]
aar = 60*60*24*365              # 1. år [sec]
tmaks = aar*3000                # Simuleringsperiode [sec]
Nt = n                          # Ant. tidssteg
tid = gt.utilities.time_geometric(dt,tmaks,Nt)
t_s = H**2/(9*a_g)              # Borefeltets karakteristiske tid
tid_skala = np.log(tid/t_s)     # dimensjonsløs logaritmisk tidsskala

# Tabell med simuleringsparametere
print()
print('Følgende parametere er brukt i simuleringen:')
print()
from tabulate import tabulate
print(tabulate([['q', r'$2\pi \lambda$'], ['H', '160']],
               headers=['Parameter', 'Verdi'], tablefmt='orgtbl'))
print()

#------------------------------------------------------------------------------
# Definerer borehullet
#------------------------------------------------------------------------------

b =[gt.boreholes.Borehole(H, D, r_b, x, y)]

try:
    # Visualisering av borehull
    gt.boreholes.visualize_field(b)
except Exception:
    pass

#------------------------------------------------------------------------------
# Definerer koordinatsystem
#------------------------------------------------------------------------------

plt.rc('figure')
fig = plt.figure()
ax1 = fig.add_subplot(111)
# Navn på aksene
ax1.set_xlabel(r'$ln(t/t_s)$')
ax1.set_ylabel(r'Temp. ved borehullvegg[($^\circ$C)]')
# aksegrenser
ax1.set_xlim([-14.0, 5])
ax1.set_ylim([0.,30.])
# ticks
ax1.xaxis.set_minor_locator(AutoMinorLocator())
ax1.yaxis.set_minor_locator(AutoMinorLocator())
#grid
ax1.grid(color = 'k', linestyle='--', linewidth=0.1)
# Justering av plottvindu
plt.tight_layout()

#------------------------------------------------------------------------------
# Beregner g-funksjonen og temperaturprofilen ved borehullveggen
#------------------------------------------------------------------------------

for field in [b]:
    gfunk = gt.gfunction.uniform_heat_extraction(b,tid, a_g, disp=True)

    # Temperatur i borehullveggen
    T_b_pyg = T_g + q/(2*np.pi*k_g)*gfunk

#------------------------------------------------------------------------------
# Evaluerer den uendelige linjesluk - løsningen
#------------------------------------------------------------------------------

# Diverse matriser
s = (n,2)
u = np.zeros(n)             # Nedre grense i integralet i ULS-løsningen
I_int = np.zeros(s)         # Verdier av integralet i ULS-løsningen
T_b_uls = np.zeros(s)       # Temperatur ved borehullveggen

# Evaluerer ULS-løsnningen
for i in range(0,n):
    # verdier for u=r_b^2/4a_gt
    u[i] = r_b**2/(4*a_g*tid[i])

    # integrasjon av integralet i ILS-løsningen
    I = lambda x: (np.exp(-x))/x
    I_int[i] = integrate.quad(I,u[i],m.inf)

    #verdier for temperaturen ved borehullveggen
    T_b_uls[i] = T_g + (q/(2*m.pi*k_g))*I_int[i]

#------------------------------------------------------------------------------
# Evaluerer sylindersluk-løsningen
#------------------------------------------------------------------------------

Fo = np.zeros(n)        # Fourier-tallet
G = np.zeros(s)         # Sylindersluk-funksjonen
T_b_uss = np.zeros(s)   # Temperatur ved borehullveggen for uss løsningen

for i in range(n):
    # Fouriertallet
    Fo[i] = (a_g/(r_b**2))*tid[i]

    # Beregner integralet i sylindersluk-funksjonen
    G[i] = integrate.quad(lambda h:((np.exp(-(h**2)*Fo[i]))-1)*((sp.j0(h)*sp.y1(h)
    - sp.y0(h)*sp.j1(h))/((h**2)*((sp.j1(h)**2)+(sp.y1(h)**2)))),0,m.inf)

    # Temperatur ved borehullveggen
    T_b_uss[i] = T_g + q/(k_g*(np.pi**2))*G[i]

#------------------------------------------------------------------------------
# Temperaturprofilen ved borehullveggen for de ulike modellene
#------------------------------------------------------------------------------

line1, = ax1.plot(tid_skala, T_b_pyg,'k-', lw=1.5, label='Pygfunction' )
line2, = ax1.plot(tid_skala, T_b_uls[:,0], 'r-', lw=1.5, label='u_linjesluk')
line3, = ax1.plot(tid_skala, T_b_uss[:,0], 'y-', lw=1.5, label='u_sylindersluk')

plt.legend(handler_map={line3: HandlerLine2D(numpoints=4)}, loc=4)
plt.show()

#------------------------------------------------------------------------------
# Temperaturdifferanse ved borehullveggen for de ulike modellene
#------------------------------------------------------------------------------

plt.rc('figure')
fig = plt.figure()
ax2 = fig.add_subplot(111)
# Tittel
ax2.set_title('Temperaturdifferanse ved borehullveggen')
# Navn på aksene
ax2.set_xlabel(r'$ln(t/t_s)$')
ax2.set_ylabel(r'Temperatur [($^\circ$C)]')
# aksegrenser
ax2.set_xlim([-14.0, 5])
ax2.set_ylim([0.,1.])
# ticks
ax2.xaxis.set_minor_locator(AutoMinorLocator())
ax2.yaxis.set_minor_locator(AutoMinorLocator())
#grid
ax2.grid(color = 'k', linestyle='--', linewidth=0.1)
# Justering av plottvindu
plt.tight_layout()

# pygfunction og sylindersluk-modellen
D_pyg_uss = np.zeros(n)
D_pyg_uls = np.zeros(n)
T_b_uls = T_b_uls[:,0]
T_b_uss = T_b_uss[:,0]

for i in range(n):
    D_pyg_uss[i] = -(T_b_pyg[i] - T_b_uss[i])
    D_pyg_uls[i] = T_b_uls[i] - T_b_pyg[i]

line1, = ax2.plot(tid_skala,D_pyg_uss, 'y-', lw=1.5, label='uss-pyg')
line2, = ax2.plot(tid_skala,D_pyg_uls, 'b-', lw=1.5, label='uls-pyg')

plt.legend(handler_map={line2: HandlerLine2D(numpoints=4)}, loc=1)
plt.show()

#------------------------------------------------------------------------------
# Graf som viser hvilket år verdien av log(tid/t_s) svarer til
#------------------------------------------------------------------------------

plt.rc('figure')
fig = plt.figure()
ax3 = fig.add_subplot(111)
# Navn på aksene
ax3.set_xlabel(r'$ln(t/t_s)$')
ax3.set_ylabel(r'År')
# aksegrenser
ax3.set_xlim([-7.0, -2.])
ax3.set_ylim([0., 10])
# ticks
ax3.xaxis.set_minor_locator(AutoMinorLocator())
ax3.yaxis.set_minor_locator(AutoMinorLocator())
#grid
ax3.grid(color = 'k', linestyle='--', linewidth=0.1)
# Justering av plottvindu
plt.tight_layout()

#Tegner graf
c = np.arange(-10, 3,step=0.2)
w = np.zeros(len(c))
for i in range(len(c)):
    w[i] = t_s*np.exp(c[i])
    w[i] = w[i]/(60*60*24*365)

ax3.plot(c,w)
ax3.grid(color = 'k', linestyle='--', linewidth=0.1)
plt.show()

# tidtaker
stop = timeit.default_timer()
print('Total beregningstid: ', stop - start, 'sec')


import cv2

img=cv2.imread("lena.jpg")

cv2.imshow("og",img)
layer=img.copy()
lst=[layer]
for i in range(5):
    layer=cv2.pyrDown(layer)
    lst.append(layer)

for i in range(5,0,-1):
    gp=cv2.pyrUp(lst[i])
    lp=cv2.subtract(lst[i-1],gp)
    cv2.imshow(str(i),lp)

cv2.waitKey(0)
cv2.destroyAllWindows()


'''
Python Variables:
This entry details how to utilize variables.
Variables are, in their most basic form, placeholders for values. In Python, initializing variables is made simple.
By assigning a value to a variable, a named reference is created that can be used to retrieve and modify the associated data.
'''

# Initializing Variables
number = 10 #int
name = "name" #string
float = 4.3 #float
is_alive = False #bool

# Assigning multiple values in one line.
a, b, c, d = (1, 2.5, 4, 5)

# Printing values.
print(a, b, c, d)

# Performing basic arithmetic.
a = a + 3 # Addition
b = b * 3 # Multiplication
c = c / 2 # Division
d = d - 4 # Subtraction

# Printing values.
print(a, b, c, d)