#! /usr/bin/env python3

import os
import sys
import subprocess

REPO_ROOT = os.path.dirname(os.path.realpath(__file__))
TESTS_DIR = os.path.join(REPO_ROOT, 'tests')

def get_untracked_files():
    output = subprocess.check_output([
        'git', 'ls-files', '--other', '--directory', '--exclude-standard', '-z'
    ],
        cwd=REPO_ROOT)
    return set(f for f in output.split(b'\0') if f)

def main():
    # Unset any PERU environment variables to make sure test runs don't get
    # thrown off by anything in your bashrc.
    for var in list(os.environ.keys()):
        if var.startswith('PERU_'):
            del os.environ[var]

    # Turn debugging features on for the asyncio library, if it's unset.
    if 'PYTHONASYNCIODEBUG' not in os.environ:
        os.environ['PYTHONASYNCIODEBUG'] = '1'

    # Make sure the tests don't create any garbage files in the repo. That
    # tends to happen when we accidentally run something in the current dir
    # that should be in a temp dir, and it's hard to track down when it does.
    old_untracked = get_untracked_files()

    # Run the actual tests.
    env = os.environ.copy()
    env['PYTHONPATH'] = REPO_ROOT
    args = sys.argv[1:]
    # We no longer run --with-coverage in CI, so this branch might bitrot over
    # time.
    if len(args) > 0 and '--with-coverage' in args:
        args.remove('--with-coverage')
        command_start = ['coverage', 'run']
    else:
        command_start = [sys.executable, '-W', 'all']
    command = command_start + ['-m', 'unittest'] + args
    try:
        subprocess.check_call(command, env=env, cwd=TESTS_DIR)
    except subprocess.CalledProcessError:
        print('ERROR: unit tests returned an error code', file=sys.stderr)
        sys.exit(1)

    new_untracked = get_untracked_files()
    if old_untracked != new_untracked:
        print(
            'Tests created untracked files:\n' + '\n'.join(
                f.decode() for f in new_untracked - old_untracked),
            file=sys.stderr)
        sys.exit(1)

    # Run the linter.
    try:
        subprocess.check_call(['flake8', 'peru', 'tests'], cwd=REPO_ROOT)
    except FileNotFoundError:
        print('ERROR: flake8 not found', file=sys.stderr)
        sys.exit(1)
    except subprocess.CalledProcessError:
        print('ERROR: flake8 returned an error code', file=sys.stderr)
        sys.exit(1)

if __name__ == '__main__':
    main()


import boto3
import botocore
import os
import random
import tempfile
from twython import Twython, TwythonError

BUCKET_NAME = 'YOU-BUCKET-NAME'
CONSUMER_KEY = 'TWITTER-APP-CONSUMER-KEY'
CONSUMER_SECRET = 'TWITTER-APP-CONSUMER-SECRET'
ACCESS_TOKEN = 'ACCESS-TOKEN'
ACCESS_SECRET = 'ACCESS-SECRET'

s3 = boto3.resource('s3')
bucket= s3.Bucket(BUCKET_NAME)

def lambda_handler(event, context):

    obj = str(random.randint(1, sum(1 for _ in bucket.objects.all())))
    KEY = obj + '.png'

    twitter = Twython(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_SECRET)

    tmp_dir = tempfile.gettempdir()
    path = os.path.join(tmp_dir, KEY)
    print("created directory at " + path)
    try:
        s3.Bucket(BUCKET_NAME).download_file(KEY, path)
        print('file moved to temp directory')
        with open(path, 'rb') as img:
            try:
                twit_resp = twitter.upload_media(media=img)
                twitter.update_status(status="The best of you is he who learns the Quran and teaches it. ~ Prophet Muhammad SAW" , media_ids=twit_resp['media_id'])
                print("image tweeted")
            except TwythonError as e:
                print(e)
    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == "404":
            print("The object does not exist.")
        else:
            raise


from PyQt5.QtWidgets import *

from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT as NavigationToolbar
from matplotlib.figure import Figure
import random
from pvi import *

class MplWidget(QWidget):

    global data
    def __init__(self, parent=None):
        QWidget.__init__(self, parent)

        #data = [random.random() for i in range(10)]

        self.canvas = FigureCanvas(Figure())
        self.toolbar = NavigationToolbar(self.canvas, self)
        vertical_layout = QVBoxLayout()
        vertical_layout.addWidget(self.toolbar)
        vertical_layout.addWidget(self.canvas)

        self.canvas.axes = self.canvas.figure.add_subplot(111)
        self.setLayout(vertical_layout)
        #self.canvas.axes.plot(data, '*-')

    def recalculateData(self,idex, funcion, xi, ti, tf, h):

        self.canvas.axes.clear()
        if (idex == 0):
            MplWidget.data = euler(xi, ti, tf, h, funcion)
            self.canvas.axes.plot(MplWidget.data[0], MplWidget.data[1], '*-')
        if (idex == 1):
            MplWidget.data = euler_mejorado(xi, ti, tf, h, funcion)
            self.canvas.axes.plot(MplWidget.data[0], MplWidget.data[1], '*-', label='Predicto')
            self.canvas.axes.plot(MplWidget.data[0], MplWidget.data[2], '*-', label='Corrector')
            leg = self.canvas.axes.legend(loc="upper left")
            #leg = self.canvas.axes.legend(loc=2, bbox_to_anchor=(1.05, 1.0))
        if (idex == 2):
            MplWidget.data = rungeKutta(xi, ti, tf, h, funcion)
            self.canvas.axes.plot(MplWidget.data[0], MplWidget.data[1], '*-')

        self.canvas.draw()


import tensorflow as tf
import math

from tensorflow.keras.initializers import VarianceScaling
from tensorflow.keras.layers import Layer

class Conv2D_CD(Layer):
    def __init__(self, filters, kernel_size=3, strides=1,
                 padding='SAME', use_bias=False, theta=0.7):
        super(Conv2D_CD, self).__init__()
        self.filters = filters
        self.kernel_size = kernel_size
        self.strides = strides
        self.padding = padding
        self.theta = theta
        self.use_bias = use_bias
        self.kernel_initializer = VarianceScaling(scale=2.0)

    def build(self, input_shape):
        self._filter = tf.compat.v1.get_variable(name='conv2d_cd', shape=[
                                                 self.kernel_size, self.kernel_size, input_shape[-1], self.filters], initializer=self.kernel_initializer)

    def call(self, inputs):
        out_normal = tf.nn.conv2d(inputs, self._filter, strides=[
                                  1, self.strides, self.strides, 1], padding=self.padding, name='conv2d_cd/normal')
        if math.fabs(self.theta - 0.0) < 1e-8:
            return out_normal
        kernel_diff = tf.math.reduce_sum(self._filter, axis=0, keepdims=True)
        kernel_diff = tf.math.reduce_sum(kernel_diff, axis=1, keepdims=True)
        kernel_diff = tf.tile(
            kernel_diff, [self.kernel_size, self.kernel_size, 1, 1])
        out_diff = tf.nn.conv2d(inputs, kernel_diff, strides=[
                                1, self.strides, self.strides, 1], padding=self.padding, name='conv2d_cd/diff')

        return out_normal - self.theta * out_diff


from functools import wraps

def arg_parser(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        n = args[0].split('\n')
        for idx, el in enumerate(n):
            n[idx] = float(el) if '.' in el else int(el)
        n.extend(args[1:])
        return func(*n, **kwargs)
    return wrapper


import os
import sys
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import pickle as pkl
import time
from datetime import timedelta

MAX_VOCAB_SIZE = 10000
UNK, PAD = '<UNK>', '<PAD>'

class TextRCNNPredict:
    def __init__(self):

        self.config = Config()
        self.model = Model(self.config)
        self.use_word = False
        self._common_config()

    def _common_config(self):
        np.random.seed(1)
        torch.manual_seed(1)
        torch.cuda.manual_seed_all(1)
        torch.backends.cudnn.deterministic = True

    def _sentence_tokenizer(self,sentence1,sentence2):
        if self.use_word:
            tokenizer = lambda x: x.split(' ')
        else:
            tokenizer = lambda x: [y for y in x]

        vocab = pkl.load(open(self.config.vocab_path, 'rb'))
        words_line = []
        sentence = sentence1 + "&&" + sentence2

        token = tokenizer(sentence)
        seq_len = len(token)
        if self.config.pad_size:
            if len(token) < self.config.pad_size:
                token.extend([PAD] * (self.config.pad_size - len(token)))
            else:
                token = token[:self.config.pad_size]
                seq_len = self.config.pad_size

        for word in token:
            words_line.append(vocab.get(word, vocab.get(UNK)))

        return [(words_line,1,seq_len)]

    def predict(self,sentence1,sentence2):

        self.model.load_state_dict(torch.load(self.config.save_path))
        self.model.to(self.config.device)
        self.model.eval()

        sentence = self._sentence_tokenizer(sentence1,sentence2)
        data_iterator = DatasetIterater(sentence,1,self.config.device)

        with torch.no_grad():
            for texts, labels in data_iterator:
                outputs = self.model(texts)
                prob = outputs.softmax(-1).cpu().numpy()
                predict = torch.max(outputs.data, 0)[1].cpu().numpy()
                return prob[1]

class DatasetIterater(object):
    def __init__(self, batches, batch_size, device):
        self.batch_size = batch_size
        self.batches = batches
        self.n_batches = len(batches) // batch_size
        self.residue = False
        if len(batches) % self.n_batches != 0:
            self.residue = True
        self.index = 0
        self.device = device

    def _to_tensor(self, datas):
        x = torch.LongTensor([_[0] for _ in datas]).to(self.device)
        y = torch.LongTensor([_[1] for _ in datas]).to(self.device)
        seq_len = torch.LongTensor([_[2] for _ in datas]).to(self.device)
        return (x, seq_len), y

    def __next__(self):
        if self.residue and self.index == self.n_batches:
            batches = self.batches[self.index * self.batch_size: len(self.batches)]
            self.index += 1
            batches = self._to_tensor(batches)
            return batches

        elif self.index >= self.n_batches:
            self.index = 0
            raise StopIteration
        else:
            batches = self.batches[self.index * self.batch_size: (self.index + 1) * self.batch_size]
            self.index += 1
            batches = self._to_tensor(batches)
            return batches

    def __iter__(self):
        return self

    def __len__(self):
        if self.residue:
            return self.n_batches + 1
        else:
            return self.n_batches

class Config(object):

    def __init__(self):
        self.os_path = sys.path[0]
        self.model_name = 'TextRCNN'
        self.model_path = self.os_path + '/Text_relevance/' + self.model_name
        self.vocab_path = self.model_path + '/data/vocab.pkl'
        self.save_path = self.model_path + '/saved_dict/' + self.model_name + '.ckpt'
        self.embedding  = self.model_path + '/data/embedding_SougouNews.npz'

        self.embedding_pretrained = torch.tensor(
            np.load(self.embedding)["embeddings"].astype('float32'))\
            if self.embedding != 'random' else None
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

        self.dropout = 1.0
        self.require_improvement = 1000
        self.num_classes = 2
        self.n_vocab = 0
        self.num_epochs = 10
        self.batch_size = 128
        self.pad_size = 64
        self.learning_rate = 1e-3
        self.embed = self.embedding_pretrained.size(1)\
            if self.embedding_pretrained is not None else 300
        self.hidden_size = 256
        self.num_layers = 1

class Model(nn.Module):
    def __init__(self, config):
        super(Model, self).__init__()
        if config.embedding_pretrained is not None:
            self.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)
        else:
            self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)
        self.lstm = nn.LSTM(config.embed, config.hidden_size, config.num_layers,
                            bidirectional=True, batch_first=True, dropout=config.dropout)
        self.maxpool = nn.MaxPool1d(config.pad_size)
        self.fc = nn.Linear(config.hidden_size * 2 + config.embed, config.num_classes)

    def forward(self, x):
        x, _ = x
        embed = self.embedding(x)
        out, _ = self.lstm(embed)
        out = torch.cat((embed, out), 2)
        out = F.relu(out)
        out = out.permute(0, 2, 1)
        out = self.maxpool(out).squeeze()
        out = self.fc(out)
        return out


import sys

from flask import Flask, jsonify, request
from flask_cors import CORS
from transformers import pipeline

app = Flask(__name__)
CORS(app)

def set_up_gen_pipeline(model_path):
    global gen_pipeline
    gen_pipeline = pipeline('text-generation', model=model_path, framework='pt')

@app.route("/")
def hello():
    res = jsonify({
        "hello": "world!"
    })
    return res

@app.route("/autocomplete")
def prompt():
    context = request.args.get('context', default = '', type = str)
    print(f'context = {context}')
    outputs = gen_pipeline(context, max_length=200, num_return_sequences=3, do_sample=True, eos_token_id=2, pad_token_id=0,
    skip_special_tokens=True, top_k=50, top_p=0.95)
    print(f'outputs = {outputs}')

    res = jsonify({
        "outputs": outputs
    })
    return res

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("Missing required argument: model path.")
        exit(0)
    set_up_gen_pipeline(sys.argv[1])
    app.run(host="localhost", port=5000, debug=True)


from typing import List

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from torch.nn.modules.loss import _Loss

class FocalLoss(nn.Module):
    """Focal loss for multi-classification
    FL(p_t)=-alpha(1-p_t)^{gamma}ln(p_t)
    Notice: y_pred is probability after softmax
    gradient is d(Fl)/d(p_t) not d(Fl)/d(x) as described in paper
    d(Fl)/d(p_t) * [p_t(1-p_t)] = d(Fl)/d(x)
    Focal Loss for Dense Object Detection
    https://arxiv.org/abs/1708.02002
    Keyword Arguments:
        gamma {float} -- (default: {2.0})
        alpha {float} -- (default: {4.0})
    """

    def __init__(self, gamma: float = 2.0, alpha: float = 1.0, size_average=True):
        super(FocalLoss, self).__init__()
        self.gamma = gamma
        self.alpha = alpha
        self.size_average = size_average

    def forward(self, inputs: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:
        """
        :param input:  model's output, shape of [batch_size, num_cls]
        :param target: ground truth labels, shape of [batch_size, num_cls]
        :return: loss
        """
        epsilon = 1.0e-9

        t: torch.Tensor = targets.to(torch.float32)
        p: torch.Tensor = inputs.to(torch.float32) + epsilon

        pt: torch.Tensor = p * t + (1 - p) * (1 - t)  # pt = p if t > 0 else 1-p
        ce: torch.Tensor = -torch.log(pt)
        weight: torch.Tensor = (1.0 - pt) ** self.gamma
        loss: torch.Tensor = weight * self.alpha * ce

        if self.size_average:
            return loss.mean()
        else:
            return loss.sum()

class ImplicitFeedbackBCELoss(nn.Module):
    def __init__(
        self, confidence_weights: List[float], weight=None, reduction="mean"
    ) -> None:
        super().__init__()
        self.confidence_weights = confidence_weights
        self.weight = weight
        self.reduction = reduction

    def forward(
        self,
        input: torch.Tensor,
        target: torch.Tensor,
        *confidence_targets: torch.Tensor
    ) -> torch.Tensor:
        assert len(self.confidence_weights) == len(confidence_targets)
        confidence = torch.ones_like(target)
        for confidence_target, confidence_weight in zip(
            confidence_targets, self.confidence_weights
        ):
            confidence += confidence_weight * confidence_target
        loss = confidence * F.binary_cross_entropy(
            input, target, weight=self.weight, reduction="none"
        )
        if self.reduction == "mean":
            return loss.mean()
        elif self.reduction == "sum":
            return loss.sum()
        else:
            return loss

class CounterfactualRiskMinimization(_Loss):
    def __init__(self, size_average=None, reduce=None, reduction="mean", clip=None):
        super().__init__(size_average, reduce, reduction)
        self.reduction = reduction
        self.clip = clip

    def forward(self, prob, target, ps):
        # from IPython import embed; embed()
        ps = 1 / (ps + 0.0001)
        if self.clip is not None:
            ps = torch.clamp(ps, max=self.clip)

        loss = F.binary_cross_entropy(prob.view(-1), target, reduction="none")

        loss = loss * ps  # (/ ps.sum())#.sum()

        if self.reduction == "mean":
            return loss.mean()
        elif self.reduction == "sum":
            return loss.sum()
        else:
            return loss

class DummyLoss(_Loss):
    def __init__(self, size_average=None, reduce=None, reduction="mean"):
        super().__init__(size_average, reduce, reduction)

    def forward(self, loss, target):
        if self.reduction == "mean":
            return loss.mean()
        elif self.reduction == "sum":
            return loss.sum()
        elif self.reduction == "none":
            return loss


from django.urls import path

from .views import PomodoroDetailView, PomodoroListView, PomodoroCreateTemplateView, PomodoroListViewByDate

urlpatterns = [
    path('add/', PomodoroCreateTemplateView.as_view(), name='add-pomodoro'),
    path('<int:pk>/detail/', PomodoroDetailView.as_view(), name='detail-pomodoro'),
    path('pomodoro-datestamp-collection/<str:datestamp>/', PomodoroListViewByDate.as_view(),
         name='pomodoro-datestamp-collection'),
    path('', PomodoroListView.as_view(), name='pomodoro-index'),
]


#!/usr/bin/env python3

import os

from flask import Flask, request, redirect, url_for, render_template, jsonify, make_response, Blueprint,\
    current_app, Response
from werkzeug.utils import secure_filename
from .ssbliga import SsbLiga

ssb_liga = Blueprint('ssb_liga', __name__, template_folder='templates')

@ssb_liga.route('/')
def start():
    return redirect(url_for('ssb_liga.upload'))

@ssb_liga.route('/upload', methods=['GET', 'POST'])
def upload():
    if request.method == 'GET':
        return render_template('ssb_liga/start.html')

    if 'file' in request.files:

        # upload file to upload location
        up_file = request.files['file']
        upload_location = os.path.join(current_app.config['UPLOAD_FOLDER'], secure_filename(up_file.filename))
        up_file.save(upload_location)

        # process file to activity
        act1 = SsbLiga(adif_file=upload_location)

        for attr in ('district_code', 'my_call', 'category', 'email'):
            act1.meta[attr] = request.values.get(attr, None)

        act1.pre_process()
        id = act1.store()

        return redirect(url_for('ssb_liga.uploaded_adif',id=id))

@ssb_liga.route('/log/<id>')
def uploaded_adif(id):

    log = SsbLiga(id=id)

    return render_template(
        'ssb_liga/render.html',
        log=log,
    )

@ssb_liga.route('/log/<id>/export/cabrillo')
def export_cabrillo(id):
    log = SsbLiga(id=id)

    # max callsign column width
    len1 = len(log.meta['my_call'])
    len2 = max([len(qso.call) for qso in log.qsos])

    output = render_template(
        'ssb_liga/export.cabrillo',
        log=log,
        formats={
            'len1': len1,
            'len2': len2
        },
    )

    my_call = log.meta['my_call'].upper()

    return Response(
        output,
        mimetype='text/plain',
        headers={
            'Content-Disposition': f'attachment; filename="ssb_liga_{my_call}.cab"',
        }
    )


import argparse
import logging
import math
import os
import random
import time
import warnings

import torch
import torch.backends.cudnn as cudnn
import torch.cuda.amp as amp
import torch.distributed as dist
import torch.multiprocessing as mp
import torch.nn as nn
import torch.nn.parallel
import torch.optim
import torch.utils.data
import torch.utils.data.distributed
import torchvision.utils as vutils
from tensorboardX import SummaryWriter

import esrgan_pytorch.models as models
from esrgan_pytorch.dataset import BaseTestDataset
from esrgan_pytorch.dataset import BaseTrainDataset
from esrgan_pytorch.loss import VGGLoss
from esrgan_pytorch.models.discriminator import discriminator_for_vgg
from esrgan_pytorch.utils.common import AverageMeter
from esrgan_pytorch.utils.common import ProgressMeter
from esrgan_pytorch.utils.common import configure
from esrgan_pytorch.utils.common import create_folder
from esrgan_pytorch.utils.estimate import test

model_names = sorted(name for name in models.__dict__
                     if name.islower() and not name.startswith("__")
                     and callable(models.__dict__[name]))

logger = logging.getLogger(__name__)
logging.basicConfig(format="[ %(levelname)s ] %(message)s", level=logging.DEBUG)

parser = argparse.ArgumentParser(description="ESRGAN: Enhanced Super-Resolution Generative Adversarial Networks.")
parser.add_argument("data", metavar="DIR",
                    help="Path to dataset.")
parser.add_argument("-a", "--arch", metavar="ARCH", default="esrgan16",
                    choices=model_names,
                    help="Model architecture: " +
                         " | ".join(model_names) +
                         ". (Default: `esrgan16`)")
parser.add_argument("-j", "--workers", default=4, type=int, metavar="N",
                    help="Number of data loading workers. (Default: 4)")
parser.add_argument("--psnr-epochs", default=4630, type=int, metavar="N",
                    help="Number of total psnr epochs to run. (Default: 4630)")
parser.add_argument("--start-psnr-epoch", default=0, type=int, metavar='N',
                    help="Manual psnr epoch number (useful on restarts). (Default: 0)")
parser.add_argument("--gan-epochs", default=1852, type=int, metavar="N",
                    help="Number of total gan epochs to run. (Default: 1852)")
parser.add_argument("--start-gan-epoch", default=0, type=int, metavar="N",
                    help="Manual gan epoch number (useful on restarts). (Default: 0)")
parser.add_argument("-b", "--batch-size", default=16, type=int,
                    metavar="N",
                    help="Mini-batch size (default: 16), this is the total "
                         "batch size of all GPUs on the current node when "
                         "using Data Parallel or Distributed Data Parallel.")
parser.add_argument("--sampler-frequency", default=1, type=int, metavar="N",
                    help="If there are many datasets, this method can be used "
                         "to increase the number of epochs. (Default:1)")
parser.add_argument("--psnr-lr", type=float, default=0.0002,
                    help="Learning rate for psnr-oral. (Default: 0.0002)")
parser.add_argument("--gan-lr", type=float, default=0.0001,
                    help="Learning rate for gan-oral. (Default: 0.0001)")
parser.add_argument("--image-size", type=int, default=128,
                    help="Image size of high resolution image. (Default: 128)")
parser.add_argument("--upscale-factor", type=int, default=4, choices=[4],
                    help="Low to high resolution scaling factor. Optional: [4] (Default: 4)")
parser.add_argument("--model-path", default="", type=str, metavar="PATH",
                    help="Path to latest checkpoint for model.")
parser.add_argument("--resume_psnr", default="", type=str, metavar="PATH",
                    help="Path to latest psnr-oral checkpoint.")
parser.add_argument("--resume_d", default="", type=str, metavar="PATH",
                    help="Path to latest -oral checkpoint.")
parser.add_argument("--resume_g", default="", type=str, metavar="PATH",
                    help="Path to latest psnr-oral checkpoint.")
parser.add_argument("--pretrained", dest="pretrained", action="store_true",
                    help="Use pre-trained model.")
parser.add_argument("--world-size", default=-1, type=int,
                    help="Number of nodes for distributed training.")
parser.add_argument("--rank", default=-1, type=int,
                    help="Node rank for distributed training. (Default: -1)")
parser.add_argument("--dist-url", default="tcp://59.110.31.55:12345", type=str,
                    help="url used to set up distributed training. (Default: `tcp://59.110.31.55:12345`)")
parser.add_argument("--dist-backend", default="nccl", type=str,
                    help="Distributed backend. (Default: `nccl`)")
parser.add_argument("--seed", default=None, type=int,
                    help="Seed for initializing training.")
parser.add_argument("--gpu", default=None, type=int,
                    help="GPU id to use.")
parser.add_argument("--multiprocessing-distributed", action="store_true",
                    help="Use multi-processing distributed training to launch "
                         "N processes per node, which has N GPUs.")
parser.add_argument("--add_noise", type=int, default=0, choices=[0, 1],
                    help="Add noise if 1 else 0. Optional: [0, 1] (Default: 0)")
parser.add_argument("--save_dir", type=str, default="",
                    help="Directory to save images")

best_psnr = 0.0

def main():
    args = parser.parse_args()
    create_folder(os.path.join(args.save_dir, "runs"))
    create_folder(os.path.join(args.save_dir, "runs/hr"))
    create_folder(os.path.join(args.save_dir, "runs/sr"))
    create_folder(os.path.join(args.save_dir, "weights"))

    if args.seed is not None:
        random.seed(args.seed)
        torch.manual_seed(args.seed)
        cudnn.deterministic = True
        warnings.warn("You have chosen to seed training. "
                      "This will turn on the CUDNN deterministic setting, "
                      "which can slow down your training considerably! "
                      "You may see unexpected behavior when restarting "
                      "from checkpoints.")

    if args.gpu is not None:
        logger.warning("You have chosen a specific GPU. This will completely disable data parallelism.")

    if args.dist_url == "env://" and args.world_size == -1:
        args.world_size = int(os.environ["WORLD_SIZE"])

    args.distributed = args.world_size > 1 or args.multiprocessing_distributed

    ngpus_per_node = torch.cuda.device_count()
    if args.multiprocessing_distributed:
        # Since we have ngpus_per_node processes per node, the total world_size
        # needs to be adjusted accordingly
        args.world_size = ngpus_per_node * args.world_size
        # Use torch.multiprocessing.spawn to launch distributed processes: the
        # main_worker process function
        mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args))
    else:
        # Simply call main_worker function
        main_worker(args.gpu, ngpus_per_node, args)

def main_worker(gpu, ngpus_per_node, args):
    global best_psnr
    args.gpu = gpu

    if args.gpu is not None:
        logger.info(f"Use GPU: {args.gpu} for training.")

    if args.distributed:
        if args.dist_url == "env://" and args.rank == -1:
            args.rank = int(os.environ["RANK"])
        if args.multiprocessing_distributed:
            # For multiprocessing distributed training, rank needs to be the
            # global rank among all the processes
            args.rank = args.rank * ngpus_per_node + gpu
        dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank)
    # create model
    generator = configure(args)
    discriminator = discriminator_for_vgg(image_size=args.image_size)

    if not torch.cuda.is_available():
        logger.warning("Using CPU, this will be slow.")
    elif args.distributed:
        # For multiprocessing distributed, DistributedDataParallel constructor
        # should always set the single device scope, otherwise,
        # DistributedDataParallel will use all available devices.
        if args.gpu is not None:
            torch.cuda.set_device(args.gpu)
            discriminator.cuda(args.gpu)
            generator.cuda(args.gpu)
            # When using a single GPU per process and per
            # DistributedDataParallel, we need to divide the batch size
            # ourselves based on the total number of GPUs we have
            args.batch_size = int(args.batch_size / ngpus_per_node)
            args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node)
            discriminator = nn.parallel.DistributedDataParallel(module=discriminator, device_ids=[args.gpu])
            generator = nn.parallel.DistributedDataParallel(module=generator, device_ids=[args.gpu])
        else:
            discriminator.cuda()
            generator.cuda()
            # DistributedDataParallel will divide and allocate batch_size to all
            # available GPUs if device_ids are not set
            discriminator = nn.parallel.DistributedDataParallel(discriminator)
            generator = nn.parallel.DistributedDataParallel(generator)
    elif args.gpu is not None:
        torch.cuda.set_device(args.gpu)
        discriminator = discriminator.cuda(args.gpu)
        generator = generator.cuda(args.gpu)
    else:
        # DataParallel will divide and allocate batch_size to all available GPUs
        if args.arch.startswith("alexnet") or args.arch.startswith("vgg"):
            discriminator.features = torch.nn.DataParallel(discriminator.features)
            generator.features = torch.nn.DataParallel(generator.features)
            discriminator.cuda()
            generator.cuda()
        else:
            discriminator = torch.nn.DataParallel(discriminator).cuda()
            generator = torch.nn.DataParallel(generator).cuda()

    # Loss = 0.01 * pixel loss + content loss + 0.005 * adversarial loss
    pixel_criterion = nn.L1Loss().cuda(args.gpu)
    content_criterion = VGGLoss().cuda(args.gpu)
    adversarial_criterion = nn.BCEWithLogitsLoss().cuda(args.gpu)
    logger.info(f"Losses function information:\n"
                f"\tPixel:       L1Loss\n"
                f"\tContent:     VGG19_35th\n"
                f"\tAdversarial: BCEWithLogitsLoss")

    # All optimizer function and scheduler function.
    psnr_optimizer = torch.optim.Adam(generator.parameters(), args.psnr_lr, (0.9, 0.999))
    psnr_epoch_indices = math.floor(args.psnr_epochs // 4)
    psnr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(psnr_optimizer, psnr_epoch_indices, 1, 1e-7)
    discriminator_optimizer = torch.optim.Adam(discriminator.parameters(), args.gan_lr, (0.9, 0.999))
    generator_optimizer = torch.optim.Adam(generator.parameters(), args.gan_lr, (0.9, 0.999))
    interval_epoch = math.ceil(args.gan_epochs // 8)
    epoch_indices = [interval_epoch, interval_epoch * 2, interval_epoch * 4, interval_epoch * 6]
    discriminator_scheduler = torch.optim.lr_scheduler.MultiStepLR(discriminator_optimizer, epoch_indices, 0.5)
    generator_scheduler = torch.optim.lr_scheduler.MultiStepLR(generator_optimizer, epoch_indices, 0.5)
    logger.info(f"Optimizer information:\n"
                f"\tPSNR learning rate:          {args.psnr_lr}\n"
                f"\tDiscriminator learning rate: {args.gan_lr}\n"
                f"\tGenerator learning rate:     {args.gan_lr}\n"
                f"\tPSNR optimizer:              Adam, [betas=(0.9,0.999)]\n"
                f"\tDiscriminator optimizer:     Adam, [betas=(0.9,0.999)]\n"
                f"\tGenerator optimizer:         Adam, [betas=(0.9,0.999)]\n"
                f"\tPSNR scheduler:              CosineAnnealingWarmRestarts, [T_0=psnr_epoch_indices, T_mult=1, eta_min=1e-7]\n"
                f"\tDiscriminator scheduler:     MultiStepLR, [milestones=epoch_indices, gamma=0.5]\n"
                f"\tGenerator scheduler:         MultiStepLR, [milestones=epoch_indices, gamma=0.5]")

    logger.info("Load training dataset")
    # Selection of appropriate treatment equipment.
    train_dataset = BaseTrainDataset(os.path.join(args.data, "train"), args.image_size, args.upscale_factor, noise_std = 0.05*args.add_noise)
    test_dataset = BaseTestDataset(os.path.join(args.data, "val"), args.image_size, args.upscale_factor, noise_std = 0.05*args.add_noise)

    if args.distributed:
        train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
    else:
        train_sampler = None

    train_dataloader = torch.utils.data.DataLoader(train_dataset,
                                                   batch_size=args.batch_size,
                                                   shuffle=(train_sampler is None),
                                                   pin_memory=True,
                                                   sampler=train_sampler,
                                                   num_workers=args.workers)
    test_dataloader = torch.utils.data.DataLoader(test_dataset,
                                                  batch_size=args.batch_size,
                                                  shuffle=False,
                                                  pin_memory=True,
                                                  num_workers=args.workers)

    logger.info(f"Dataset information:\n"
                f"\tTrain Path:              {os.getcwd()}/{args.data}/train\n"
                f"\tTest Path:               {os.getcwd()}/{args.data}/test\n"
                f"\tNumber of train samples: {len(train_dataset)}\n"
                f"\tNumber of test samples:  {len(test_dataset)}\n"
                f"\tNumber of train batches: {len(train_dataloader)}\n"
                f"\tNumber of test batches:  {len(test_dataloader)}\n"
                f"\tShuffle of train:        True\n"
                f"\tShuffle of test:         False\n"
                f"\tSampler of train:        {bool(train_sampler)}\n"
                f"\tSampler of test:         None\n"
                f"\tWorkers of train:        {args.workers}\n"
                f"\tWorkers of test:         {args.workers}")

    # optionally resume from a checkpoint
    if args.resume_psnr:
        if os.path.isfile(args.resume_psnr):
            logger.info(f"Loading checkpoint '{args.resume_psnr}'.")
            if args.gpu is None:
                checkpoint = torch.load(args.resume_psnr)
            else:
                # Map model to be loaded to specified single gpu.
                checkpoint = torch.load(args.resume_psnr, map_location=f"cuda:{args.gpu}")
            args.start_psnr_epoch = checkpoint["epoch"]
            best_psnr = checkpoint["best_psnr"]
            if args.gpu is not None:
                # best_psnr may be from a checkpoint from a different GPU
                best_psnr = best_psnr.to(args.gpu)
            generator.load_state_dict(checkpoint["state_dict"])
            psnr_optimizer.load_state_dict(checkpoint["optimizer"])
            logger.info(f"Loaded checkpoint '{args.resume_psnr}' (epoch {checkpoint['epoch']}).")
        else:
            logger.info(f"No checkpoint found at '{args.resume_psnr}'.")

    if args.resume_d or args.resume_g:
        if os.path.isfile(args.resume_d) or os.path.isfile(args.resume_g):
            logger.info(f"Loading checkpoint '{args.resume_d}'.")
            logger.info(f"Loading checkpoint '{args.resume_g}'.")
            if args.gpu is None:
                checkpoint_d = torch.load(args.resume_d)
                checkpoint_g = torch.load(args.resume_g)
            else:
                # Map model to be loaded to specified single gpu.
                checkpoint_d = torch.load(args.resume_d, map_location=f"cuda:{args.gpu}")
                checkpoint_g = torch.load(args.resume_g, map_location=f"cuda:{args.gpu}")
            args.start_gan_epoch = checkpoint_g["epoch"]
            best_psnr = checkpoint_g["best_psnr"]
            if args.gpu is not None:
                # best_psnr may be from a checkpoint from a different GPU
                best_psnr = best_psnr.to(args.gpu)
            discriminator.load_state_dict(checkpoint_d["state_dict"])
            discriminator_optimizer.load_state_dict(checkpoint_d["optimizer"])
            generator.load_state_dict(checkpoint_g["state_dict"])
            generator_optimizer.load_state_dict(checkpoint_g["optimizer"])
            logger.info(f"Loaded checkpoint '{args.resume_d}' (epoch {checkpoint_d['epoch']}).")
            logger.info(f"Loaded checkpoint '{args.resume_g}' (epoch {checkpoint_g['epoch']}).")
        else:
            logger.info(f"No checkpoint found at '{args.resume_d}' or '{args.resume_g}'.")

    cudnn.benchmark = True

    # The mixed precision training is used in PSNR-oral.
    scaler = amp.GradScaler()
    logger.info("Turn on mixed precision training.")

    # Create a SummaryWriter at the beginning of training.
    psnr_writer = SummaryWriter(f"runs/{args.arch}_psnr_logs")
    gan_writer = SummaryWriter(f"runs/{args.arch}_gan_logs")

    logger.info(f"Train information:\n"
                f"\tPSNR-oral epochs: {args.psnr_epochs}\n"
                f"\tGAN-oral epochs:  {args.gan_epochs}")

    for epoch in range(args.start_psnr_epoch, args.psnr_epochs):
        if args.distributed:
            train_sampler.set_epoch(epoch)

        train_psnr(dataloader=train_dataloader,
                   model=generator,
                   criterion=pixel_criterion,
                   optimizer=psnr_optimizer,
                   epoch=epoch,
                   scaler=scaler,
                   writer=psnr_writer,
                   args=args)

        psnr_scheduler.step()

        # Test for every epoch.
        psnr, ssim, lpips, gmsd = test(dataloader=test_dataloader, model=generator, gpu=args.gpu)
        gan_writer.add_scalar("Test/PSNR", psnr, epoch + 1)
        gan_writer.add_scalar("Test/SSIM", ssim, epoch + 1)
        gan_writer.add_scalar("Test/LPIPS", lpips, epoch + 1)
        gan_writer.add_scalar("Test/GMSD", gmsd, epoch + 1)

        is_best = psnr > best_psnr
        best_psnr = max(psnr, best_psnr)

        if not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0):
            #torch.save({"epoch": epoch + 1,
            #            "arch": args.arch,
            #            "best_psnr": best_psnr,
            #            "state_dict": generator.state_dict(),
            #            "optimizer": psnr_optimizer.state_dict(),
            #            }, os.path.join("weights", f"PSNR_epoch{epoch}.pth"))
            if is_best:
                torch.save(generator.state_dict(), os.path.join(args.save_dir, "weights", f"PSNR.pth"))
                torch.save({"epoch": epoch + 1,
                        "arch": args.arch,
                        "best_psnr": best_psnr,
                        "state_dict": generator.state_dict(),
                        "optimizer": psnr_optimizer.state_dict(),
                        }, os.path.join(args.save_dir, "weights", f"PSNR_best.pth"))

    # Load best model weight.
    best_psnr = 0.0
    generator.load_state_dict(torch.load(os.path.join(args.save_dir, "weights", f"PSNR.pth"), map_location=f"cuda:{args.gpu}"))

    for epoch in range(args.start_gan_epoch, args.gan_epochs):
        if args.distributed:
            train_sampler.set_epoch(epoch)

        # train for one epoch
        train_gan(dataloader=train_dataloader,
                  discriminator=discriminator,
                  discriminator_optimizer=discriminator_optimizer,
                  generator=generator,
                  generator_optimizer=generator_optimizer,
                  pixel_criterion=pixel_criterion,
                  content_criterion=content_criterion,
                  adversarial_criterion=adversarial_criterion,
                  epoch=epoch,
                  scaler=scaler,
                  writer=gan_writer,
                  args=args)

        discriminator_scheduler.step()
        generator_scheduler.step()

        # Test for every epoch.
        psnr, ssim, lpips, gmsd = test(dataloader=test_dataloader, model=generator, gpu=args.gpu)
        gan_writer.add_scalar("Test/PSNR", psnr, epoch + 1)
        gan_writer.add_scalar("Test/SSIM", ssim, epoch + 1)
        gan_writer.add_scalar("Test/LPIPS", lpips, epoch + 1)
        gan_writer.add_scalar("Test/GMSD", gmsd, epoch + 1)

        is_best = psnr > best_psnr
        best_psnr = max(psnr, best_psnr)

        if not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0):
            #torch.save({"epoch": epoch + 1,
            #            "arch": "vgg",
            #            "state_dict": discriminator.state_dict(),
            #            "optimizer": discriminator_optimizer.state_dict()
            #            }, os.path.join("weights", f"Discriminator_epoch{epoch}.pth"))
            #torch.save({"epoch": epoch + 1,
            #            "arch": args.arch,
            #            "best_psnr": best_psnr,
            #            "state_dict": generator.state_dict(),
            #            "optimizer": generator_optimizer.state_dict()
            #            }, os.path.join("weights", f"Generator_epoch{epoch}.pth"))
            if is_best:
                torch.save(generator.state_dict(), os.path.join(args.save_dir,"weights", f"GAN.pth"))
                torch.save({"epoch": epoch + 1,
                        "arch": "vgg",
                        "state_dict": discriminator.state_dict(),
                        "optimizer": discriminator_optimizer.state_dict()
                        }, os.path.join(args.save_dir, "weights", f"Discriminator_best.pth"))
                torch.save({"epoch": epoch + 1,
                            "arch": args.arch,
                            "best_psnr": best_psnr,
                            "state_dict": generator.state_dict(),
                            "optimizer": generator_optimizer.state_dict()
                            }, os.path.join(args.save_dir, "weights", f"Generator_best.pth"))

def train_psnr(dataloader: torch.utils.data.DataLoader,
               model: nn.Module,
               criterion: nn.L1Loss,
               optimizer: torch.optim.Adam,
               epoch: int,
               scaler: amp.GradScaler,
               writer: SummaryWriter,
               args: argparse.ArgumentParser.parse_args):
    batch_time = AverageMeter("Time", ":6.4f")
    losses = AverageMeter("Loss", ":.6f")
    progress = ProgressMeter(num_batches=len(dataloader),
                             meters=[batch_time, losses],
                             prefix=f"Epoch: [{epoch}]")

    # switch to train mode
    model.train()

    end = time.time()
    for i, (lr, hr) in enumerate(dataloader):
        # Move data to special device.
        if args.gpu is not None:
            lr = lr.cuda(args.gpu, non_blocking=True)
            hr = hr.cuda(args.gpu, non_blocking=True)

        optimizer.zero_grad()

        with amp.autocast():
            sr = model(lr)
            loss = criterion(sr, hr)

        scaler.scale(loss).backward()
        scaler.step(optimizer)
        scaler.update()

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        # measure accuracy and record loss
        losses.update(loss.item(), lr.size(0))

        iters = i + epoch * len(dataloader) + 1
        writer.add_scalar("Train/Loss", loss.item(), iters)

        # Output results every 100 batches.
        if i % 100 == 0:
            progress.display(i)

        # Save image every 300 batches.
        if iters % 300 == 0:
            vutils.save_image(hr.detach(), os.path.join(args.save_dir, "runs", "hr", f"PSNR_{iters}.bmp"))
            vutils.save_image(sr.detach(), os.path.join(args.save_dir, "runs", "sr", f"PSNR_{iters}.bmp"))

def train_gan(dataloader: torch.utils.data.DataLoader,
              discriminator: nn.Module,
              discriminator_optimizer: torch.optim.Adam,
              generator: nn.Module,
              generator_optimizer: torch.optim.Adam,
              pixel_criterion: nn.L1Loss,
              content_criterion: VGGLoss,
              adversarial_criterion: nn.BCEWithLogitsLoss,
              epoch: int,
              scaler: amp.GradScaler,
              writer: SummaryWriter,
              args: argparse.ArgumentParser.parse_args):
    batch_time = AverageMeter("Time", ":.4f")
    d_losses = AverageMeter("D Loss", ":.6f")
    g_losses = AverageMeter("G Loss", ":.6f")
    pixel_losses = AverageMeter("Pixel Loss", ":6.4f")
    content_losses = AverageMeter("Content Loss", ":6.4f")
    adversarial_losses = AverageMeter("Adversarial Loss", ":6.4f")

    progress = ProgressMeter(num_batches=len(dataloader),
                             meters=[batch_time, d_losses, g_losses, pixel_losses, content_losses, adversarial_losses],
                             prefix=f"Epoch: [{epoch}]")

    # switch to train mode
    discriminator.train()
    generator.train()

    end = time.time()
    for i, (lr, hr) in enumerate(dataloader):
        # Move data to special device.
        if args.gpu is not None:
            lr = lr.cuda(args.gpu, non_blocking=True)
            hr = hr.cuda(args.gpu, non_blocking=True)
        batch_size = lr.size(0)

        # The real sample label is 1, and the generated sample label is 0.
        real_label = torch.full((batch_size, 1), 1, dtype=lr.dtype).cuda(args.gpu, non_blocking=True)
        fake_label = torch.full((batch_size, 1), 0, dtype=lr.dtype).cuda(args.gpu, non_blocking=True)

        ##############################################
        # (1) Update D network: E(hr)[fake(C(D(hr) - E(sr)C(sr)))] + E(sr)[fake(C(fake) - E(real)C(real))]
        ##############################################
        discriminator_optimizer.zero_grad()

        with amp.autocast():
            sr = generator(lr)
            # It makes the discriminator distinguish between real sample and fake sample.
            real_output = discriminator(hr)
            fake_output = discriminator(sr.detach())

            # Adversarial loss for real and fake images (relativistic average GAN)
            d_loss_real = adversarial_criterion(real_output - torch.mean(fake_output), real_label)
            d_loss_fake = adversarial_criterion(fake_output - torch.mean(real_output), fake_label)

            # Count all discriminator losses.
            d_loss = (d_loss_real + d_loss_fake) / 2

        scaler.scale(d_loss).backward()
        scaler.step(discriminator_optimizer)
        scaler.update()

        ##############################################
        # (2) Update G network: E(hr)[sr(C(D(hr) - E(sr)C(sr)))] + E(sr)[sr(C(fake) - E(real)C(real))]
        ##############################################
        generator_optimizer.zero_grad()

        with amp.autocast():
            sr = generator(lr)
            # It makes the discriminator unable to distinguish the real samples and fake samples.
            real_output = discriminator(hr.detach())
            fake_output = discriminator(sr)

            # Calculate the absolute value of pixels with L1 loss.
            pixel_loss = pixel_criterion(sr, hr.detach())
            # # The 35th layer in VGG19 is used as the feature extractor by default.
            content_loss = content_criterion(sr, hr.detach())
            # Adversarial loss for real and fake images (relativistic average GAN)
            adversarial_loss = adversarial_criterion(fake_output - torch.mean(real_output), real_label)

            # Count all generator losses.
            g_loss = 0.01 * pixel_loss + 1 * content_loss + 0.005 * adversarial_loss

        scaler.scale(g_loss).backward()
        scaler.step(generator_optimizer)
        scaler.update()

        # Set generator gradients to zero.
        generator.zero_grad()

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        # measure accuracy and record loss
        d_losses.update(d_loss.item(), lr.size(0))
        g_losses.update(g_loss.item(), lr.size(0))
        pixel_losses.update(pixel_loss.item(), lr.size(0))
        content_losses.update(content_loss.item(), lr.size(0))
        adversarial_losses.update(adversarial_loss.item(), lr.size(0))

        iters = i + epoch * len(dataloader) + 1
        writer.add_scalar("Train/D_Loss", d_loss.item(), iters)
        writer.add_scalar("Train/G_Loss", g_loss.item(), iters)
        writer.add_scalar("Train/Pixel_Loss", pixel_loss.item(), iters)
        writer.add_scalar("Train/Content_Loss", content_loss.item(), iters)
        writer.add_scalar("Train/Adversarial_Loss", adversarial_loss.item(), iters)

        # Output results every 100 batches.
        if i % 100 == 0:
            progress.display(i)

        # Save image every 300 batches.
        if iters % 300 == 0:
            vutils.save_image(hr.detach(), os.path.join(args.save_dir, "runs", "hr", f"GAN_{iters}.bmp"))
            vutils.save_image(sr.detach(), os.path.join(args.save_dir, "runs", "sr", f"GAN_{iters}.bmp"))

if __name__ == "__main__":
    print("##################################################\n")
    print("Run Training Engine.\n")

    create_folder("runs")
    create_folder("runs/hr")
    create_folder("runs/sr")
    create_folder("weights")

    logger.info("TrainingEngine:")
    print("\tAPI version .......... 0.2.2")
    print("\tBuild ................ 2021.04.15")
    print("##################################################\n")
    main()
    logger.info("All training has been completed successfully.\n")


from urllib import request

url = 'http://httpbin.org/ip'
# url = 'http://www.ip138.com'
# 無代理
# resp = request.urlopen(url)
# print(resp.read()) #b'{\n  "origin": "117.173.131.189, 117.173.131.189"\n}\n'

# 使用代理

# 1.使用ProxyHandler 傳入代理，構建hanlder
handler = request.ProxyHandler({'http':'118.24.172.37:1080'})
# 2. 使用構建的handler構建一個opener
opener = request.build_opener(handler)
# 3. 使用opener發送請求
resp = opener.open(url)
print(resp.read())

handler = request.ProxyHandler({'http':'187.53.61.50:80'})
opener = request.build_opener(handler)
resp = opener.open(url)
print(resp.read())


###########################################################
################  Module importing   ######################
###########################################################

import io
import os
import pandas as pd
import time
import tqdm
import logging
import json
import sys
from google.cloud import vision

###########################################################
################  Path description  #######################
###########################################################

# Opening JSON file
f = open('Config/experiments.json')

# returns JSON object as  a dictionary
json_experiment = json.load(f)

experiment_id = sys.argv[1]

if experiment_id not in json_experiment:
  raise Exception('experiment_id should be in experiments.json.')

IMG_DIR = 'img/' + experiment_id
OUTPUT_DIR = 'data/processed/'

###########################################################
################  Logger Initiation  ######################
###########################################################

logging.basicConfig(filename=f'tmp/log_google_{experiment_id}_{time.strftime("%Y%m%d-%H%M%S")}.txt',
                    level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(name)s %(message)s')
logger=logging.getLogger(__name__)

###########################################################
################  Authentication    #######################
###########################################################

KEY_PATH = 'secrets/google_cred.json'
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = KEY_PATH

client = vision.ImageAnnotatorClient()

###########################################################
################  Images identification    ################
###########################################################

# Retrieve all file_names required
file_names = []

# Gets absolute path of all images in directory
for folder, subs, files in os.walk(IMG_DIR):
  for filename in files:
    file_names.append(os.path.abspath(os.path.join(folder, filename)))

# Gets files of already outputted results
files = os.listdir(OUTPUT_DIR)
files = list(filter(lambda f: f.endswith('_google.csv'), files))

# Iterates over files on already outputted results, adding the loan_ids in the files in the processed_ids set
processed_ids = set()
for file in files:
  df_aux = pd.read_csv(f'{OUTPUT_DIR}{file}')
  processed_ids.update(df_aux['loan_id'].astype(str))

###########################################################
################  Images identification    ################
###########################################################

def get_emotions_google(path):

    """
    Retrieves the emotions from a local image, based on Google's Computer Vision API.

    Args:
        path: Absolute path of an image.

    Returns:
        res: Dictionary with fields loan_id, anger, contempt, disgust, fear, happiness, neutral, sadness, surprise.
    """
    if os.path.getsize(path) == 0:
      return None
    # Loads the image into memory
    with io.open(path, 'rb') as image_file:
        content = image_file.read()

    image = vision.Image(content=content)

    response = client.face_detection(image=image)

    # if response.error.message:
    #     raise Exception(
    #         '{}\nFor more info on error messages, check: '
    #         'https://cloud.google.com/apis/design/errors'.format(
    #             response.error.message))
    try:
      face = response.face_annotations[0]

      # Names of likelihood from google.cloud.vision.enums
      likelihood_name = ('UNKNOWN', 'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE',
                         'LIKELY', 'VERY_LIKELY')

      res = {'loan_id' : os.path.split(path)[1].replace('.jpg',''),
      'detection_confidence': face.detection_confidence,
      'landmarking_confidence': face.landmarking_confidence,
      'joy_likelihood': format(likelihood_name[face.joy_likelihood]),
      'sorrow_likelihood': format(likelihood_name[face.sorrow_likelihood]),
      'anger_likelihood': format(likelihood_name[face.anger_likelihood]),
      'surprise_likelihood': format(likelihood_name[face.surprise_likelihood]),
      'under_exposed_likelihood': format(likelihood_name[face.under_exposed_likelihood]),
      'blurred_likelihood': format(likelihood_name[face.blurred_likelihood]),
      'headwear_likelihood': format(likelihood_name[face.headwear_likelihood])}

      return res
    except:
      logger.exception('No face detected from image {}'.format(path))

# Creates list for storing results
res = []

for file_name in tqdm.tqdm(file_names):
  loan_id = os.path.split(file_name)[1].replace('.jpg','')

  if (str(loan_id) in processed_ids):
    print('Loan ID {} already processed'.format(str(loan_id)))
  else :
    print('Processing Loan ID {}'.format(str(loan_id)))
    out = get_emotions_google(file_name)
    print('Loan ID {} processed correctly'.format(str(loan_id)))
    if out is not None:
        res.append(out)
    else:
        print('Loan ID {} had no results'.format(str(loan_id)))

# Concatenate to DataFrame and write to file
if len(res)>0:
  df = pd.DataFrame(res)
  filename = OUTPUT_DIR  + experiment_id + '_' + time.strftime("%Y%m%d-%H%M%S") + '_google.csv'
  df.to_csv(filename)


import argparse
import glob
import json
import logging
import os
import pickle
from typing import Optional
import torch.distributed as dist
import torch.utils.data.distributed
from torch.utils.data import DataLoader, Dataset
from tqdm import tqdm
from transformers import PreTrainedModel, RobertaConfig, RobertaModel
from transformers import RobertaTokenizer
from transformers.file_utils import is_torch_available, requires_backends

logger = logging.getLogger(__name__)

if is_torch_available():
    import torch

class AnceEncoder(PreTrainedModel):
    config_class = RobertaConfig
    base_model_prefix = 'ance_encoder'
    load_tf_weights = None
    _keys_to_ignore_on_load_missing = [r'position_ids']
    _keys_to_ignore_on_load_unexpected = [r'pooler', r'classifier']

    def __init__(self, config: RobertaConfig):
        requires_backends(self, 'torch')
        super().__init__(config)
        self.config = config
        self.roberta = RobertaModel(config)
        self.embeddingHead = torch.nn.Linear(config.hidden_size, 768)
        self.norm = torch.nn.LayerNorm(768)
        self.init_weights()

    # Copied from transformers.models.bert.modeling_bert.BertPreTrainedModel._init_weights
    def _init_weights(self, module):
        """ Initialize the weights """
        if isinstance(module, (torch.nn.Linear, torch.nn.Embedding)):
            # Slightly different from the TF version which uses truncated_normal for initialization
            # cf https://github.com/pytorch/pytorch/pull/5617
            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
        elif isinstance(module, torch.nn.LayerNorm):
            module.bias.data.zero_()
            module.weight.data.fill_(1.0)
        if isinstance(module, torch.nn.Linear) and module.bias is not None:
            module.bias.data.zero_()

    def init_weights(self):
        self.roberta.init_weights()
        self.embeddingHead.apply(self._init_weights)
        self.norm.apply(self._init_weights)

    def forward(
            self,
            input_ids: torch.Tensor,
            attention_mask: Optional[torch.Tensor] = None,
    ):
        input_shape = input_ids.size()
        device = input_ids.device
        if attention_mask is None:
            attention_mask = (
                torch.ones(input_shape, device=device)
                if input_ids is None
                else (input_ids != self.roberta.config.pad_token_id)
            )
        outputs = self.roberta(input_ids=input_ids, attention_mask=attention_mask)
        sequence_output = outputs.last_hidden_state
        pooled_output = sequence_output[:, 0, :]
        pooled_output = self.norm(self.embeddingHead(pooled_output))
        return pooled_output

class QueryData_MSMARCO(Dataset):
    def __init__(self, query_path, tokenizer):
        self.data = self.read_file(query_path)
        self.tokenizer = tokenizer

    def read_file(self, data_path):
        data = []
        with open(data_path) as fin:
            for line in fin:
                example = json.loads(line.strip())
                data.append(example)
        return data

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        return self.data[index]

    def collect_fn(self, data):
        qids = []
        queries = []
        for example in data:
            qids.append(example["_id"])
            queries.append(example["text"])
        inputs = self.tokenizer(
            queries,
            max_length=64,
            padding='longest',
            truncation=True,
            add_special_tokens=True,
            return_tensors='pt'
        )
        return {
            "qids": qids,
            "queries": queries,
            "query_tokens": inputs["input_ids"]
        }

class QueryData_NQ(Dataset):
    def __init__(self, query_path, tokenizer):
        self.data = self.read_file(query_path)
        self.tokenizer = tokenizer

    def read_file(self, data_paths):
        data = []
        for data_path in data_paths:
            with open(data_path) as fin:
                for line in fin:
                    example = json.loads(line.strip())
                    data.append(example)
        return data

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        return self.data[index]

    def collect_fn(self, data):
        qids = []
        queries = []
        for example in data:
            qids.append(example["qid"])
            queries.append(example["text"])
        inputs = self.tokenizer(
            queries,
            max_length=64,
            padding='longest',
            truncation=True,
            add_special_tokens=True,
            return_tensors='pt'
        )
        return {
            "qids": qids,
            "queries": queries,
            "query_tokens": inputs["input_ids"]
        }

def merge_files(split_pattern, output_file):
    splits = glob.glob(split_pattern)
    print("Temperary validation files: {}.".format(splits))
    res_dict = {}
    for s in splits:
        with open(s, 'rb') as f:
            data = pickle.load(f)
            res_dict.update(data)
        os.remove(s)
    with open(output_file, 'wb') as f:
        pickle.dump(res_dict, f)
    print("Merge total {} lines.".format(len(res_dict.keys())))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset', type=str, required=True, help="Dataset name like MSMARCO, NQ.")
    parser.add_argument('--tokenizer', type=str, metavar='name or path', required=True,
                        help="Path to a hgf tokenizer name or path.")
    parser.add_argument('--encoder', type=str, required=True,
                        help="Path to an encoder name or path.")
    parser.add_argument('--data_path', type=str, required=True, help="Path to datasets.")
    parser.add_argument('--output_path', type=str, metavar='path', required=True, help="Path to output file.")
    parser.add_argument('--batch_size', type=int, required=False, default=128, help="Batch size.")
    parser.add_argument('--local_rank', type=int, default=-1, help="For distributed training: local_rank.")
    args = parser.parse_args()

    logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.INFO)
    logger.info(args)

    dist.init_process_group(backend='nccl', init_method='env://')
    torch.cuda.set_device(args.local_rank)
    device = torch.device("cuda", args.local_rank)

    logger.info('Loading ANCE model!')
    query_encoder = AnceEncoder.from_pretrained(args.encoder).eval()
    tokenizer = RobertaTokenizer.from_pretrained(args.tokenizer)

    if args.local_rank != -1:
        query_encoder = torch.nn.parallel.DistributedDataParallel(
            query_encoder.to(device),
            device_ids=[args.local_rank],
            output_device=args.local_rank,
            find_unused_parameters=True
        )
    if args.dataset.lower() == 'msmarco':
        query_data = QueryData_MSMARCO(os.path.join(args.data_path, "queries.jsonl"), tokenizer)
    elif args.dataset.lower() == 'nq':
        data_paths = ["nq-dev.json", "nq-train.json", "nq-test.json"]
        data_paths = [os.path.join(args.data_path, path) for path in data_paths]
        query_data = QueryData_NQ(data_paths, tokenizer)

    query_sampler = torch.utils.data.distributed.DistributedSampler(query_data)
    query_loader = DataLoader(query_data, batch_size=args.batch_size, sampler=query_sampler,
                              collate_fn=query_data.collect_fn)
    qembed_path = os.path.join(args.output_path, "q_embed{0}.pkl".format(args.local_rank))
    query_embed_dict = {}

    logger.info('Encoding query!')
    for batch in tqdm(query_loader):
        query_embeds = query_encoder(batch["query_tokens"]).detach().cpu().numpy()
        query_ids = batch["qids"]
        assert len(query_ids) == len(query_embeds)
        for qid, qembed in zip(query_ids, query_embeds):
            query_embed_dict[qid] = qembed

    logger.info('Saving query file!')
    with open(qembed_path, "wb") as f_qembed:
        pickle.dump(query_embed_dict, f_qembed)
    dist.barrier()
    if args.local_rank == 0:
        merge_files(os.path.join(args.output_path, "q_embed*.pkl"), os.path.join(args.output_path, "qembed.pkl"))


from config import *
import megatexture

class Engine:
    """
        Responsible for drawing scenes
    """

    def __init__(self, width, height):
        """
            Initialize a flat raytracing context

                Parameters:
                    width (int): width of screen
                    height (int): height of screen
        """
        self.screenWidth = width
        self.screenHeight = height

        self.targetFrameRate = 60
        self.frameRateMargin = 10

        #general OpenGL configuration
        self.shader = self.createShader("shaders/frameBufferVertex.txt",
                                        "shaders/frameBufferFragment.txt")

        self.rayTracerShader = self.createComputeShader("shaders/rayTracer.txt")

        glEnable(GL_BLEND)
        glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)

        glUseProgram(self.shader)

        self.createQuad()
        self.createNoiseTexture()
        self.createResourceImages()
        self.createLODChain()
        self.createColorBuffers()
        self.createMegaTexture()

    def createQuad(self):
        # x, y, z, s, t
        self.vertices = np.array(
            ( 1.0,  1.0, 0.0, 1.0, 1.0, #top-right
             -1.0,  1.0, 0.0, 0.0, 1.0, #top-left
             -1.0, -1.0, 0.0, 0.0, 0.0, #bottom-left
             -1.0, -1.0, 0.0, 0.0, 0.0, #bottom-left
              1.0, -1.0, 0.0, 1.0, 0.0, #bottom-right
              1.0,  1.0, 0.0, 1.0, 1.0), #top-right
             dtype=np.float32
        )

        self.vertex_count = 6

        self.vao = glGenVertexArrays(1)
        glBindVertexArray(self.vao)
        self.vbo = glGenBuffers(1)
        glBindBuffer(GL_ARRAY_BUFFER, self.vbo)
        glBufferData(GL_ARRAY_BUFFER, self.vertices.nbytes, self.vertices, GL_STATIC_DRAW)

        glEnableVertexAttribArray(0)
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 20, ctypes.c_void_p(0))
        glEnableVertexAttribArray(1)
        glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 20, ctypes.c_void_p(12))

    def createNoiseTexture(self):

        self.noiseTextures = []

        for i in range(4):
            noise = []
            for pixel in range(1024):
                vector = randomInUnitSphere()
                noise.append(vector[0])
                noise.append(vector[1])
                noise.append(vector[2])
            noise = np.array(noise, dtype=np.float32)

            noiseTexture = glGenTextures(1)
            glActiveTexture(GL_TEXTURE1 + i)
            glBindTexture(GL_TEXTURE_2D, noiseTexture)

            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)

            glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA32F,32,32,0,GL_RGB,GL_FLOAT,bytes(noise))

            self.noiseTextures.append(noiseTexture)

    def createResourceImages(self):

        """
            allocate storage for up to 1024 spheres and 1024 planes (why not?)
        """

        sphereData = []

        for i in range(1024):
            for attribute in range(8):
                sphereData.append(0.0)
        self.sphereData = np.array(sphereData, dtype=np.float32)

        self.sphereDataTexture = glGenTextures(1)
        glActiveTexture(GL_TEXTURE5)
        glBindTexture(GL_TEXTURE_2D, self.sphereDataTexture)

        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)

        glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA32F,2,1024,0,GL_RGBA,GL_FLOAT,bytes(self.sphereData))

        planeData = []

        for i in range(1024):
            for attribute in range(20):
                planeData.append(0.0)
        self.planeData = np.array(planeData, dtype=np.float32)

        self.planeDataTexture = glGenTextures(1)
        glActiveTexture(GL_TEXTURE6)
        glBindTexture(GL_TEXTURE_2D, self.planeDataTexture)

        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)

        glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA32F,5,1024,0,GL_RGBA,GL_FLOAT,bytes(self.planeData))

    def createLODChain(self):

        self.resolutions = [(self.screenWidth, self.screenHeight)]

        width,height = (self.screenWidth,self.screenHeight)
        while width > 2 and height > 2:
            width = int(width / 1.25)
            height = int(height / 1.25)
            self.resolutions.append((width, height))

        self.resolutionLevel = len(self.resolutions) - 1

        self.screenWidth,self.screenHeight = self.resolutions[self.resolutionLevel]

    def createColorBuffers(self):

        self.colorBuffers = []

        for resolution in self.resolutions:

            width,height = resolution

            newColorBuffer = glGenTextures(1)
            self.colorBuffers.append(newColorBuffer)
            glActiveTexture(GL_TEXTURE0)
            glBindTexture(GL_TEXTURE_2D, newColorBuffer)

            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)

            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, width, height, 0, GL_RGBA, GL_FLOAT, None)

        self.colorBuffer = self.colorBuffers[self.resolutionLevel]

    def createMegaTexture(self):

        filenames = [
            "AlienArchitecture", "AlternatingColumnsConcreteTile", "BiomechanicalPlumbing",
            "CarvedStoneFloorCheckered", "ChemicalStrippedConcrete", "ClayBrick",
            "CrumblingBrickWall", "DiamondSquareFlourishTiles", "EgyptianHieroglyphMetal",
            "FancyHotelWallDirty", "LatheAndPlasterWall"
        ]

        self.megaTexture = megatexture.MegaTexture(filenames)

    def adaptResolution(self, frameRate):

        if frameRate > self.targetFrameRate + self.frameRateMargin and self.resolutionLevel > 0:
            #increase resolution
            self.resolutionLevel -= 1
        elif frameRate < self.targetFrameRate - self.frameRateMargin and self.resolutionLevel < len(self.resolutions) - 1:
            #reduce resolution
            self.resolutionLevel += 1

        self.screenWidth,self.screenHeight = self.resolutions[self.resolutionLevel]
        self.colorBuffer = self.colorBuffers[self.resolutionLevel]

    def createShader(self, vertexFilepath, fragmentFilepath):
        """
            Read source code, compile and link shaders.
            Returns the compiled and linked program.
        """

        with open(vertexFilepath,'r') as f:
            vertex_src = f.readlines()

        with open(fragmentFilepath,'r') as f:
            fragment_src = f.readlines()

        shader = compileProgram(compileShader(vertex_src, GL_VERTEX_SHADER),
                                compileShader(fragment_src, GL_FRAGMENT_SHADER))

        return shader

    def createComputeShader(self, filepath):
        """
            Read source code, compile and link shaders.
            Returns the compiled and linked program.
        """

        with open(filepath,'r') as f:
            compute_src = f.readlines()

        shader = compileProgram(compileShader(compute_src, GL_COMPUTE_SHADER))

        return shader

    def fetchDeviceLimits(self):

        self.workGroupCountX = glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_COUNT, 0)[0]
        self.workGroupCountY = glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_COUNT, 1)[0]
        self.workGroupCountZ = glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_COUNT, 2)[0]

        self.workGroupSizeX = glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 0)[0]
        self.workGroupSizeY = glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1)[0]
        self.workGroupSizeZ = glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 2)[0]

        self.workGroupInvocations = glGetIntegerv(GL_MAX_COMPUTE_WORK_GROUP_INVOCATIONS)

        print(f"max global (total) work group counts x:{self.workGroupCountX} y:{self.workGroupCountY} z:{self.workGroupCountZ}")
        print(f"max global (total) work group sizes x:{self.workGroupSizeX} y:{self.workGroupSizeY} z:{self.workGroupSizeZ}")
        print(f"max local work group invocations: {self.workGroupInvocations}")

    def recordSphereData(self, i, sphereCount, sphere):

        self.sphereData[8*(i + sphereCount)    ] = sphere.center[0]
        self.sphereData[8*(i + sphereCount) + 1] = sphere.center[1]
        self.sphereData[8*(i + sphereCount) + 2] = sphere.center[2]

        self.sphereData[8*(i + sphereCount) + 3] = sphere.radius

        self.sphereData[8*(i + sphereCount) + 4] = sphere.material

    def recordPlaneData(self, i, planeCount, plane):

        self.planeData[20*(i + planeCount)     ] = plane.normal[0]
        self.planeData[20*(i + planeCount) +  1] = plane.normal[1]
        self.planeData[20*(i + planeCount) +  2] = plane.normal[2]

        self.planeData[20*(i + planeCount) +  3] = plane.center[0]
        self.planeData[20*(i + planeCount) +  4] = plane.center[1]
        self.planeData[20*(i + planeCount) +  5] = plane.center[2]

        self.planeData[20*(i + planeCount) +  6] = plane.tangent[0]
        self.planeData[20*(i + planeCount) +  7] = plane.tangent[1]
        self.planeData[20*(i + planeCount) +  8] = plane.tangent[2]

        self.planeData[20*(i + planeCount) +  9] = plane.bitangent[0]
        self.planeData[20*(i + planeCount) + 10] = plane.bitangent[1]
        self.planeData[20*(i + planeCount) + 11] = plane.bitangent[2]

        self.planeData[20*(i + planeCount) + 12] = plane.uMin
        self.planeData[20*(i + planeCount) + 13] = plane.uMax
        self.planeData[20*(i + planeCount) + 14] = plane.vMin
        self.planeData[20*(i + planeCount) + 15] = plane.vMax

        self.planeData[20*(i + planeCount) + 16] = plane.material

    def updateDataResources(self, scene):

        glUseProgram(self.rayTracerShader)

        scene.outdated = False

        sphereCount = 0

        for i,sphere in enumerate(scene.spheres):

            self.recordSphereData(i, sphereCount, sphere)

        sphereCount += len(scene.spheres)

        for room in scene.active_rooms:

            for i,sphere in enumerate(room.spheres):

                self.recordSphereData(i, sphereCount, sphere)

            sphereCount += len(room.spheres)

        glUniform1f(glGetUniformLocation(self.rayTracerShader, "sphereCount"), sphereCount)

        glActiveTexture(GL_TEXTURE5)
        glBindTexture(GL_TEXTURE_2D, self.sphereDataTexture)
        glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA32F,2,1024,0,GL_RGBA,GL_FLOAT,bytes(self.sphereData))

        planeCount = 0

        for i,plane in enumerate(scene.planes):

            self.recordPlaneData(i, planeCount, plane)

        planeCount += len(scene.planes)

        for room in scene.active_rooms:

            for i,plane in enumerate(room.planes):

                self.recordPlaneData(i, planeCount, plane)

            planeCount += len(room.planes)

        for door in scene.active_rooms[0].doors:

            for i,plane in enumerate(door.planes):

                self.recordPlaneData(i, planeCount, plane)

            planeCount += len(door.planes)

        glUniform1f(glGetUniformLocation(self.rayTracerShader, "planeCount"), planeCount)

        glActiveTexture(GL_TEXTURE6)
        glBindTexture(GL_TEXTURE_2D, self.planeDataTexture)
        glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA32F,5,1024,0,GL_RGBA,GL_FLOAT,bytes(self.planeData))

    def prepareComputeShader(self, scene):

        glUseProgram(self.rayTracerShader)

        glActiveTexture(GL_TEXTURE0)
        glBindImageTexture(0, self.colorBuffer, 0, GL_FALSE, 0, GL_WRITE_ONLY, GL_RGBA32F)

        #noise textures
        for i in range(4):
            glActiveTexture(GL_TEXTURE1 + i)
            glBindImageTexture(1 + i, self.noiseTextures[i], 0, GL_FALSE, 0, GL_READ_ONLY, GL_RGBA32F)

        glActiveTexture(GL_TEXTURE5)
        glBindImageTexture(5, self.sphereDataTexture, 0, GL_FALSE, 0, GL_READ_ONLY, GL_RGBA32F)

        glActiveTexture(GL_TEXTURE6)
        glBindImageTexture(6, self.planeDataTexture, 0, GL_FALSE, 0, GL_READ_ONLY, GL_RGBA32F)

        glActiveTexture(GL_TEXTURE7)
        glBindImageTexture(7, self.megaTexture.texture, 0, GL_FALSE, 0, GL_READ_ONLY, GL_RGBA32F)

        #write material data

        for i,material in enumerate(scene.materials):
            glUniform3fv(glGetUniformLocation(self.rayTracerShader, f"materials[{i}].color"), 1, material.color)
            glUniform1f(glGetUniformLocation(self.rayTracerShader, f"materials[{i}].gloss"), material.reflectance)

        #write camera data
        glUniform3fv(glGetUniformLocation(self.rayTracerShader, "viewer.forwards"), 1, scene.camera.forwards)
        glUniform3fv(glGetUniformLocation(self.rayTracerShader, "viewer.up"), 1, scene.camera.up)
        glUniform3fv(glGetUniformLocation(self.rayTracerShader, "viewer.right"), 1, scene.camera.right)
        glUniform3fv(glGetUniformLocation(self.rayTracerShader, "viewer.position"), 1, scene.camera.position)
        glUniformMatrix4fv(glGetUniformLocation(self.rayTracerShader, "view"),
            1, False, pyrr.matrix44.create_look_at(
                eye = scene.camera.position,
                target = scene.camera.position + scene.camera.forwards,
                up = scene.camera.up,
                dtype = np.float32
            )
        )

    def renderScene(self, scene):
        """
            Draw all objects in the scene
        """

        if scene.outdated:
            self.updateDataResources(scene)

        self.prepareComputeShader(scene)

        glDispatchCompute(self.screenWidth, self.screenHeight, 1)

        # make sure writing to image has finished before read
        glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT)
        glBindImageTexture(0, 0, 0, GL_FALSE, 0, GL_WRITE_ONLY, GL_RGBA32F)
        self.drawScreen()

    def drawScreen(self):
        glUseProgram(self.shader)
        glBindFramebuffer(GL_FRAMEBUFFER, 0)
        glActiveTexture(GL_TEXTURE0)
        glBindTexture(GL_TEXTURE_2D, self.colorBuffer)
        glBindVertexArray(self.vao)
        glDrawArrays(GL_TRIANGLES, 0, self.vertex_count)
        pg.display.flip()

    def destroy(self):
        """
            Free any allocated memory
        """
        glUseProgram(self.rayTracerShader)
        glMemoryBarrier(GL_ALL_BARRIER_BITS)
        glDeleteProgram(self.rayTracerShader)
        glDeleteVertexArrays(1, (self.vao,))
        glDeleteBuffers(1, (self.vbo,))
        glDeleteTextures(len(self.colorBuffers), self.colorBuffers)
        glDeleteProgram(self.shader)


import sys
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
import math
import hexgrid

class HexWidget(QWidget):
	"""Widget displaying a given Grid.

	This class should only be called to embed it in another window.

	Attributes:
		grid (Grid): a given Grid instance.
		height (int): the height of the hexagons in the grid in pixels.
	"""

	def __init__(self, grid, height):
		super(HexWidget, self).__init__()

		self.grid = grid
		self.hexsize = height / 2.0
		self.hexheight = height
		self.fieldheight = self.hexheight * self.grid.radius * 1.5 # *0.75 for every hex (they overlap 0.25) *2 for radius to diameter
		self.center = (self.fieldheight * 2) / 2
		self.initUI()

	def initUI(self):

		self.setGeometry(0, 0, self.fieldheight * 2, self.fieldheight * 2)
		self.setWindowTitle('Hexes')
		self.show()

	def paintEvent(self, e):

		qp = QPainter()
		qp.begin(self)
		self.drawHexes(qp)
		qp.end()

	def drawHexes(self, qp):
		for hexa in self.grid.grid:
			hexa.drawHex(qp, self.hexsize, self.center)

class HexWindow(object):
	"""Displays a window with a HexWidget.

	Attributes:
		grid (Grid): a given Grid instance.
		height (int): the height of the hexagons in the grid in pixels.
	"""
	def __init__(self, grid, height):
		self.grid = grid
		self.height = height

	def show(self):
		app = QApplication(sys.argv)
		widget = HexWidget(self.grid, self.height)
		sys.exit(app.exec_())


# Codigo desechado

# Obtenemos frecuencias absolutas y relativas
# Para cada variable
n_df_train = df[df.subset=='train'].shape[0]
b_abs = df[df.subset=='train'].brand_name.value_counts(dropna=False)
b_rel = round(b_abs/n_df_train*100,4)
b_ar = pd.concat([b_abs, b_rel], axis=1)
c1_abs = df[df.subset=='train'].cat_lev_1.value_counts(dropna=False)
c1_rel = round(c1_abs/n_df_train*100,4)
c1_ar = pd.concat([c1_abs, c1_rel], axis=1)
c2_abs = df[df.subset=='train'].cat_lev_2.value_counts(dropna=False)
c2_rel = round(c2_abs/n_df_train*100,4)
c2_ar = pd.concat([c2_abs, c2_rel], axis=1)
c3_abs = df[df.subset=='train'].cat_lev_3.value_counts(dropna=False)
c3_rel = round(c3_abs/n_df_train*100,4)
c3_ar = pd.concat([c3_abs, c3_rel], axis=1)
# Borramos objetos sobrantes
del b_abs, b_rel, c1_abs, c1_rel, c2_abs, c2_rel, c3_abs, c3_rel, n_df_train
gc.collect()
# Para el primer par de variables
c12_abs = pd.crosstab(index=df[df.subset=='train'].cat_lev_2,
                      columns=df[df.subset=='train'].cat_lev_1, dropna=False)
c12_abs = c12_abs.reset_index(level=c12_abs.index.names)
c12_ar = pd.melt(c12_abs, id_vars=c12_abs.columns.values[0],
                 value_vars=c12_abs.columns.tolist()[1:])
c12_rel_c1 = round(pd.crosstab(index=df[df.subset=='train'].cat_lev_2,
                               columns=df[df.subset=='train'].cat_lev_1,
                               dropna=False, normalize='columns')*100,4)
c12_rel_c1 = c12_rel_c1.reset_index(level=c12_rel_c1.index.names)
aux = pd.melt(c12_rel_c1, id_vars=c12_rel_c1.columns.values[0],
                 value_vars=c12_rel_c1.columns.tolist()[1:])
c12_ar = pd.concat([c12_ar, aux.value], axis=1)
c12_rel = round(pd.crosstab(index=df[df.subset=='train'].cat_lev_2,
                            columns=df[df.subset=='train'].cat_lev_1,
                            dropna=False, normalize='all')*100,4)
c12_rel = c12_rel.reset_index(level=c12_rel.index.names)
aux = pd.melt(c12_rel, id_vars=c12_rel.columns.values[0],
              value_vars=c12_rel.columns.tolist()[1:])
c12_ar = pd.concat([c12_ar, aux.value], axis=1)
c12_ar.columns = c12_ar.columns.tolist()[0:2] + ['c12_abs', 'c12_rel_c1', 'c12_rel']
c12_ar = c12_ar.iloc[:,[1,0]+list(range(2,5))]
c12_ar = c12_ar[c12_ar.c12_abs>0]
# Borramos objetos sobrantes
del c12_abs, c12_rel, c12_rel_c1, aux; gc.collect()


import struct

ELF_MAGIC_OFFSET   = 0
ELF_MAGIC_BYTES    = [0x7f, ord('E'), ord('L'), ord('F')]
ELF_CLASS_OFFSET   = 4
ELF_CLASS_BYTES    = [2]
ELF_EI_DATA_OFFSET = 5
ELF_EI_DATA_BYTES  = [1]
ELF_ENTRY_OFFSET   = 24
ELF_ENTRY_BYTES    = [0x70, 0x1b, 0x40, 0, 0, 0, 0, 0]  # call _start
# ELF_ENTRY_BYTES    = [0x90, 0x16, 0x40, 0, 0, 0, 0, 0]  # call main
# ELF_ENTRY_BYTES    = [0xa0, 0x1c, 0x40, 0, 0, 0, 0, 0]  # call call_me


import os

from setuptools import setup, find_packages

__version__ = '1.4.2'

requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt')
with open(requirements_path) as requirements_file:
    requirements = requirements_file.readlines()

setup(
    name='amundsen-search',
    version=__version__,
    description='Search Service for Amundsen',
    url='https://github.com/lyft/amundsensearchlibrary.git',
    maintainer='Lyft',
    maintainer_email='dp-tools@lyft.com',
    packages=find_packages(exclude=['tests*']),
    include_package_data=True,
    zip_safe=False,
    dependency_links=[],
    install_requires=requirements,
    python_requires=">=3.6"
)


from flask import Blueprint
from flask import render_template, request, jsonify, redirect, url_for
from database import MongoDB
import jwt
from dotenv import load_dotenv
import os

# Connect DB
db = MongoDB().db

# load env SECRET_KEY
load_dotenv()
SECRET_KEY = os.environ.get('SECRET_KEY')

my_question = Blueprint('my_question', __name__)

@my_question.route("/myquestion", methods=["GET"])
def check_user_info_get():
    token_receive = request.cookies.get('mytoken')
    try:
        payload = jwt.decode(token_receive, SECRET_KEY, algorithms=['HS256'])
        signin_status = True
        user = db.user.find_one({"email": payload["id"]})
        solved_list = user['solved']
        question_list = []
        for solved in solved_list:
            question_collection = db.question.find_one({"num": int(solved)})
            question_num = question_collection['num']
            question = question_collection['question']
            answer_collection = db.answer.find_one({"question_num": int(solved), "user_email": payload["id"]})
            if answer_collection != "None":
                answer = answer_collection['answer']
                like_count = answer_collection['like_count']
            # print(question_collection)
            # print(answer_collection)
            question_list.append((question_num, question, answer, like_count))

        return render_template('my-question.html', signin_status=signin_status, user_info=user, question_list=question_list)
    except (jwt.ExpiredSignatureError, jwt.exceptions.DecodeError):
        return render_template("sign-in.html")


import scrapy
from ..WasiItem import WasiItem

class WasiLapSpider(scrapy.Spider):
    name = 'wasi_lap'
    start_urls = [
        'https://www.wasi.lk/product-category/electronic-devices/laptops/page/1'
    ]
    page_number = 2

    def parse(self, response):

        detail_links = response.css('div > div.mf-product-details > div.mf-product-details-hover > h2 > a::attr(href)').getall()
        max_pagenumber = 5
        for link in detail_links:
            yield scrapy.Request(
                response.urljoin(link),
                callback=self._parse_nextpage
            )

        next_page = 'https://www.wasi.lk/product-category/electronic-devices/laptops/' + \
                     str(WasiLapSpider.page_number) + ''
        if WasiLapSpider.page_number <= int(max_pagenumber):
            WasiLapSpider.page_number += 1
            yield response.follow(next_page, callback=self.parse)

    def _parse_nextpage(self, response):
        product_name = response.css('.entry-title::text').get()
        product_price = response.css('.amount::text').get()
        product_discount = response.css('.onsale::text').get()
        product_image =  response.css('div.mf-product-detail > div.woocommerce-product-gallery.woocommerce-product-gallery--with-images.woocommerce-product-gallery--columns-5.images.without-thumbnails > figure > div > a::attr(href)').get()
        detailed_url = response.request.url

        #Checking whether the scraped data is a laptop or not
        if(product_name.find('Laptop')>=0):
            newProduct = WasiItem()

            newProduct['product_name'] = product_name
            newProduct['product_price'] = product_price
            newProduct['product_discount'] = product_discount
            newProduct['product_image'] = product_image
            newProduct['detailed_url'] = detailed_url

            yield newProduct


#entertainment_center_part2.py file feeds fresh_tomatoes.py
#with needed class and title info to
#create a web page.

#fresh_tomatoes is a py file that sets up the web site html template,
#and extracts movie content
#from entertainment_center.py file.
import fresh_tomatoes

#media is a class that entertainment_center_part2 re-uses for each
#movie trailer instance.
import media

#code that calls media.Movie class to create each movie trailer.
#used google url shortener to reduce trailer jpeg link size.
Gods_not_dead = media.Movie("Gods not dead",
                            "College student stands up for his faith",
                            "http://goo.gl/vM8hOn",
                            "https://www.youtube.com/watch?v=Mqf5gAYcFWs")

Bruce_Lee = media.Movie("Enter The Dragon",
                        "Bruce Lee's last completed movie",
                        "http://goo.gl/6T7mrh",
                        "https://www.youtube.com/watch?v=tB-QGOChuQc")

StarWars_vs_StarTrek = media.Movie("Star Wars vs Star Trek",
                                   "A well done trailer that got my geek going",
                                   "http://goo.gl/1jH4ZD",
                                   "http://www.youtube.com/watch?v=o5S5tBuNJdM",)

Grownups = media.Movie("Grownups",
                       "Adam Sandler makes another movie with his friends",
                       "http://goo.gl/ilwfVo",
                       "https://www.youtube.com/watch?v=e01NVCveGkg")

Facing_the_giants = media.Movie("Facing The Giants",
                                "A low budget movie with a good message",
                                "http://goo.gl/6o3RxT",
                                "http://www.youtube.com/watch?v=jtVJqyhk97U",)

Book_of_eli = media.Movie("Book Of Eli",
                          "2010 Denzel movie about perserving a copy of the Bible",
                          "http://goo.gl/Dg52vF",
                          "http://www.youtube.com/watch?v=kAMUv22y1og")

# movies list populates fresh_tomatoes()
movies = Grownups, Bruce_Lee, Book_of_eli, StarWars_vs_StarTrek, Gods_not_dead, Facing_the_giants

# fresh_tomatoes.open_movies_page(movies) launches fresh_tomatoes py file operation.
fresh_tomatoes.open_movies_page(movies)

# prints scrip info from media.py using __doc__ class
print (media.Movie.__doc__)


from rasa.core.agent import Agent
from rasa.utils.endpoints import EndpointConfig
import asyncio

url = 'http://localhost:5056/webhook'

print('action endpoint: "{}"'.format(url))
model = 'C:/Users/gsl/Desktop/rasa_alt/20220310-113317.tar.gz'

agent = Agent.load(
    model,
    action_endpoint=EndpointConfig(url)
)

async def pharse(user_input):
    nlu = await agent.parse_message_using_nlu_interpreter(user_input)
    return nlu

nlu = asyncio.run(pharse('12365478'))
print(nlu)


# SOURCE of names: https://www.reddit.com/r/TheoryOfReddit/comments/1f7hqc/the_200_most_active_subreddits_categorized_by/
SUBREDDIT_NAMES = [
    'AskReddit',
    'IAmA',
    'bestof',
    'fatpeoplestories',
    'pettyrevenge',
    'TalesFromRetail',
    'DoesAnybodyElse',
    'CrazyIdeas',
    'WTF',
    'aww',
    'cringepics',
    'cringe',
    'MorbidReality',
    'rage',
    'mildlyinfuriating',
    'creepy',
    'creepyPMs',
    'nosleep',
    'nostalgia',
    'gaming',
    'leagueoflegends',
    'pokemon',
    'Minecraft',
    'starcraft',
    'Games',
    'DotA2',
    'skyrim',
    'tf2',
    'magicTCG',
    'wow',
    'KerbalSpaceProgram',
    'mindcrack',
    'Fallout',
    'roosterteeth',
    'Planetside',
    'gamegrumps',
    'battlefield3',
    'zelda',
    'darksouls',
    'masseffect',
    'arresteddevelopment',
    'gameofthrones',
    'doctorwho',
    'mylittlepony',
    'community',
    'breakingbad',
    'adventuretime',
    'startrek',
    'TheSimpsons',
    'futurama',
    'HIMYM',
    'DunderMifflin',
    'thewalkingdead',
    'Music',
    'movies',
    'harrypotter',
    'StarWars',
    'DaftPunk',
    'hiphopheads',
    'anime',
    'comicbooks',
    'geek',
    'batman',
    'TheLastAirbender',
    'Naruto',
    'FanTheories',
    'funny',
    'AdviceAnimals',
    'fffffffuuuuuuuuuuuu',
    '4chan',
    'firstworldanarchists',
    'circlejerk',
    'MURICA',
    'facepalm',
    'Jokes',
    'wheredidthesodago',
    'polandball',
    'TrollXChromosomes',
    'comics',
    'nottheonion',
    'britishproblems',
    'videos',
    'gifs',
    'reactiongifs',
    'mildlyinteresting',
    'woahdude',
    'FiftyFifty',
    'FoodPorn',
    'HistoryPorn',
    'wallpapers',
    'youtubehaiku',
    'Unexpected',
    'photoshopbattles',
    'AnimalsBeingJerks',
    'cosplay',
    'awwnime',
    'PerfectTiming',
    'tumblr',
    'techsupportgore',
    'PrettyGirls',
    'itookapicture',
    'todayilearned',
    'science',
    'askscience',
    'space',
    'AskHistorians',
    'YouShouldKnow',
    'explainlikeimfive',
    'trees',
    'MakeupAddiction',
    'cats',
    'LifeProTips',
    'RedditLaqueristas',
    'Random_Acts_Of_Amazon',
    'food',
    'guns',
    'tattoos',
    'corgi',
    'teenagers',
    'GetMotivated',
    'motorcycles',
    'sex',
    'progresspics',
    'DIY',
    'bicycling',
    'Fitness',
    'lifehacks',
    'longboarding',
    'Frugal',
    'drunk',
    'Art',
    'loseit',
    'Military',
    'politics',
    'worldnews',
    'news',
    'TrueReddit',
    'offbeat',
    'canada',
    'toronto',
    'australia',
    'unitedkingdom',
    'atheism',
    'TwoXChromosomes',
    'MensRights',
    'gaybros',
    'lgbt',
    'nbasoccer',
    'hockey',
    'nfl',
    'formula1',
    'baseball',
    'MMA',
    'SquaredCircle',
    'technology',
    'Android',
    'Bitcoin',
    'programming',
    'apple',
    'Python',
    'HowToPython',
    'LearnPython',
    'PythonProjects2',
    'MadeInPython',
    'datascience',
    'Home',
    'Damnthatsinteresting',
    'WhitePeopleTwitter',
    'pics',
    'PublicFreakout',
    'therewasanattempt',
    'HolUp',
    'interestingasfuck',
    'wallstreetbets',
    'entertainment',
    'NoStupidQuestions',
    'LivestreamFail',
    'meirl',
    'AmItheAsshole',
    'nextfuckinglevel'
]

SUBREDDIT_TYPES = ['hot', 'new']


t = int(input())
for _ in range(t):
    a,b,c,d,k = [int(i) for i in input().split()]
    # it is all about the manhattan distance
    diff = abs(c-a)+abs(d-b)
    if (k == diff):
        print('YES')
        continue
    if k>diff:
        if (k-diff)%2 == 0: #excess number of steps is even
            print('YES')
        else:
            print('NO')
        continue
    print('NO')

Rating 1200 - 1400/TomAndJerry_Rt_1379.py