import random
import timm
import torch
import train as t
import wandb
from config import Config_03_Train as config

if __name__ == "__main__":
    random.seed(config.seed)

    project_name = config.project_name

    # Check for GPU
    device = "cuda" if torch.cuda.is_available() else "cpu"
    print("Using {} device".format(device))

    ######################## PARAMS ###############################

    dropout = config.dropout
    l2 = config.l2
    lr = config.lr
    milestones = config.milestones
    lsmooth = config.lsmooth

    ######################## MODEL ###############################
    if project_name == 'BinaryClass':
        num_classes = 1
    elif project_name == 'MultiClass':
        num_classes = 3

    pretrain = True  # True
    model_name = config.model_name
    model = timm.create_model(model_name,
                              pretrained=pretrain,
                              num_classes=num_classes,
                              drop_rate=dropout)

    ######################### TRAINING #########################
    param_name = '_dropout=' + str(config.dropout) + '_l2=' + str(config.l2) + '_lr=' + str(
        config.lr) + '_milestones=' + str(config.milestones) + '_lsmooth_' + str(config.lsmooth)
    exp_name = config.project_name + param_name
    best_val_acc = t.training(device, project_name, exp_name, model_name, model, config.data_path,
                              config.txt_path + 'train.txt', config.txt_path + 'test.txt',
                              config.epochs, config.batch_size, lr, dropout, l2, milestones, lsmooth,
                              config.pos_weight, config.dim, pretrain)
    wandb.finish()


from getmac import get_mac_address
import serial.tools.list_ports
import yaml
import pandas as pd
# Change Accordingly
mintsDefinitions          = yaml.load(open('mintsXU4/credentials/mintsDefinitions.yaml'),Loader=yaml.FullLoader)
# dataFolder                = mintsDefinitions['dataFolder']
# dataFolderReference       = mintsDefinitions['dataFolder'] + "/reference"
# dataFolderMQTTReference   = mintsDefinitions['dataFolder'] + "/referenceMqtt"  # The path of your MQTT Reference Data
# dataFolderMQTT            = mintsDefinitions['dataFolder'] + "/rawMqtt"        # The path of your MQTT Raw Data
tlsCert                   = mintsDefinitions['tlsCert']     # The path of your TLS cert

# latestOn                  = False

# mqttOn                    = True

credentialsFile           = 'mintsXU4/credentials/credentials.yaml'
credentials               = yaml.load(open(credentialsFile))

# sensorInfoFile            = 'mintsXU4/credentials/sensorIDs.yml'
sensorInfo                  = pd.read_csv('https://raw.githubusercontent.com/mi3nts/AirQualityAnalysisWorkflows/main/influxdb/nodered-docker/sensorIDs.csv')
print("Sensor Info: " + str(sensorInfo))

portInfo                  = pd.read_csv('https://raw.githubusercontent.com/mi3nts/AirQualityAnalysisWorkflows/main/influxdb/nodered-docker/portIDs.csv')
print("Port Info: " + str(portInfo))

nodeInfo                  = pd.read_csv('https://raw.githubusercontent.com/mi3nts/AirQualityAnalysisWorkflows/main/influxdb/nodered-docker/id_lookup.csv')
print("Node Info: " + str(nodeInfo))

# Add in AWS MQTT Credentials
mqttBrokerDC              = "mqtt.circ.utdallas.edu"
mqttBrokerLoRa            = "mqtt.lora.trecis.cloud"

mqttPortDC                = 8883  # Secure port
mqttPortLoRa              = 1883  # Secure port

print()
print("----- AWS Support -----")


import os
from this import d
from PyPDF2 import PdfFileReader, PdfFileMerger

merger = PdfFileMerger()

while True:
  menu = input("Precisa de ajuda? (S/N)")
  print()
  if(menu == "S" or menu == "s"):
    print("Para renomear adequadamente os arquivos, selecione todos, do último para o primeiro.")
    print("Clique com o botão da direita do mouse no primeiro arquivo e selecione 'Renomear' e digite o nome escolhido.")
    print("Por padrão o nome dos arquivos será o que você escolher, porém o windows colocará (1), (2), etc na frente.")
    print("Quando o programa for executado, digite exatamente o nome que você escolheu quando perguntado qual o nome inicial do arquivo.")
    print("Siga as instruções que aparecerem na tela...")
    break
  if(menu == "N" or menu == "n"):
    break
  else:
    print("Digite S para sim ou N para não.")

nomeEntrada = str(input("Digite o nome inicial de todos os arquivos: "))
print()

numeroArquivos = int(input("Digite o número de arquivos a serem juntados: "))
print()

####### criando os nomes dos arquivos a serem lidos
filename = [nomeEntrada + " (1).pdf"]
for i in range(1, numeroArquivos):
  filename.append(nomeEntrada + " (" + str(i + 1) + ").pdf")
print(filename)

nomeSaida = str(input("Digite o nome do arquivo de saída: "))

#####ler os arquivos pdf
for i in range(0, numeroArquivos):
  merger.append(PdfFileReader(filename[i], "rb"))
merger.write(os.path.join(nomeSaida + ".pdf"))


def getAllIssues(cursor):
    # cursor.execute("""SELECT * FROM issued_books""")
    cursor.execute("""SELECT
  i.issueId,
  b.title,
  m.full_name,
  i.issued_at,
  i.is_returned,
                   i.bookId
FROM
  members m
INNER JOIN
  issued_books i ON m.member_id = i.memberId
INNER JOIN
  books b ON i.bookId = b.bookId
ORDER BY
  i.is_returned ASC,
  i.issued_at DESC
""")
    res = cursor.fetchall()
    issued_books = []
    for row in res:
        item = {
            "issueId": row[0],
            "title": row[1],
            "full_name": row[2],
            "issued_at": row[3],
            "is_returned": row[4],
            "bookId": row[5]
        }
        issued_books.append(item)

    return issued_books

def createIssue(conn, data):
    try:
        bookId = data["book_id"]
        memberId = data["member_id"]

        cursor = conn.cursor()

        cursor.execute(f"""SELECT COUNT(*) FROM issued_books where memberid = {memberId} and is_returned = false """)

        res = cursor.fetchone()
        totalIssued = res[0]

        if totalIssued >=  5:
            return "OVERDUE"

        cursor.execute("""INSERT INTO issued_books (memberId, bookId, is_returned) VALUES (%s, %s, %s)""", (memberId, bookId, False))

        conn.commit()

        cursor.execute(f"""UPDATE books SET stock = stock -1 WHERE bookId = {bookId}""")
        conn.commit()
        return "SUCCESS"
    except Exception as e:
        print("An error occurred:", e)
        conn.rollback()
        return "FAILURE"

    return data

#    ISSUE_BOOKS TABLE SCHEMA
#   issueId serial PRIMARY KEY,
#   memberId int NOT NULL,
#   bookId int NOT NULL,
#   issued_at timestamp DEFAULT now(),
#   returned_at timestamp,
#   is_returned boolean NOT NULL


from datetime import datetime, timedelta

from django.contrib import messages
from django.contrib.auth import views
from django.core.exceptions import ValidationError
from django.core.mail import EmailMultiAlternatives
from django.conf import settings
from django.http.response import HttpResponseRedirect
from django.shortcuts import redirect
from django.template.loader import render_to_string
from django.urls import reverse, reverse_lazy
from django.utils.translation import gettext as _, gettext_lazy
from django.views.generic import View as BaseView

from conlangdev.forms.auth import AuthenticationForm, RegisterForm
from conlangdev.utils.token import (
    check_confirm_email_token,
    generate_confirm_email_token,
)
from conlangdev.views.base import FormView, View

class ConfirmEmailView(BaseView):
    success_url = reverse_lazy("login")

    def get(self, request, *args, **kwargs):
        token = self.kwargs.get("token")
        user = check_confirm_email_token(token)
        if user:
            user.email_confirmed = True
            user.email_confirmed_at = datetime.now()
            user.save()
            messages.success(
                request,
                _("Thanks for confirming your email address! You can now log in."),
            )
        else:
            messages.error(
                request,
                _(
                    "Sorry, that confirmation token either wasn't valid or has "
                    "expired."
                ),
            )
        return redirect(self.success_url)

class LoginView(FormView, views.LoginView):
    form_class = AuthenticationForm
    template_name = "conlangdev/form.jinja"
    success_url = reverse_lazy("homepage")

    verb = gettext_lazy("Login")
    verb_icon = "fa-arrow-right-to-bracket"
    page_icon = "fa-arrow-right-to-bracket"
    cancel_label = "Register"
    cancel_icon = "fa-user-plus"
    cancel_url = reverse_lazy("register")

    @property
    def hidden_fields(self):
        return {
            "next": self.request.GET.get("next", self.success_url),
        }

    def get(self, request, *args, **kwargs):
        if request.user.is_authenticated:
            return HttpResponseRedirect(self.success_url)
        return super().get(request, *args, **kwargs)

class LogoutView(View, views.LogoutView):
    next_page = "login"

class RegisterView(FormView):
    form_class = RegisterForm
    success_url = reverse_lazy("login")

    verb = gettext_lazy("Register")
    verb_icon = "fa-user-plus"
    page_icon = "fa-user-plus"

    def form_valid(self, form):
        form.instance.set_password(form.cleaned_data["password1"])
        form.instance.save()

        token = generate_confirm_email_token(
            form.instance, datetime.now() + timedelta(hours=1)
        )
        link = reverse("email_confirm", kwargs={"token": token})
        context = {
            "username": form.instance.username,
            "link": f"{settings.SITE_ROOT_URL}{link}",
        }
        message_text = render_to_string("conlangdev/email/confirm_email.txt", context)
        message_html = render_to_string("conlangdev/email/confirm_email.html", context)
        message = EmailMultiAlternatives(
            _("Confirm your email address"),
            message_text,
            settings.EMAIL_FROM,
            [form.instance.email],
        )
        message.attach_alternative(message_html, "text/html")
        message.send()

        messages.info(
            self.request,
            _(
                "Account created successfully! Check your emails - we've sent "
                "you a confirmation link to click before you can log in."
            ),
        )
        return redirect(self.success_url)

    def form_invalid(self, form):
        if "captcha" in form.errors:
            form.errors["captcha"] = ValidationError(
                _(
                    "Please confirm you are not a robot by ticking the verification "
                    "check."
                )
            )
        return super().form_invalid(form)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

# import
from matplotlib import pyplot as plt
import pandas as pd

# In[2]:

file = pd.read_excel("KB_mike3.xlsx")
print(file)

# In[3]:

# extract from csv
b_fdm = file["B FDM"]
k_fdm = file["K FDM"]
b_tacho = file["B tacho"]
k_tacho = file["K tacho"]
b_foaw = file["B FOAW"]
k_foaw = file["K FOAW"]

# In[5]:

# plotting
plt.scatter(b_fdm,k_fdm)
plt.plot(b_fdm,k_fdm,label='FDM')

plt.scatter(b_tacho,k_tacho)
plt.plot(b_tacho,k_tacho,label='Tacho')

plt.scatter(b_foaw,k_foaw)
plt.plot(b_foaw,k_foaw,label='FOAW')

plt.legend()
plt.xlabel('B [(Nm*s)/deg]')
plt.ylabel('K [Nm/deg]')
plt.title('MIKE 3 KB-Plot')
plt.savefig("Mike3_KB_Plot.pdf")
plt.show()


import serial
from json.decoder import JSONDecodeError

# configure the serial connections (the parameters differs on the device you are connecting to)
ser = serial.Serial(
    port='/dev/ttyUSB0',
    baudrate=115200
)

# import time
# time.sleep(3)
import json
import struct
run = True
while run:
    try:

        input("Dale")
        ser.write(json.dumps({"command": "status"}).encode("utf-8"))
        line = ser.readline().decode('utf-8')
        print(line)
        ser.write(json.dumps({"command": "read"}).encode("utf-8"))
        line = ser.readline().decode('utf-8')
        print(line)
        input("DOLO")
  # buffer[0] = 'D';
  # buffer[1] = 'S';
  # buffer[2] = '1';
  # buffer[3] = '8';
  # buffer[4] = 'B';
  # buffer[5] = '2';
  # buffer[6] = '0';
  # buffer[7] = 0;
  # buffer[8] = 8;
  # buffer[9] = 0x28;
  # buffer[10] = 0xFF;
  # buffer[11] = 0x10;
  # buffer[12] = 0x93;
  # buffer[13] = 0x6F;
  # buffer[14] = 0x14;
  # buffer[15] = 0x4;
  # buffer[16] = 0x11;
  #
  # buffer[17] = 'D';
  # buffer[18] = 'S';
  # buffer[19] = '1';
  # buffer[29] = '8';
  # buffer[21] = 'B';
  # buffer[22] = '2';
  # buffer[23] = '0';
  # buffer[24] = 0;
  # buffer[25] = 8;
  # buffer[26] = 1;
  # buffer[27] = 2;
  # buffer[28] = 3;
  # buffer[29] = 4;
  # buffer[30] = 5;
  # buffer[31] = 6;
  # buffer[32] = 7;
  # buffer[33] = 8;
        sensor_type = "DS18B20"
        sensor_type_packed = struct.pack("{size}s".format(size=len(sensor_type)), bytes(sensor_type, 'ascii'))
        address = [0x28, 0xFF, 0x10, 0x93, 0x6F, 0x14, 0x4, 0x11]
        address_packed = struct.pack("8B", *address)
        # print(address)
        data = sensor_type_packed + struct.pack("BB", 0, len(address)) + address_packed
        # print(data)
        # print(len(data))
        print(json.dumps({"command": "writeConfig", "totalSensors": 1, "dataSize": len(data)}))
        ser.write(json.dumps({"command": "writeConfig", "totalSensors": 1, "dataSize": len(data)}).encode("utf-8"))
        ser.write(data)
        line = ser.readline().decode('utf-8')
        print(line)
        message = json.loads(line)
        print("FALLO" if message["error"] else "EXITO")
        # for i in message["data"]:
        #     print("Sensor:", i["name"])
        #     print("Valores", i["values"])

    except KeyboardInterrupt:
        run = False

ser.close()


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
import pyttsx3
import pywhatkit as whatsapp
from datetime import datetime

driver =webdriver.Chrome("C:/Program Files (x86)/chromedriver.exe")
mouth=pyttsx3.init()
def driveSite(weblink):
    driver.get(weblink)
def loginWhatsapp():
    driveSite("http://web.whatsapp.com")
    mouth.say("Kindly Scan this QR Code from your cell phone")
    mouth.runAndWait()
    driver.implicitly_wait(60)
def insert_contact(partial_name, contactnumber):
    file = open("Contacts.txt")
    file.write(partial_name+","+contactnumber+"\n")
def getList(files):
    file=open(files,'r')
    content = file
    d=dict()
    for line in content:
        cn, n=line.split(",")
        d[cn]=n
    return d
def sendMessage(message):
    contactsdict=getList("Contacts.txt")
    for i in contactsdict:
        number = contactsdict.get(i)
        print(number)
        current_hour=datetime.now().hour
        current_minute=datetime.now().minute+1
        whatsapp.sendwhatmsg(number, message,current_hour,current_minute)
def closeWeb():
    driver.close()
loginWhatsapp()
sendMessage("Hello this is automated message sent by Aadarsh Mehdi")
closeWeb()


import numpy as np
import matplotlib.pyplot as plt
from math import sqrt

def get_probability(delta_energy, beta):
    return np.exp(-delta_energy*beta)

def get_energy():
    return -np.sum(
    interaction * spins * np.roll(spins, 1, axis=0) +
    interaction * spins * np.roll(spins, -1, axis=0) +
    interaction * spins * np.roll(spins, 1, axis=1) +
    interaction * spins * np.roll(spins, -1, axis=1)
  )/2 - moment * np.sum(field * spins)

def update(beta):
    global spins
    # global current_energy
    global spins_sum
    i = np.random.randint(spins.shape[0])
    j = np.random.randint(spins.shape[1])
    delta_energy=2*spins[i,j]*(spins[(i+1)%L,j]+spins[(i-1)%L,j]+spins[i,(j+1)%L]+spins[i,(j-1)%L])
    if get_probability(delta_energy, beta) > np.random.random():
        spins[i,j] *= -1
        # current_energy += delta_energy
        # spins_sum += spins[i,j]*2

def isweep(N,beta):
    for i in range(N):
        update(beta)
        # print("isweep",i)

if __name__ == '__main__':
    L = 16
    shape = (L, L)
    N = L*L
    moment = 1
    delta=0.05
    # External magnetic field
    field = np.full(shape, 0)

    # Temperature (in units of energy)
    temperature = 4.0
    beta=1/temperature
    # Interaction (ferromagnetic if positive, antiferromagnetic if negative)
    interaction = 1
    global spins
    spins = np.full(shape,1)
    N_total=1000000
    isweep(N_total,beta)
    plt.matshow(spins)
    plt.show()


# using python to write likelihood part.

import numpy as np
from scipy.interpolate import interp1d
from scipy.integrate import quad
from scipy.optimize import brute, fmin, minimize_scalar
from scipy.special import gammainc
import os

jfactor_path = "/home/jguo/workspace/data/Fermi_Data/"

dwarves_list = ['carina', 'draco', 'fornax','leo_I','leo_II','sculptor','sextans',
                 'ursa_minor', 'bootes_I', 'canes_venatici_II', 'coma_berenices',
                 'hercules', 'segue_1', 'ursa_major_II', 'willman_1']
# dwarves_list = ['coma_berenices', 'draco']

class WhiteDWarfs():

    def __init__(self, dwares=dwarves_list, jfactor_path=jfactor_path):
        self.dwares = dwares
        self.jfactor_path = jfactor_path
        self.dw_infos = self.get_dw_infos

    def get_jfactors(self):
        dwarves_dict = {}
        jfactor_data = np.loadtxt(self.jfactor_path+"Jfactors.dat")
        with open(self.jfactor_path+"Jfactors.dat") as f:
            for line in f.readlines():
                if ("Row" in line):
                    _, _, num, dwarve = line.split()
                    dwarves_dict[dwarve] = int(num)
        dwarves_jfactor = np.array([jfactor_data[dwarves_dict[dw]] for dw in self.dwares])
        jfactor_roi = np.stack([dwarves_jfactor[:, 3], dwarves_jfactor[:, 4]])
        return jfactor_roi

    def get_ll_tabels(self):
        ll_tables = [np.loadtxt(self.jfactor_path+"likelihoods/like_"+dw+".txt") for dw in self.dwares]
        ll_tables = np.array(ll_tables)
        return ll_tables

    @property
    def get_dw_infos(self):
        dw_info_list = []
        jfactor_info = self.get_jfactors()
        ll_tables = self.get_ll_tabels()
        n_dw = ll_tables.shape[0]
        for i in range(n_dw):
            dw_i_dict = {}
            dw_i_dict['mu_j']  = jfactor_info[0][i]
            dw_i_dict['sigma_j']  = jfactor_info[1][i]
            dw_i_dict['ll_table'] = ll_tables[i]
            dw_info_list.append(dw_i_dict)
        self._dw_infos = dw_info_list  # np.array(dw_info_list)
        return self._dw_infos

class FermiPvalue():

    def __init__(self, whitedwarfs, spectrum_path, mdm):
        self.whitedwarfs = WhiteDWarfs().dw_infos # class of WhiteDwarfs
        self.nBin = 24
        self.j0 = 3.086e21
        self.sigmav0 = 1e-26
        self.mdm = mdm
        self.spectrum_path = spectrum_path
        self.spectrum = self.get_spectrum
        self.spectrum_path = spectrum_path

    @property
    def get_spectrum(self):
        fe2dndx = np.loadtxt(self.spectrum_path) # fe is some function of energy, dnde is dn/de.
        fe = fe2dndx[:, 0] # f(E) = np.log10(E/mdm).
        e = np.power(10, fe) * self.mdm
        dndx = fe2dndx[:, 1]
        dnde = dndx / (e * np.log(10.))
        self._spectrum = np.stack([e, dnde])
        return self._spectrum

    def  espectrum_e(self, e):
        '''Get the dn/de of certain E, return a function object calculating e * dn/de.'''
        energy = self.spectrum[0]
        dnde = self.spectrum[1]
        logfspectrum = interp1d(np.log10(energy), np.log10(dnde))
        if (e<energy.max() and e>energy.min()):
            return np.power(10, logfspectrum(np.log10(e)))* e  # np.exp(logfspectrum(np.log(e)))[0] * e
        else:
            return 0.0

    def get_up_dowm(self):
        """Get the up and down bound."""
        bin_down = self.whitedwarfs[0]["ll_table"][:, 0]  # get the energy bins from first dwarf likelihood table.
        bin_down = np.array(list(set(bin_down)))*1e-3
        bin_down = np.sort(bin_down)
        bin_up = self.whitedwarfs[0]["ll_table"][:, 1]  # get the energy bins from first dwarf likelihood table.
        bin_up = np.array(list(set(bin_up)))*1e-3
        bin_up = np.sort(bin_up)
        return bin_up, bin_down

    def get_predict_fluxes(self, sigmav):
        """Get the predict fluxes by the spectrum."""
        bin_down = self.whitedwarfs[0]["ll_table"][:, 0]  # get the energy bins from first dwarf likelihood table.
        bin_down = np.array(list(set(bin_down)))*1e-3
        bin_down = np.sort(bin_down)
        bin_up = self.whitedwarfs[0]["ll_table"][:, 1]  # get the energy bins from first dwarf likelihood table.
        bin_up = np.array(list(set(bin_up)))*1e-3
        bin_up = np.sort(bin_up)
        tol = min(self.espectrum_e(1e2),self.espectrum_e(1e5))*1e-10
        predict_fluxes = [quad(self.espectrum_e, down, up, epsabs=tol)[0] for down, up in zip(bin_down, bin_up)]
        predict_fluxes = np.array(predict_fluxes)
        predict_fluxes = predict_fluxes * self.sigmav0 * self.j0 / (4 * 2 * np.pi * self.mdm**2)
        predict_fluxes = predict_fluxes * (sigmav/self.sigmav0)
        return predict_fluxes

    def _dwraf_likelihood(self, dwarf, predict_fluxes):
        """Get the log likelihood of one dwarf."""
        j = dwarf['mu_j']
        sigma_j = dwarf['sigma_j']
        ll_table = dwarf["ll_table"]
        n_bin = len(set(ll_table[:, 0]))
        flux2ll = ll_table[:, 2:].reshape([n_bin, -1, 2])
        flux2ll_funcs = [interp1d(flux2ll[i][:, 0] *1e-3, flux2ll[i][:, 1], bounds_error=False, fill_value=-1e5) for i in range(n_bin)]
        def likeli(x):
            fluxes = predict_fluxes * 10**(j+x*sigma_j)/self.j0
            likeli_flux = np.array([func(flux) for func, flux in zip(flux2ll_funcs, fluxes)]).sum()
            likeli_jfactor = - 0.5*x**2 * np.log(np.sqrt(2*np.pi))
            likeli_dm = -1 * (likeli_flux + likeli_jfactor)  # Get minimum.
            return likeli_dm
        optimizer = minimize_scalar(likeli,method='bounded',bounds=(-5.0,5.0))
        likeli_dm = -optimizer.fun
        likeli_null = np.array([func(0) for func in flux2ll_funcs]).sum()
        return likeli_null, likeli_dm

    def dwarfs_likelihood(self,sigmav):
        """Get the combiine dwarfs loglikelihoods, including predict
        likelihood and null likelihood.
        """
        predict_fluxes = self.get_predict_fluxes(sigmav)
        dwarfs = self.whitedwarfs
        likelihoods = []
        for dwarf in dwarfs:
            likelihood = self._dwraf_likelihood(dwarf, predict_fluxes)
            likelihoods.append(likelihood)
        likelihoods = np.array(likelihoods)
        likelihoods = np.sum(likelihoods, axis=0)
        ts = 2 * (likelihoods[0] - likelihoods[1])
        pval = 1-gammainc(1/2.0, ts/2.0)
        return 1 - pval

class GetExcluding():

    def __init__(self, sigmavrange, spectrum_path, cls):
        self.sigmavrange=sigmavrange
        self.cls = cls
        self.whitedwarfs = WhiteDWarfs().dw_infos
        self.spectrum_path = spectrum_path

    def loss(self, fermip, sigmav):
        pval = fermip.dwarfs_likelihood(sigmav)
        atanhpval = np.arctanh(pval - 1e-9)
        atanclval = np.arctanh(self.cls - 1e-9)
        loss_value = (atanhpval - atanclval)**2
        return loss_value

    def get_excluding_point(self, mdm):
        """Get the up limit of sigmav with DM mass mdm."""
        fermip = FermiPvalue(self.whitedwarfs, self.spectrum_path, mdm)
        def loss_fn(logsigmav):
            sigmav = np.power(10., logsigmav)
            return self.loss(fermip, sigmav)
        logsigmav_min = np.log10(self.sigmavrange)[0]
        logsigmav_max = np.log10(self.sigmavrange)[1]
        n_steps = int(5 * (logsigmav_max - logsigmav_min))
        search = brute(loss_fn, [(logsigmav_min,logsigmav_max)], Ns=n_steps, full_output=False, finish=fmin)
        print(search[0])
        sigmav_ul = np.power(10, search[0])
        print("cls predict: ", fermip.dwarfs_likelihood(sigmav_ul))
        print("cls predict 2: ", fermip.get_predict_fluxes(0))
        return sigmav_ul

    def get_excluding_line(self, mrange):
        """Get the excluding line in DM mass range mrange in `GeV`"""
        mdm_down = mrange[0]
        mdm_up = mrange[1]
        mass = []
        sigmav_uls = []
        for mdm in range(mdm_down, mdm_up):
            sigmav_ul = self.get_excluding_point(mdm)
            print("one step!", sigmav_ul)
            mass.append(mdm)
            sigmav_uls.append(sigmav_ul)
        masses = np.array(mass)
        sigmav_uls = np.array(sigmav_uls)
        excluding_line = np.concatenate([masses, sigmav_uls], axis=1)
        return excluding_line

if __name__ == '__main__':
    cls = 0.9
    mmin = 130
    mmax = 10000
    spectrum_path = "./gammas_spectrum.dat"
    step = (np.log10(mmax) - np.log10(mmin))/20.
    mrange = np.arange(np.log10(mmin), np.log10(mmax), step)
    print(mrange)
    sigmavrange = np.array([1e-28, 1e-24])
    result_f = open('result.txt', 'a')
    # excluding_line = excluding.get_excluding_line(mrange)
    for logmdm in mrange:
        mdm = np.power(10, logmdm)
        print("mdm: ", mdm)
        command = "./spectrummc {}".format(mdm)
        os.system(command)
        excluding = GetExcluding(sigmavrange, spectrum_path, cls)
        sigmav_ul = excluding.get_excluding_point(mdm)
        result_f.write("{}     {}\r".format(mdm, sigmav_ul))
    result_f.close()


from markupsafe import Markup, escape
from odoo import _, api, fields, models
from pprint import pprint,pformat
from odoo import http
from odoo.osv import expression
from odoo.tools import consteq, plaintext2html
from odoo.addons.helpdesk.controllers import portal
from odoo.addons.portal.controllers import mail
from odoo.addons.mail.models.mail_message import Message
from odoo.addons.mail.models.mail_thread import MailThread
from odoo.addons.web.controllers.main import Binary
from odoo.http import request
import logging

from odoo.exceptions import ValidationError

_logger = logging.getLogger(__name__)

class HelpdeskPrivacy(models.Model):
    _inherit = 'mail.mail'

    def dump(self, obj):
        for attr in dir(obj):
            _logger.info("obj.%s = %r" % (attr, getattr(obj, attr)))

    def send(self, auto_commit=False, raise_exception=False):
        for mail_id in self.ids:
            mail = None
            try:
                mail = self.browse(mail_id)
                if mail.state == 'outgoing' and mail.model == 'helpdesk.ticket':
                    #
                    # Overwrite sender
                    #
                    mail.email_from = mail.reply_to
                res = super(HelpdeskPrivacy, self).send()
                return res
            except Exception as e:
                _logger.exception('failed sending mail')

class HelpdeskPrivacyCustomerPortal(portal.CustomerPortal):

    def dump(self, obj):
        for attr in dir(obj):
            _logger.info("obj.%s = %r" % (attr, getattr(obj, attr)))

    @http.route([
        "/helpdesk/ticket/<int:ticket_id>",
        "/helpdesk/ticket/<int:ticket_id>/<access_token>",
        '/my/ticket/<int:ticket_id>',
        '/my/ticket/<int:ticket_id>/<access_token>'
    ], type='http', auth="public", website=True)
    def tickets_followup(self, ticket_id=None, access_token=None, **kw):
        try:
            ticket_sudo = self._document_check_access('helpdesk.ticket', ticket_id, access_token)
        except (AccessError, MissingError):
            return request.redirect('/my')

        values = self._ticket_get_page_view_values(ticket_sudo, access_token, **kw)
        return request.render("odoo_helpdesk_privacy.tickets_followup", values)

class BinaryPrivacy(Binary):

    def dump(self, obj):
        for attr in dir(obj):
            _logger.info("obj.%s = %r" % (attr, getattr(obj, attr)))

    @http.route(['/web/image',
        '/web/image/<string:xmlid>',
        '/web/image/<string:xmlid>/<string:filename>',
        '/web/image/<string:xmlid>/<int:width>x<int:height>',
        '/web/image/<string:xmlid>/<int:width>x<int:height>/<string:filename>',
        '/web/image/<string:model>/<int:id>/<string:field>',
        '/web/image/<string:model>/<int:id>/<string:field>/<string:filename>',
        '/web/image/<string:model>/<int:id>/<string:field>/<int:width>x<int:height>',
        '/web/image/<string:model>/<int:id>/<string:field>/<int:width>x<int:height>/<string:filename>',
        '/web/image/<int:id>',
        '/web/image/<int:id>/<string:filename>',
        '/web/image/<int:id>/<int:width>x<int:height>',
        '/web/image/<int:id>/<int:width>x<int:height>/<string:filename>',
        '/web/image/<int:id>-<string:unique>',
        '/web/image/<int:id>-<string:unique>/<string:filename>',
        '/web/image/<int:id>-<string:unique>/<int:width>x<int:height>',
        '/web/image/<int:id>-<string:unique>/<int:width>x<int:height>/<string:filename>'], type='http', auth="public")
    def content_image(self, xmlid=None, model='ir.attachment', id=None, field='datas',
                      filename_field='name', unique=None, filename=None, mimetype=None,
                      download=None, width=0, height=0, crop=False, access_token=None,
                      **kwargs):

        if model == 'mail.message' and width and height:
            #
            # TODO: show the company logo if possible
            #
            id = 0
        return super(BinaryPrivacy, self).content_image(xmlid, model, id, field,
                      filename_field, unique, filename, mimetype,
                      download, width, height, crop, access_token,
                      **kwargs)

class MailMessage(Message):
    _inherit = 'mail.message'

    # for attr in dir(author_avatar):
    #     _logger.info("author_avatar.%s = %r" % (attr, getattr(author_avatar, attr)))

    def dump(self, obj):
        for attr in dir(obj):
            _logger.info("obj.%s = %r" % (attr, getattr(obj, attr)))

    def is_internal_user(self, id):
        query = """select * from res_users where partner_id=%(id)s;"""
        self._cr.execute(
            query,
            {
                'id': id
            },
        )
        res = self._cr.fetchone()
        if res:
            return True
        return False

    def _message_format(self, fnames, format_reply=True):
        """ Override the method to add information about a publisher comment
        on each rating messages if requested, and compute a plaintext value of it.
        """
        vals_list = super(MailMessage, self)._message_format(fnames, format_reply=format_reply)

        for vals in vals_list:
            # self.dump(vals.items)
            if 'is_internal' in vals.keys() and not vals['is_internal']:
                if vals['model'] == 'helpdesk.ticket':
                    if self.is_internal_user(vals['author_id'][0]):
                        vals['author_id'] = (1, self.env.company.name)

        return vals_list

    # @api.model
    # def default_get(self, fields):
    #     res = super(MailMessage, self).default_get(fields)
    #     missing_author = 'author_id' in fields and 'author_id' not in res
    #     missing_email_from = 'email_from' in fields and 'email_from' not in res
    #     if missing_author or missing_email_from:
    #         author_id, email_from = self.env['mail.thread']._message_compute_author(res.get('author_id'), res.get('email_from'), raise_exception=False)
    #         if missing_email_from:
    #             res['email_from'] = email_from
    #         if missing_author:
    #             res['author_id'] = author_id
    #     return res

#
# Outgoing emails from admin direct send mail
# Effect: Change signature
#
class PrivacyMailThread(MailThread):
    _inherit = 'mail.thread'

    def dump(self, obj):
        for attr in dir(obj):
            _logger.info("obj.%s = %r" % (attr, getattr(obj, attr)))

    @api.model
    def _notify_prepare_template_context(self, message, msg_vals, model_description=False, mail_auto_delete=True):

        res = super(PrivacyMailThread, self)._notify_prepare_template_context(message, msg_vals, model_description, mail_auto_delete)
        model = msg_vals.get('model') if msg_vals else message.model

        if model == 'helpdesk.ticket':
            signature = '--
%s' % res['company'].name
            signature = Markup(signature)
            res['signature'] = signature

        return res


"""
Install script

modifications file: modifications.json, this structure (small example):

{
	"old":
	{
		"tech_tree":
		{
			"basicRocketry":
			{
				"title":"#autoLOC_501020 //#autoLOC_501020 = Basic Rocketry",
				"description":"#autoLOC_501023 //#autoLOC_501023 = How hard can Rocket Science be anyway?",
				"cost":5,
				"hideEmpty":"False",#TODO: what does this do?
				"nodeName":"node1_basicRocketry",
				"anyToUnlock":"False",#TODO: what does this do?
				"icon":"RDicon_rocketry-basic",
				"pos":"-2300,1200,0",
				"scale":"0.6",
				"parents":
				[
					{
						"parentID":"start",
						"lineFrom":"RIGHT",
						"lineTo":"LEFT"
					},
				],
			},

			#testNode is not present
		},
		"parts":
		{
			"noseCone":
			{
				"cfg_path":"Parts/Aero/aerodynamicNoseCone/aerodynamicNoseCone.cfg",
				"tech_id":"stability",
				#all other fields (e.g. title) ignored
			},
			"fairingSize1":
			{
				"cfg_path":"Parts/Aero/fairings/fairingSize1.cfg",
				"tech_id":"advConstruction",
				#all other fields (e.g. title) ignored
			},
		},
	},
	"new",
	{
		"tech_tree":
		{
			#basicRocketry node is not present
			"testNode":
			{
				exists:true,
				#id is populated based on the id provided 2 levels up from here

				"title":"Test Node",
				"description":"This is a Test R&D node",
				"cost":50,
				#hideEmpty is false except for nanolathing and experimental motors??? what does it do? for now we're going to populate ours as false always
				#nodeName is populated automatically with node<depth [distance in tree from start]>_id
				#anyToUnlock (what does this do????)
				"icon":"RDicon_test_node",
				#pos is calculated automatically
				#scale is always 0.6?
				"parents":
				[
					{
						"parentID":"engineering101",
						#from is always 'RIGHT'
						#to is always 'LEFT'
					},
					{
						"parentID":"basicRocketry"
					},
				],
			},
		},
		"parts":
		{
			"noseCone":
			{
				"cfg_path":"Parts/Aero/aerodynamicNoseCone/aerodynamicNoseCone.cfg",
				"tech_id":"stability",#unchanged
				#all other fields (e.g. title) ignored
			},
			"fairingSize1":
			{
				"cfg_path":"Parts/Aero/fairings/fairingSize1.cfg",
				"tech_id":"testNode",#changed
				#all other fields (e.g. title) ignored
			},
		}
	},
}

NOTE: the modifications file will generally only define 'new' OR 'old' but not both -- running this script with 'old' defined will revert, running it with 'new' defined will install
"""

import json
import argparse
import re
import time
import os
import warnings

# warnings.filterwarnings('error',category=SyntaxWarning)

MODIFIERS_PARENTS_LIST_KEY = "parents"
#TODO (eventually) support for non-squad tree data?
TECH_TREE_CFG_FILE_LOC_FROM_GAMEDATA_DIR = "/Squad/Resources/TechTree.cfg"

X_MIN = -2500
Y_MIN = 500
X_GAP = 200
Y_GAP = 60

clean_bracket_re = re.compile(r"\s*[{}]\s*")
dirty_bracket_re = re.compile(r"(.*)[{}](.*)")
whitespace_only_re = re.compile(r"^\s*$")
cfg_comment_re = re.compile(r"([^/]*)//(.*)")
comment_describes_autoLOC_re = re.compile(r"(.*)(title|description)\s*=\s*(?:.*//\s*(?:#?autoLOC_\S*\s*=)?\s*)?(.*)")
defn_re = re.compile(r"\s+(\w+)\s*=\s?(.*)")#capture group 1: identifier; group 2: definition
rd_node_re = re.compile(r"\s*RDNode")
par_begin_re = re.compile(r"\s*Parent")
defn_ident_is_id_re = re.compile(r"[Ii][Dd].*")
id_re = re.compile(r"\s*name\s*=\s*(\S+)")
tech_req_re = re.compile(r"\s*TechRequired\s*=\s*(\S+)")
partdef_begin_re = re.compile(r"(?:.{0,3}|\s*)PART")
open_brace_re = re.compile(r".*\{")
closed_brace_re = re.compile(r".*\}")
part_title_re = re.compile(r".*title\s*=\s*(?:.*//\s*(?:#?autoLOC_\S*\s*=)?\s*)?(.*)")

#whitelist this (only "PART" is allowed)
part_igdef_re = re.compile(r"\s*[A-Z]+\s*$")
def line_begins_ignored_defn(line):
	if partdef_begin_re.match(line) is not None:
		return False
	#add any other whitelisted matches here ^^^
	return part_igdef_re.match(line) is not None

tree_auto_fields = ['id',
					'hideEmpty',
					'nodeName',
					'anyToUnlock',
					'pos',
					'scale']
tree_parent_auto_fields = ['lineFrom',
						   'lineTo']

def parse_existing_tree_file(tree_path):
	flines = None
	with open(tree_path,'r') as f:
		flines = f.readlines()
		flines = [line[:-1] for line in flines]

	#preprocess the lines
	#delete the first line (it just says 'TechTree', or if it doesn't, we have the wrong file anyway)
	if('TechTree' != flines[0][-8:]):
		raise ValueError("File provided for the tech tree config ({}) is not a tech tree file (the first line is not 'TechTree' with no whitespace)".format(tree_path))
	else:
		flines.pop(0)
	#delete all the brackets (they're not necessary)
	flines_updated = []
	for line in flines:
		clean_match = clean_bracket_re.match(line)
		if clean_match is None:
			dirty_match = dirty_bracket_re.match(line)
			if dirty_match is not None:
				#either the beginning or the end of the line has non-whitespace
				#this line must be separated before the bracket can be deleted
				for group in range(1,2+1):
					#add all and only full and not whitespace-only groups
					if (dirty_match.group(group) is not None) and (whitespace_only_re.match(dirty_match.group(group)) is not None):
						flines_updated.append(dirty_match.group(group))
			else:
				#non bracket line
				flines_updated.append(line)
		else:
			#clean bracket line
			#do nothing
			pass
	flines = flines_updated
	#delete all the comments
	for i,line in enumerate(flines):
		comment_match =  cfg_comment_re.match(line)
		if comment_match is not None:
			autoloc_match = comment_describes_autoLOC_re.match(line)
			if autoloc_match is None:
				flines[i] = ""
			else:
				#drop all of the autoLOC references from here, but keep the description
				flines[i] = autoloc_match.group(1) + autoloc_match.group(2) + " = " + autoloc_match.group(3)
	flines_updated = []
	#clear whitespace
	for i,line in enumerate(flines):
		if whitespace_only_re.match(line) is None:
			flines_updated.append(line)
	flines = flines_updated

	out = {}
	current_dict_path = []#tells us where we're adding to in the out-dict

	searching_for_real_id = False#tells us whether we still haven't found this node's ID (we'll warn if we hit the next node before we do)
	'''
	temp_id_index = 0
	temp_id_prefix = "TEMPORARY_ID_{}_".format(time.time())#ensures this is unique
	def get_temp_id():
		r = temp_id_prefix + str(temp_id_index) #what the fuck. why is temp_id_index out of scope.
		temp_id_index += 1
		return r
	'''

	def get_temp_id():
		return "TEMPORARY_ID_{}".format(time.time())

	for line in flines:
		#rdnode defn begin
		rdnode_begin = rd_node_re.match(line) is not None
		#parent definition begin
		par_begin = par_begin_re.match(line) is not None
		#definition of variable
		def_match = defn_re.match(line)
		def_ident = None
		def_val = None
		if def_match is not None:
			def_ident = def_match.group(1)
			def_val = def_match.group(2)

		#process
		if rdnode_begin:
			#check if we got an ID for the last node
			if searching_for_real_id:
				#we didn't, warn the user
				warnings.warn("'id = <val>' definition was not found for an RDNode. Its ID will be {}".format(current_dict_path[0]), SyntaxWarning)
			#prepare to process the new rd-node
			#create a temp ID (we'll replace it when we learn the new ID, or throw a warning if we never do)
			current_dict_path = [get_temp_id()]
			out.update({current_dict_path[0]:{}})
			searching_for_real_id = True

		elif par_begin:
			#if this is the first parent, add 'parents' to the dict path and make a new entry
			#otherwise just make a new entry
			if 3 != len(current_dict_path):
				par_list_idx = -1
				out[current_dict_path[0]].update({MODIFIERS_PARENTS_LIST_KEY:[]})
				current_dict_path.append(MODIFIERS_PARENTS_LIST_KEY)
			else:
				par_list_idx = current_dict_path.pop(-1)#drop (and record) the parent list index

			out[current_dict_path[0]][current_dict_path[1]].append({})#the new parent object
			current_dict_path.append(par_list_idx + 1)#which has index <prev> + 1

		elif def_match is not None:

			#check if the ident is 'id'
			if defn_ident_is_id_re.match(def_ident) is not None:
				#we need to change the key that refers to this node
				#copy contents
				out[def_val] = out[current_dict_path[0]]
				#delete old stuff
				del out[current_dict_path[0]]
				#update the path
				current_dict_path[0] = def_val
				#unset the flag
				searching_for_real_id = False

			#otherwise, just add it in
			#different location if processing parents
			if 3 == len(current_dict_path):
				out[current_dict_path[0]][current_dict_path[1]][current_dict_path[2]].update({def_ident:def_val})
			else:
				out[current_dict_path[0]].update({def_ident:def_val})

	return out

def parse_existing_part_files(path_to_parts_dir,parts_dict=None):

	if parts_dict is None:
		parts_dict = {}#consists of <path>:{ "old_tech_id":<id>, "new_tech_id":"" }

	ignore_files = {'VariantThemes.cfg'}
	filetype_filter = '.cfg'

	#first, find every .cfg file under this directory (dropping anything from the ignore list)
	flist = set()
	for dirpath,dirnames,filenames in os.walk(path_to_parts_dir):
		flist = flist.union({dirpath + '/' + fname for fname in filenames if (fname[-4:] == filetype_filter) and (fname not in ignore_files)})

	for fpath in flist:
		flines = []
		with open(fpath,'r',errors='replace') as f:
			flines = f.readlines()
			flines = [line[:-1] for line in flines]

		flines_re_out = []
		re_out_file = False
		line_id = 0
		while line_id < len(flines):
			#start by looking for a part-definition beginning
			partdef_found = False
			part_begin_loc = None
			for line in flines[line_id:]:
				line_id += 1
				flines_re_out.append(line)
				if partdef_begin_re.match(line) is not None:
					partdef_found = True
					part_begin_loc = line_id-1
					break

			if not partdef_found:
				break

			#look for the id and the tech-req (order doesn't matter)
			id = None
			treq = None
			#optional: look for a part title match (can help with templating)
			title = None

			ignored_def_brace_level = 0  #used to ignore certain definitions
			watch_for_ignore_open_brace = False

			for line in flines[line_id:]:
				line_id += 1
				flines_re_out.append(line)

				#ignored sections
				if (ignored_def_brace_level > 0) or (watch_for_ignore_open_brace):
					#look for an open brace
					if open_brace_re.match(line) is not None:
						if watch_for_ignore_open_brace:
							#we found it, stop looking
							watch_for_ignore_open_brace = False
						ignored_def_brace_level += 1
					#look for a close brace
					elif closed_brace_re.match(line) is not None:
						ignored_def_brace_level -= 1

					continue#don't do anything to process this part (it's either closed brace [do nothing anyway] or within a module definition [needs to  be ignored])
				#check if there's a new ignored definition starting
				elif line_begins_ignored_defn(line):
					ignored_def_brace_level = 0
					watch_for_ignore_open_brace = True#need this because the open brace should  be on the next line (or later, if the dev decides to put blank lines between us  and it *RAGE*)
					continue

				#new part definition
				if partdef_begin_re.match(line) is not None:
					line_id -= 1
					break

				idmatch = id_re.match(line)
				treqmatch = tech_req_re.match(line)
				titlematch = part_title_re.match(line)
				if idmatch is not None:
					id = idmatch.group(1)
					#reorder if they were out of order
					if(treq is not None):
						warnings.warn("config file {} contained a part (part def begins on line {}) with tech-req defined BEFORE part id (field: name). Reordering in the file".format(fpath,part_begin_loc), SyntaxWarning)
						re_out_file = True
						#delete the id line from the new lines
						idline = flines_re_out.pop(-1)
						#add it in at the beginning
						flines_re_out.insert(part_begin_loc+2,idline)
				if treqmatch is not None:
					treq = treqmatch.group(1)
				if titlematch is not None:
					title = titlematch.group(1)

			if (id is None) or (treq is None):
				warnings.warn("config file {} contained a part (part def begins on line {}, error found on line {})  that failed to define both tech-requirement and part id (field: name). Ignoring this part (most likely: part does not have a tech requirement [e.g. flags or eva suits])".format(fpath,part_begin_loc,line_id-1), SyntaxWarning)
			else:
				id_attach_val = 0
				new_id = id
				while new_id in parts_dict:
					#attach an index to make it unique
					warnings.warn("more than one part has name {}".format(new_id), SyntaxWarning)
					new_id = id + str(id_attach_val)
					id_attach_val += 1

				id = new_id
				if title is not None:
					parts_dict.update({id:{
						'cfg_path':fpath,
						'tech_id':treq,
						'title':title
						}})
				else:
					#warn that we didn't find the title for this one
					warnings.warn("no title field was found for part with id {} (file {})".format(id,fpath))
					parts_dict.update({id: {
						'cfg_path': fpath,
						'tech_id': treq
					}})

		if re_out_file:
			#something was wrong in the file, fix it and re-output the file
			with open(fpath,'w',errors='replace') as f:
				f.writelines([line + '\n' for line in flines_re_out])

	return parts_dict

def get_modifications(mod_file):
	return json.load(open(mod_file,'r'))

def generate_nodes_depth(tech_tree):
	#how far is each node from 'start'
	return {node:get_node_depth(tech_tree,node) for node in tech_tree}

def generate_nodes_pos(tech_tree,node_depths=None):
	#TODO improve the loooks of this layout algorithm
	if node_depths is None:
		node_depths = generate_nodes_depth(tech_tree)

	#make a version of the tree that's "forwards" (nodes map to lists of their children)
	forward_tree = {node:[] for node in tech_tree}
	for node in tech_tree:
		if 'parents' not in tech_tree[node]:
			continue
		for par in tech_tree[node]['parents']:
			forward_tree[par['parentID']].append(node)

	#assign x-pos (start = X_MIN, then each node goes by its depth)
	for node in tech_tree:
		tech_tree[node].update({'pos':[X_MIN + (X_GAP * node_depths[node]), None, 0]})

	#create a histogram of node depths
	depth_hist = {i:0 for i in range(max(node_depths.values())+1)}
	for node in node_depths:
		depth_hist[node_depths[node]] += 1

	#the widest (y is width here) part will be perfectly dense (implicitly)
	num_in_widest_depth = max(depth_hist.values())
	ymax = Y_MIN + (Y_GAP * num_in_widest_depth)

	#now we do a depth-first search through the tree. the first path we traverse goes along the top edge (lowest available y-values), then so on from there
	stack = ['start']
	next_yv_by_depth = {d:Y_MIN for d in range(num_in_widest_depth)}
	tech_tree['start']['pos'][1] = next_yv_by_depth[0]#this is normally added on by the parent, but start has no parent, so do it now
	next_yv_by_depth[0] += ymax / depth_hist[0]#this shouldn't be necessary -- only 'start' should be at depth 0
	#we'll use the y-value in tech_tree[node]['pos'] as a 'seen' value
	while len(stack) > 0:
		cur = stack.pop(-1)#take from end

		#y-value is already assigned, just add children to the stack
		for ch in forward_tree[cur]:
			#check if seen
			if tech_tree[ch]['pos'][1] is not None:
				#seen
				continue
			else:
				#not seen, give y-pos and add to stack
				stack.append(ch)
				tech_tree[ch]['pos'][1] = next_yv_by_depth[node_depths[ch]]
				#move the yv for this depth to the next position
				next_yv_by_depth[node_depths[ch]] += ymax / depth_hist[node_depths[ch]]

	#all of the modifications were done in-place, so we are done

def auto_populate_missing_fields(tree_mods):
	#only touches certain fields:
	#	id
	#	hideEmpty
	#	nodeName
	#	anyToUnlock
	#	pos
	#	scale
	#	<parent>
	#		lineFrom
	#		lineTo
	#and even then only if the user didn't already populate them

	for tech_id in tree_mods:
		#populate id
		if 'id' not in tree_mods[tech_id]:
			tree_mods[tech_id].update({'id':tech_id})
		#hideEmpty
		if 'hideEmpty' not in tree_mods[tech_id]:
			tree_mods[tech_id].update({'hideEmpty':'True'})
		#anyToUnlock
		if 'anyToUnlock' not in tree_mods[tech_id]:
			tree_mods[tech_id].update({'anyToUnlock':'True'})#TODO wtf is this
		#scale
		if 'scale' not in tree_mods[tech_id]:
			tree_mods[tech_id].update({'scale':'0.6'})

		#parent info
		if MODIFIERS_PARENTS_LIST_KEY in tree_mods[tech_id]:
			for i in range(len(tree_mods[tech_id][MODIFIERS_PARENTS_LIST_KEY])):
				if 'lineFrom' not in tree_mods[tech_id][MODIFIERS_PARENTS_LIST_KEY][i]:
					tree_mods[tech_id][MODIFIERS_PARENTS_LIST_KEY][i].update({'lineFrom':'RIGHT'})
				if 'lineTo' not in tree_mods[tech_id][MODIFIERS_PARENTS_LIST_KEY][i]:
					tree_mods[tech_id][MODIFIERS_PARENTS_LIST_KEY][i].update({'lineTo':'LEFT'})

	#do pos and nodeName (depth) outside of the main loop
	node_depths = generate_nodes_depth(tree_mods)
	#nodeName
	#node<depth>_<id>
	for node in node_depths:
		tree_mods[node].update({'nodeName': 'node{}_{}'.format(node_depths[node],node)})
	#pos
	generate_nodes_pos(tree_mods,node_depths=node_depths)
	#done

def output_modifications(mods,mod_file):
	json.dump(mods,open(mod_file,'w'),indent='\t')

def apply_tree_modifications(tree_mods,tree_path):
	with open(tree_path,'w') as f:
		f.write('TechTree\n')
		f.write('{\n')

		for rdnode in tree_mods:
			f.write('\tRDNode\n')
			f.write('\t{\n')
			for field in tree_mods[rdnode]:
				#parents section handled separately
				if MODIFIERS_PARENTS_LIST_KEY == field:
					continue#parents should come last
				else:
					if 'pos' == field:
						f.write('\t\t{} = {},{},{}\n'.format(field,*tree_mods[rdnode][field]))
					else:
						f.write('\t\t{} = {}\n'.format(field,tree_mods[rdnode][field]))
			if MODIFIERS_PARENTS_LIST_KEY in tree_mods[rdnode]:
				for par in tree_mods[rdnode][MODIFIERS_PARENTS_LIST_KEY]:
					f.write('\t\tParent\n')
					f.write('\t\t{\n')
					for parfield in par:
						f.write('\t\t\t{} = {}\n'.format(parfield,par[parfield]))
					f.write('\t\t}\n')
			f.write('\t}\n')

		f.write('}\n')

def apply_part_modifications(part_mods):
	for part_id in part_mods:
		path = part_mods[part_id]['cfg_path']
		treq = part_mods[part_id]['tech_id']

		flines = []
		with open(path,'r',errors='replace') as f:
			flines = f.readlines()
			flines = [line[:-1] for line in flines]

		#look for the 'name' field that matches
		name_fd_idx = None
		for i,line in enumerate(flines):
			if id_re.match(line) is not None:
				name_fd_idx = i
				break

		if name_fd_idx is None:
			raise ValueError("part ID {} not found in file {}".format(part_id,path))
		#look for the NEXT 'TechRequired' field
		treq_idx = None
		for i,line in enumerate(flines[name_fd_idx:]):
			if tech_req_re.match(line) is not None:
				treq_idx = i + name_fd_idx
				break

		#change the line
		flines[treq_idx] = "\tTechRequired = {}".format(treq)

		#output the file again
		with open(path,'w',errors='replace') as f:
			f.writelines([l + '\n' for l in  flines])

def get_node_depth(tech_tree,node):
	if('parents' not in tech_tree[node]):
		return 0

	return min([get_node_depth(tech_tree,par['parentID']) for par in tech_tree[node]['parents']]) + 1

if __name__ == '__main__':
	#main stuff

	parser = argparse.ArgumentParser(description="KSP tech tree modification install/uninstall/template creation")

	parser.add_argument('kspdir',type=str,help='KSP top level directory (this is the directory that contains the Launcher.exe executable and the GameData directory)')
	parser.add_argument('action',type=str,choices=['install','uninstall','template'],default='template',help='What do you want this program to do? (note: "template" will create a template of all of the parts in your game directory and the existing tech tree in the format this program expects)')
	parser.add_argument('modfile',type=str,help='Location of the file which contains (or will contain, in the case of template creation) the modifications to make to the tech tree. NOTE: expected file type/format: json')

	args = parser.parse_args()

	ksp_dir = args.kspdir
	game_data_dir = ksp_dir + '/GameData'
	action = args.action
	mod_file = args.modfile

	#verify inputs

	#make sure kspdir exists and does contain a subdirectory called GameData
	if not os.path.isdir(ksp_dir):
		raise ValueError("{} -- directory not found".format(ksp_dir))
	if not os.path.isdir(game_data_dir):
		raise ValueError("{} is not a valid KSP top level directory (GameData subdirectory does not exist)".format(ksp_dir))

	#argparse verifies action for us

	#check if the modfile exists
	if not os.path.isfile(mod_file):
		#it doesn't -- this is a problem in all types except template
		if 'template' != action:
			raise ValueError("{} -- file not found. Only template mode may fail to specify an existing file")
	#warn if the file the user gave doesn't end in '.json'
	if '.json' != mod_file[-5:]:
		warnings.warn("mod file path provided does not use the '.json' suffix -- the data stored in this file is in json format", SyntaxWarning)
	#trying to load the json in get_modifications will throw an error if it isn't syntactically correct, so no need to do so here

	#load/parse the existing tech tree
	current_tech_tree = parse_existing_tree_file(game_data_dir + TECH_TREE_CFG_FILE_LOC_FROM_GAMEDATA_DIR)
	#load/parse the existing parts
	#	find all of the 'Parts' directories
	pdirs = {dirpath for dirpath,_,_ in os.walk(game_data_dir) if '\\Parts' == dirpath[-6:]}
	#	parse all of the parts from these directories
	current_parts = {}
	for pdir in pdirs:
		current_parts = parse_existing_part_files(pdir, parts_dict = current_parts)

	#template creation
	if 'template' == action:
		#don't load the json from the file given
		#throw away any fields which we would auto-populate anyway (at best they confuse the user)
		new_tech_tree = {}
		for tech in current_tech_tree:
			new_tech_tree.update({tech:{field:current_tech_tree[tech][field] for field in current_tech_tree[tech] if (field not in tree_auto_fields) and ('parents' != field)}})
			if 'parents' in current_tech_tree[tech]:
				new_tech_tree[tech].update({'parents':[
					{field:current_tech_tree[tech]['parents'][i][field] for field in current_tech_tree[tech]['parents'][i] if field not in tree_parent_auto_fields}
					for i in range(len(current_tech_tree[tech]['parents'])) ]})

		current_tech_tree = new_tech_tree

		#format the dict we're going to throw into the mod file
		modf_data = {'old':{'tech_tree':{},'parts':{}},
					 'new':{'tech_tree':{},'parts':{}}}
		#put the stuff we just read into 'new'
		modf_data['new']['tech_tree'] = current_tech_tree
		modf_data['new']['parts'] = current_parts

		#output the data and exit normally
		output_modifications(modf_data,mod_file)
		exit(0)
	else:
		#load in the json
		all_modf_data = get_modifications(mod_file)
		#installation
		if 'install' == action:
			#auto populate missing stuff from the file
			auto_populate_missing_fields(all_modf_data['new']['tech_tree'])
			#format the old modfile data
			all_modf_data.update({'old':{'tech_tree':current_tech_tree, 'parts':current_parts}})
			# exit(1)
			#push the changes to the json file
			output_modifications(all_modf_data,mod_file)
			#finally, do the install itself
			apply_tree_modifications(all_modf_data['new']['tech_tree'], game_data_dir + TECH_TREE_CFG_FILE_LOC_FROM_GAMEDATA_DIR)
			apply_part_modifications(all_modf_data['new']['parts'])
			#done, exit normally
			exit(0)
		#uninstallation (revert based on file)
		elif 'uninstall' == action:
			#works just like install, but we populate the 'old' values, not the 'new' ones
			#just do the (un)install itself
			apply_tree_modifications(all_modf_data['old']['tech_tree'],
									 game_data_dir+TECH_TREE_CFG_FILE_LOC_FROM_GAMEDATA_DIR)
			apply_part_modifications(all_modf_data['old']['parts'])
			#done, exit normally
			exit(0)


"""Web Routes."""

from masonite.routes import Get, Post, Put, Delete, RouteGroup
ROUTES = [
    Get("/", "WelcomeController@show").name("welcome"),

    RouteGroup([
        Get("/", "NoteController@index").name("index"),
        Get("/@id", "NoteController@show").name("show"),
        Post('/', "NoteController@create").name("create"),
        Put('/@id', "NoteController@update").name("update"),
        Delete('/@id', "NoteController@destroy").name("destroy")
    ], prefix="/notes", name="notes")
]


import decimal
import time

print("""
=========================================================================================
| 888888b.  YP                                                                          |
| 88   Y8b          Pi_Calculator by julien_Blue | inspired by DorFuchs                 |
| 88   d8P  88      v2.00    <2019>                                                     |
| 888888P'  88      Simple Pi Calculator using the Brent & Salamin algorithm for Pi     |
| 88        88      See changelog.txt for further information                           |
| 88        88                                                                          |
=========================================================================================
""")

def main():
    decimal.getcontext().prec = int(input("How many decimal places of Pi should be calculated?\nSet: ")) + 1

    try:
        d = open("pi_" + str(decimal.getcontext().prec - 1) + ".txt", "w")  # open pi.txt file
        d.write(str(pi()))  # calls pi() function and writes return value as a string to ext. file
    except PermissionError:
        import sys
        print("Failed to create or open external file... (PermissionError)")
        input("Press Enter to Exit")
        sys.exit(0)  # interrupts program if failed
    finally:
        d.close()  # close opened file to ensure it's consistence
        print("Value successfully written to external file (pi_" + str(decimal.getcontext().prec - 1) + ".txt)")
        input("Press Enter to Exit")

def pi():
    decimal.getcontext().prec += 2  # extra digits to ensure proper rounding
    decimal.getcontext().rounding = 'ROUND_DOWN'    # 'ROUND_DOWN' to ensure last digit of calculation is right
    start_time = time.time()

    # start values
    prev = 0    # 'prev' has to be unequal to 'a' to enter while loop
    a = 1
    b = 1 / decimal.Decimal(2).sqrt()
    s = 1 / decimal.Decimal(4)
    n = 0

    while(a != prev):  # recalculates value until maximum precision is reached
        prev = a
        A = (a + b) / 2 # arithmetic mean
        B = decimal.Decimal(a * b).sqrt()   # geometric mean
        S = s - 2**n * (a - A)**2   # variable s by Brent & Salamin

        # upper case letter -> n+1
        # lower case letter -> n

        # updating values for next step of calculation
        a = A
        b = B
        s = S
        n += 1

    pi = A**2/S # 'A' squared divided by 'S' equals 'pi'

    finish_time = time.time()
    decimal.getcontext().prec -= 2  # removing extra digits to write only the intended amout of decimal places to external file
    print("Done in", n + 1, "steps! (" + str(round(finish_time - start_time, 3)) + "sec)")
    return decimal.getcontext().plus(pi)    # applies the context precision and roundings to 'pi'

main()  # main function call

# end


# -*- coding: utf-8 -*-
"""
Created on Thu May 23 21:58:22 2019

@author: zsl
"""
from sklearn import datasets
'''
car
'''
iris = datasets.load_iris()
wine = datasets.load_wine()
X, y = iris.data, iris.target

dataSet = []
labels = []
fileIn = open('car.data')
for line in fileIn.readlines():
	lineArr = line.strip().split('\t')
	dataSet.append([float(lineArr[0]), float(lineArr[1])])
	labels.append(float(lineArr[2]))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Mar  2 02:03:08 2018

@author: chandan
"""
"""
Contains whole code for chatbot.

1. How to build it
2. how to train it
3. How we test it

"""
# Building a ChatBot with DeepNLP

#Importing the libraries
# numpy - to work with arrays. tensorflow - Deeplearning re - to clean the text. time - to measure traiing time

import numpy as np
import tensorflow as tf
import re
import time

############################# PART 1 - DATA PREPROCESSING  #######################

#Importing the dataset

lines = open('movie_lines.txt', encoding = 'utf-8', errors = 'ignore').read().split('\n')
conversations = open('movie_conversations.txt', encoding = 'utf-8', errors = 'ignore').read().split('\n')

#creating a dictionary that maps each line and its id

id2line = {}

for line in lines:
    _line = line.split(' +++$+++ ')
    if len(_line) == 5:
        id2line[_line[0]] = _line[4]

#Creating a list of all of the cobersation

conversations_ids = []

for conversation in conversations[:-1]:
    _conversation = conversation.split(' +++$+++ ')[-1][1:-1].replace("'", "").replace(" ","")
    conversations_ids.append(_conversation.split(','))

#Getting separately the questions and the answers

questions = []
answers = []

for conversation in conversations_ids:
    for i in range(len(conversation)-1):
        questions.append(id2line[conversation[i]])
        answers.append(id2line[conversation[i+1]])

#Cleaning the texts

def clean_text(text):
    text = text.lower()
    text = re.sub(r"i'm","i am", text)
    text = re.sub(r"he's","he is", text)
    text = re.sub(r"she's","she is", text)
    text = re.sub(r"that's","that is", text)
    text = re.sub(r"what's","what is", text)
    text = re.sub(r"where's","where is", text)
    text = re.sub(r"\'ll"," will", text)
    text = re.sub(r"\'ve"," have", text)
    text = re.sub(r"\'re"," are", text)
    text = re.sub(r"\'d"," would", text)
    text = re.sub(r"won't","will not", text)
    text = re.sub(r"can't","cannot", text)
    text = re.sub(r"\'bout","about", text)
    text = re.sub(r"[-()\"#/@;:<>{}+=~|.?,]","",text)
    return text

#cleaning the questions

clean_questions = []

for question in questions:
    clean_questions.append(clean_text(question))

#clean the answers

clean_answers = []

for answer in answers:
    clean_answers.append(clean_text(answer))

#creating a dictionary that maps each word to its number of occurences
wordCount = {}

for question in clean_questions:
    for word in question.split():
        if word not in wordCount:
            wordCount[word] = 1
        else:
            wordCount[word] += 1

for answer in clean_answers:
    for word in answer.split():
        if word not in wordCount:
            wordCount[word] = 1
        else:
            wordCount[word] += 1

#Creating two dictionary that map the questions words and the answers words to a unique integer

threshold = 20
questionswords2int = {}
word_number = 0
for word, count in wordCount.items():
    if count >= threshold:
        questionswords2int[word] = word_number
        word_number += 1

answerswords2int = {}
word_number = 0
for word, count in wordCount.items():
    if count >= threshold:
        answerswords2int[word] = word_number
        word_number += 1

#Adding the last tokens to these two dictionaries

tokens = ['<PAD>', '<EOS>','<OUT>','<SOS>']
for token in tokens:
    questionswords2int[token] = len(questionswords2int) + 1

for token in tokens:
    answerswords2int[token] = len(answerswords2int) + 1

#Creating the inverse dictionary of the answersword2int dictionary
answersint2word = {w_i: w for w, w_i in answerswords2int.items()}

#Adding EOS string to every answers

for i in range (len (clean_answers)):
    clean_answers[i] += ' <EOS>'

#Translating all the questions and the answers into integers
#and replacing all the words that were filtered by <out>

questionstoint = []
for question in clean_questions:
    ints = []
    for word in question.split():
        if word not in questionswords2int:
            ints.append(questionswords2int['<OUT>'])
        else:
            ints.append(questionswords2int[word ])
    questionstoint.append(ints)

answerstoint = []
for answer in clean_answers:
    ints = []
    for word in answer.split():
        if word not in answerswords2int:
            ints.append(answerswords2int['<OUT>'])
        else:
            ints.append(answerswords2int[word ])
    answerstoint.append(ints)

#Sorting the questions and answers by the length of questions

sorted_clean_questions = []
sorted_clean_answers = []

for length in range(1,25 + 1):
    for i in enumerate(questionstoint):
        if len(i[1]) == length:
            sorted_clean_questions.append(questionstoint[i[0]])
            sorted_clean_answers.append(answerstoint[i[0]])

################################PART 2 - Building the Seq2Seq Model ###########################

#Creating placeholders for the inputs and the targets

def model_inputs():
    inputs = tf.placeholder(tf.int32,[None, None], name = 'input')
    targets = tf.placeholder(tf.int32,[None, None], name = 'targets')
    lr = tf.placeholder(tf.float32,name = 'learning_rate')
    keep_prob = tf.placeholder(tf.float32,name = 'keep_prob') # controls dropout rate.
    return inputs, targets, lr, keep_prob

# Preprocessing the targets

def preprocess_targets(targets, word2int,batch_size):
    left_side = tf.fill([batch_size,1],word2int['<SOS>'])
    rigth_side = tf.strided_slice(targets,[0,0],[batch_size, -1],[1,1])
    preprocessed_targets = tf.concat([left_side,rigth_side], 1)
    return preprocessed_targets

#Encoding layer of Seq2Seq model.
#step 1: Creating the Encoder RNN Layer.

def encoder_rnn_layer(rnn_inputs, rnn_size, num_layers, keep_prob, sequence_length):
    lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)
    lstm_dropout = tf.contrib.rnn.DropoutWrapper(lstm,input_keep_prob = keep_prob)
    encoder_cell = tf.contrib.rnn.MultiRNNCell([lstm_dropout] * num_layers)
    encoder_output, encoder_state = tf.nn.bidirectional_dynamic_rnn(cell_fw = encoder_cell,
                                                       cell_bw = encoder_cell,
                                                       sequence_length = sequence_length,
                                                       inputs = rnn_inputs,
                                                       dtype = tf.float32)
    return encoder_state

#Decoding the training set

def decode_training_set(encoder_state, decoder_cell, decoder_embedded_input,sequence_length,decoding_scope, output_function,keep_prob, batch_size):
    attention_states = tf.zeros([batch_size, 1, decoder_cell.output_size])
    attention_keys, attention_values, attaention_score_function, attention_construct_function = tf.contrib.seq2seq.prepare_attention(attention_state, attention_option = 'bahdanu', num_units = decoder_cell.output_size)
    training_decoder_function = tf.contrib.seq2seq.attention_decode_fn_train(encoder_state[0],
                                                                            attention_keys,
                                                                            attention_values,
                                                                            attaention_score_function,
                                                                            attention_construct_function,
                                                                            name = "attn_dec_train")
    decoder_output, decoder_final_state, decoder_final_state =  tf.contrib.seq2seq.dynamic_rnn_decoder(decoder_cell,
                                                                                training_decoder_function,
                                                                                decoder_embedded_input,
                                                                                sequence_length,
                                                                                scope = decoding_scope
                                                                                )
    decoder_output_dropout = tf.nn.dropout(decoder_output, keep_prob)
    return output_function(decoder_output_dropout)

# Decoding the test/validation set
def decode_test_set(encoder_state, decoder_cell, decoder_embeddings_matrix, sos_id, eos_id, maximum_length, num_words, sequence_length, decoding_scope, output_function, keep_prob, batch_size):
    attention_states = tf.zeros([batch_size, 1, decoder_cell.output_size])
    attention_keys, attention_values, attaention_score_function, attention_construct_function = tf.contrib.seq2seq.prepare_attention(attention_state, attention_option = 'bahdanu', num_units = decoder_cell.output_size)
    test_decoder_function = tf.contrib.seq2seq.attention_decode_fn_inference(output_function,
                                                                             encoder_state[0],
                                                                             attention_keys,
                                                                             attention_values,
                                                                             attaention_score_function,
                                                                             attention_construct_function,
                                                                             decoder_embeddings_matrix,
                                                                             sos_id,
                                                                             eos_id,
                                                                             maximum_length,
                                                                             num_words,
                                                                             name = "attn_dec_inf")
    test_predictions, decoder_final_state, decoder_final_state =  tf.contrib.seq2seq.dynamic_rnn_decoder(decoder_cell,
                                                                                test_decoder_function,
                                                                                scope = decoding_scope
                                                                                )
    return test_predictions


"""
 This code originally from ***Wav2Lip*** repository

 Link repo: https://github.com/Rudrabha/Wav2Lip

 This code has been modified to preprocess Facial Landmark dataset instead of face images

"""

import sys

if sys.version_info[0] < 3 and sys.version_info[1] < 2:
    raise Exception("Must be using >= Python 3.2")

from os import listdir, path
import multiprocessing as mp
from concurrent.futures import ThreadPoolExecutor, as_completed
import numpy as np
import argparse, os, cv2, traceback, subprocess
from tqdm import tqdm
from glob import glob
import utils.audio as audio
from hparams import hparams as hp
import face_alignment

import torch
import matplotlib.pyplot as plt
from utils.plot import vis_landmark_on_img

print('Running Preprocess FL ')
parser = argparse.ArgumentParser()

parser.add_argument('--ngpu', help='Number of GPUs across which to run in parallel', default=1, type=int)
parser.add_argument('--batch_size', help='Single GPU Face detection batch size', default=16, type=int)
parser.add_argument("--data_root", help="Root folder of the LRS2 dataset" , default="/media/peter/peterwish/dataset/lrs2_v1/mvlrs_v1/main/")
parser.add_argument("--preprocessed_root", help="Root folder of the preprocessed dataset", default="../lrs2_test/")

args = parser.parse_args()

fa = [face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, flip_input=False,
                                    device='cuda:{}'.format(id)) for id in range(args.ngpu)]

fa_landmark = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, device='cuda', flip_input=True)
template = 'ffmpeg -loglevel panic -y -i {} -strict -2 {}'
# template2 = 'ffmpeg -hide_banner -loglevel panic -threads 1 -y -i {} -async 1 -ac 1 -vn -acodec pcm_s16le -ar 16000 {}'

def detect_bug_fl_batch(fls):

    for i in range(len(fls)):

        if len(fls[i]) > 68:

            bug = fls[i]

            fl = bug[:68]

            fls[i] = fl

    return fls

def process_video_file(vfile, args, gpu_id):
    video_stream = cv2.VideoCapture(vfile)

    frames = []
    while 1:
        still_reading, frame = video_stream.read()
        if not still_reading:
            video_stream.release()
            break
        #print(np.array(frames).shape)
        frame = cv2.resize(frame, (256,256))
        frames.append(frame)

    vidname = os.path.basename(vfile).split('.')[0]
    dirname = vfile.split('/')[-2]

    fulldir = path.join(args.preprocessed_root, dirname, vidname)
    os.makedirs(fulldir, exist_ok=True)

    batches = [frames[i:i + args.batch_size] for i in range(0, len(frames), args.batch_size)]
    i = -1

    for batch in batches:

        batch = np.array(batch)

        batch = np.transpose(batch, (0,3,1,2))

        batch = torch.from_numpy(batch)

        preds = fa[gpu_id].get_landmarks_from_batch(batch)

        preds = detect_bug_fl_batch(preds)

        for j, fl in enumerate(preds):

            i +=1

            if len(fl) == 0:

                continue

            np.savetxt(path.join(fulldir,'{}.txt'.format(i)),fl ,fmt='%.4f')

def process_audio_file(vfile, args):
    vidname = os.path.basename(vfile).split('.')[0]
    dirname = vfile.split('/')[-2]

    fulldir = path.join(args.preprocessed_root, dirname, vidname)
    os.makedirs(fulldir, exist_ok=True)

    wavpath = path.join(fulldir, 'audio.wav')

    command = template.format(vfile, wavpath)
    subprocess.call(command, shell=True)

def mp_handler(job):
    vfile, args, gpu_id = job
    try:
        process_video_file(vfile, args, gpu_id)
    except KeyboardInterrupt:
        exit(0)
    except:
        traceback.print_exc()

def main(args):
    print('Started processing for {} with {} GPUs'.format(args.data_root, args.ngpu))

    print("Saving Path at {}".format(args.preprocessed_root))

    filelist = glob(path.join(args.data_root, '*/*.mp4'))

    jobs = [(vfile, args, i%args.ngpu) for i, vfile in enumerate(filelist)]

    p = ThreadPoolExecutor(args.ngpu)

    futures = [p.submit(mp_handler, j) for j in jobs]

    _ = [r.result() for r in tqdm(as_completed(futures), total=len(futures))]

    print('Dumping audios...')

    for vfile in tqdm(filelist):
        try:
            process_audio_file(vfile, args)
        except KeyboardInterrupt:
            exit(0)
        except:
            traceback.print_exc()
            continue

if __name__ == '__main__':
    main(args)


from cmd import Cmd
import Pysys_str as s
import PSDK as p
import json

# just test start
s.user = 'defult'
builtin_apps = ['settings', 'info', 'dev', 'Pyox', 'Pyrandcard', 'Pyrandom']
thirdparty_apps = []
# just test end

class Run(Cmd):
    """运行"""

    intro = s.wel
    if s.user == 'defult':
        prompt = s.defult_user_opr
    elif s.user == 'xuser':
        prompt = s.xuser_opr
    else:
        prompt = s.guest_opr

    # q退出
    def do_q(self, none):
        """exit or quit"""
        return True

    # 设置settings：设置系统
    def do_settings(self, none):
        """system settings"""
        p.wel('settings')
        while True:
            settings_do = 'What do you want to do(type "q" to quit)? '
            new_psw = 'Type your new password: '
            print('Type "cpsw" to change your password.')
            print('Type "info" to see the info.')
            s.mess = input(settings_do)
            if s.mess == 'cpsw':
                cpsw = p.check_psw()
                if cpsw:
                    s.mess = input(new_psw)
                    with open(s.ui) as f:
                        s.user_info = json.load(f)
                        s.user_info['password'] = s.mess
                    with open(s.ui, 'w') as f:
                        json.dump(s.user_info, f)
                    print('Change succefully')
            elif s.mess == 'info':
                p.info(name='settings', info='System settings')
            elif s.mess == 'q':
                p.ext('settings')
                break
            else:
                print('We do not have this option.')

    # 应用列表app list：查看所有拥有的应用
    def do_app(self, opt):
        """list apps"""
        if opt:
            if opt == 'list':
                print('These are built in apps:')
            for app in builtin_apps:
                print(app)
            print('These are third party apps:')
            for app in thirdparty_apps:
                print(app)
        else:
            print('These are built in apps:')
            for app in builtin_apps:
                print(app)
            print('These are third party apps:')
            for app in thirdparty_apps:
                print(app)

    # 系统信息info：查看系统信息
    def do_info(self, none):
        """see info of the system"""
        p.wel('info')
        print('These are the info of Pysys:')
        p.info(name='Pysys', info='A breakthrough system based on Python', version='v0')
        p.ext('info')

    # 开发者设置dev：在开发者的角度设置系统
    def do_dev(self, none):
        p.wel('dev')
        p.ext('dev')

    # 游戏Pyox：OX棋
    def do_Pyox(self, none):
        """DO NOT PLAY IT"""
        s.mess = input(s.fob)
        if s.mess == 'y':
            import Pyox
            pyox = Pyox.Pyox()
            pyox.run()

    # 抽牌Pyrandcard：抽牌器
    def do_Pyrandcard(self, none):
        """random cards"""
        import Pyrandcard
        pyrandcard = Pyrandcard.RandomCard()
        pyrandcard.run()

    # 随机抽取Pyrandom：抽奖
    def do_Pyrandom(self, none):
        """random"""
        import Pyrandom

    def emptyline(self):
        print("empty line!")

test = Run()
test.cmdloop()


"""Adaptive Categorical Discretization"""

import torch
from torch.distributions import Distribution, constraints
import torch.nn.functional as F
from functools import reduce

def to_numpy(*args):
    return [arg.numpy() for arg in args]

def product(shape):
    if len(shape) == 0: return 1
    return reduce(lambda x, y: x * y, shape)

def gather_l(x, v):
    dim = len(x.shape)
    b_shape, f_shape = v.shape[:-dim+1], v.shape[-dim+1:]
    b, f = product(b_shape), product(f_shape)
    assert (x.shape[:-1] == f_shape)

    v = v.view(b, f).t()
    x = x.view(f, -1)

    return x.gather(-1, v).t().reshape(*b_shape, *f_shape)

def searchsorted_l(x, v):
    dim = len(x.shape)
    b_shape, f_shape = v.shape[:-dim+1], v.shape[-dim+1:]
    b, f = product(b_shape), product(f_shape)
    assert x.shape[:-1] == f_shape

    v = v.view(b, f).t()
    x = x.view(f, -1)
    return torch.searchsorted(x, v).t().reshape(*b_shape, *f_shape)

class Adacat(Distribution):
    arg_constraints = {'logits': constraints.real}
    support = constraints.unit_interval

    @property
    def mean(self):
        return ((self.x_cum - self.x_sizes / 2.) * self.y_sizes).sum(dim=-1)

    @property
    def x_sizes(self):
        return F.softmax(self.x_logits, dim=-1)

    @property
    def x_cum(self):
        return torch.cumsum(self.x_sizes, dim=-1)

    @property
    def y_sizes(self):
        return F.softmax(self.y_logits, dim=-1)

    @property
    def y_cum(self):
        return torch.cumsum(self.y_sizes, dim=-1)

    @property
    def log_x_sizes(self):
        return F.log_softmax(self.x_logits, dim=-1)

    @property
    def log_y_sizes(self):
        return F.log_softmax(self.y_logits, dim=-1)

    def __init__(self, logits, validate_args=None):
        self.logits = logits
        assert logits.size(-1) % 2 == 0
        self.n_knobs = logits.size(-1) // 2
        self.x_logits, self.y_logits = logits.split(self.n_knobs, dim=-1)

        batch_shape = self.x_logits.size()[:-1]
        super(Adacat, self).__init__(batch_shape, validate_args=validate_args)

    def expand(self, batch_shape, _instance=None):
        new = self._get_checked_instance(Adacat, _instance)
        batch_shape = torch.Size(batch_shape)
        new.x_logits = self.x_logits.expand(batch_shape + (self.n_knobs,))
        new.y_logits = self.y_logits.expand(batch_shape + (self.n_knobs,))
        super(Adacat, new).__init__(batch_shape, validate_args=False)
        new._validate_args = self._validate_args
        return new

    def sample(self, sample_shape=torch.Size()):
        if not isinstance(sample_shape, torch.Size):
            sample_shape = torch.Size(sample_shape)

        indices = torch.multinomial(self.y_sizes.reshape(-1, self.n_knobs), sample_shape.numel(), True).t()

        indices = indices.view(sample_shape.numel(), *self.y_sizes.shape[:-1])

        x_right = gather_l(self.x_cum, indices)
        x_size = gather_l(self.x_sizes, indices)
        x = x_right - x_size * torch.rand_like(x_size)

        ret = x.reshape(self._extended_shape(sample_shape))

        return ret

    def _validate_distribution(self, d):
        assert self.support == d.support

    def log_prob(self, value):
        if isinstance(value, Distribution):
            # if passed in a distribution of value, we compute the expectation of log_prob
            # the distribution should have the same batch shape as the current distribution
            x_cum = F.pad(self.x_cum, (1, 0))
            shape = x_cum.size()[:-1]
            numel = product(shape)
            x_cum = x_cum.view(-1, self.n_knobs+1).t().view(self.n_knobs+1, *shape)
            cdfs = value.cdf(x_cum.clamp(min=0., max=1.))
            ws = (cdfs[1:] - cdfs[:-1]).view(self.n_knobs, numel).t().view(*shape, self.n_knobs)
            return (ws * (self.log_y_sizes - self.log_x_sizes)).sum(dim=-1)

        else:
            # compute this normally
            if self._validate_args:
                self._validate_sample(value)

            indices = searchsorted_l(self.x_cum, value).clamp(
                0, self.n_knobs-1)

            log_x_size = gather_l(self.log_x_sizes, indices)
            log_y_size = gather_l(self.log_y_sizes, indices)

            return log_y_size - log_x_size

    def cdf(self, value):
        if self._validate_args:
            self._validate_sample(value)

        indices = searchsorted_l(self.x_cum, value).clamp(
            0, self.n_knobs-1)

        x_cum = gather_l(self.x_cum, indices)
        y_cum = gather_l(self.y_cum, indices)
        x_size = gather_l(self.x_sizes, indices)
        y_size = gather_l(self.y_sizes, indices)

        return y_cum - (x_cum - value) / x_size * y_size

    def icdf(self, value):
        if self._validate_args:
            self._validate_sample(value)

        indices = searchsorted_l(self.y_cum, value).clamp(
            0, self.n_knobs-1)

        x_cum = gather_l(self.x_cum, indices)
        y_cum = gather_l(self.y_cum, indices)
        x_size = gather_l(self.x_sizes, indices)
        y_size = gather_l(self.y_sizes, indices)

        return x_cum - (y_cum - value) / y_size * x_size

    def entropy(self):
        return ((self.log_y_sizes - self.log_x_sizes) * self.x_sizes).sum(dim=-1)


import urllib.request
import sys
import os
import pandas as pd

duration = int(sys.argv[1])

cur = os.path.dirname(os.path.abspath(__file__))

df = pd.read_csv('./audio_library_non_license.csv', header=0)
df_s = df.sample(frac=1)

def calc_second(text):
    texts = text.split(':')
    return int(texts[0]) * 60 + int(texts[1])

# 音楽ファイルによっては、音楽ファイルの長さ - 5秒くらいで曲が終わるものがあり、
# 終盤無音状態でアニメーションが再生されてしまうことがよくあるので、
# 動画ファイルよりも10秒くらい長い音楽ファイルを取得する
# 音楽ファイルは終盤5秒でフェードアウトさせる
duration += 10
music_max_length = 984

while duration <= music_max_length:
    for index, row in df_s.iterrows():
        if duration == calc_second(row[2]):
            req = urllib.request.Request(row[6])

            xml_resposnse = urllib.request.urlopen(req).read()

            os.makedirs("{}/music/{}".format(cur, duration), exist_ok=True)
            path = "{}/music/{}/{}.mp3".format(cur, duration, row[0].replace(' ', '_'))

            with open(path, mode='wb') as f:
                f.write(xml_resposnse)

            print(path)
            break
    else:
        duration += 1
        continue

    break


import math

import matplotlib
import numpy as np
from matplotlib import pyplot as plt

def create_data(f, x_vals):
    y_vals = []
    for i in x_vals:
        y_vals.append(f(i))
    return np.array(y_vals)

def fit_exp_data(x_vals, y_vals):
    log_vals = []
    for y in y_vals:
        log_vals.append(math.log(y, 2))
    fit = np.polyfit(x_vals, log_vals, 1)
    return fit, 2

if __name__ == '__main__':
    x_vals = range(10)
    f = lambda x: 3 ** x
    y_vals = create_data(f, x_vals)

    matplotlib.use('TkAgg')
    plt.plot(x_vals, y_vals, 'ko', label='Actual values')

    fit, base = fit_exp_data(x_vals, y_vals)
    predicted_y_vals = []
    for x in x_vals:
        predicted_y_vals.append(base ** np.polyval(fit, x))
    plt.plot(x_vals, predicted_y_vals, label='Predicted values')

    plt.title('Fitting an Exponential Function')
    plt.legend(loc='upper left')
    plt.savefig('fit-exp')
    plt.show()

    print(f'f(20) = {f(20)}')
    print(f'Predicted value = {int(base ** (np.polyval(fit, [20])))}')

1 Intro CS/02.Intro to CS/20.Understanding Experimental Data/20.2.The Behavior of Projectiles/20.3.Fitting Exponentially Distributed Data/fit_exponential.py


def gram_matrix(A):
    """
    Argument:
    A -- matrix of shape (n_C, n_H*n_W)

    Returns:
    GA -- Gram matrix of A, of shape (n_C, n_C)
    """

    ### START CODE HERE ### (≈1 line)
    GA = None
    ### END CODE HERE ###

    return GA

tf.reset_default_graph()

with tf.Session() as test:
    tf.set_random_seed(1)
    a_C = tf.random_normal([1, 4, 4, 3], mean=1, stddev=4)
    a_G = tf.random_normal([1, 4, 4, 3], mean=1, stddev=4)
    J_content = compute_content_cost(a_C, a_G)
    print("J_content = " + str(J_content.eval()))


import random

simple_grammar = """
sentence => noun_phrase verb_phrase
noun_phrase => Article Adj* noun
Adj* => Adj | Adj Adj*
verb_phrase => verb noun_phrase
Article =>  一个 | 这个
noun =>   女人 |  篮球 | 桌子 | 小猫
verb => 看着   |  坐在 |  听着 | 看见
Adj =>   蓝色的 |  好看的 | 小小的"""
def get_generation_by_gram(grammar_str: str, target, stmt_split='=', or_split='|'):
    rules = dict()  # key is the @statement, value is @expression
    for line in grammar_str.split('\n'):
        if not line: continue
        # skip the empty line
        #  print(line)
        stmt, expr = line.split(stmt_split)

        rules[stmt.strip()] = expr.split(or_split)

    generated = generate(rules, target=target)

    return generated
def generate(grammar_rule, target):
    if target in grammar_rule: # names
        candidates = grammar_rule[target]  # ['name names', 'name']
        candidate = random.choice(candidates) #'name names', 'name'
        return ''.join(generate(grammar_rule, target=c.strip()) for c in candidate.split())
    else:
        return target

print(get_generation_by_gram(simple_grammar, target='sentence', stmt_split='=>'))


# 신호를 보낼 수 있는 객체를 생성하기. (QObject를 상속받은 pyqtSignal)

# 1. 마우스클릭
# 2. mousePressEvent가 발생.
# 3. closeApp 객체에 신호가 방출되었으면 한다.Emit()
# 4. closeApp 과 연결된(.connect) self.close를 실행 시킨다.

import sys
from PyQt5.QtCore import pyqtSignal, QObject
from PyQt5.QtWidgets import QApplication, QMainWindow

class Communicate(QObject):
    closeApp = pyqtSignal()     # pyqtSignal()을 쓰기 위해서 위에서 QObject를 상속 받음.
                                # pyqtSignal() : 신호를 보낼 수 있는 객체를 생성한다.
                                # 이제 "closeApp"은 신호를 보낼 수 있는 객체가 되었다.
class Example(QMainWindow):
    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):
        self.c = Communicate()                  # c 객체는 위에 메서드의 기능을 갖음.
        self.c.closeApp.connect(self.close)     # c 의 closeApp 에 "창을 닫는 이벤트(self.close)"를 연결.

        self.setGeometry(300, 300, 290, 150)    # 초기 창 위치좌표와 크기 설정.
        self.setWindowTitle('Emit signal')      # 창이름
        self.show()

    # 마우스를 눌렀을 때 이벤트.
    def mousePressEvent(self, event):
        self.c.closeApp.emit()                  # 이벤트가 발생하면(emit), closeApp과 연결된 slot에 신호를 줌.
                                                # 결론: closeApp과 연결된 메서드(함수)를 호출하라.
if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = Example()
    sys.exit(app.exec_())

6_이벤트, 시그널/5_마우스가 클릭되었을 때 이벤트 발생, emit 함수를 이용해서 다양한 이벤트와 연결이 가능.py


from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
from torch.autograd import Variable
import torch.optim as optim
import torch.nn.functional as F
import torch.nn as nn
import torch
import numpy as np
import conllu

# Parse the data using the CoNLL-U format parser
train_data = conllu.parse(
    open("./Dataset/en_atis-ud-train.conllu", "r", encoding="utf-8").read())
valid_data = conllu.parse(
    open("./Dataset/en_atis-ud-dev.conllu", "r", encoding="utf-8").read())
test_data = conllu.parse(
    open("./Dataset/en_atis-ud-test.conllu", "r", encoding="utf-8").read())

train_data = [[(word['form'], word['upostag'])
               for word in sentence] for sentence in train_data]
valid_data = [[(word['form'], word['upostag'])
               for word in sentence] for sentence in valid_data]
test_data = [[(word['form'], word['upostag'])
              for word in sentence] for sentence in test_data]

# Glove Embedding

def load_glove_embeddings(file_path):
    with open(file_path, 'r') as f:
        embeddings = {}
        for line in f:
            values = line.split()
            word = values[0]
            vector = np.array(values[1:], dtype='float32')
            embeddings[word] = vector
    return embeddings

# embeddings = load_glove_embeddings('./glove.6B/glove.6B.100d.txt')
# embeddings_list = [embeddings[word] for word in embeddings]
# embeddings_matrix = np.array(embeddings_list)
# add for PAD
# embeddings_matrix = np.vstack(
    # (np.zeros(100).astype(np.float32), embeddings_matrix))
# add for UNK
# embeddins_matrix = np.vstack(
    # (embeddings_matrix, np.random.rand(100).astype(np.float32)))

# np.save('embeddings_matrix.npy', embeddings_matrix)

# Create a dictionary of words and their corresponding indices
def dictWCT(data):
    wordCount = {}
    tagCount = {}
    wordFreq = {}
    for sentence in data:
        for word in sentence:
            if word[0] not in wordCount:
                wordCount[word[0]] = len(wordCount)+1
                wordFreq[word[0]] = 1
            else:
                wordFreq[word[0]] += 1
            if word[1] not in tagCount:
                tagCount[word[1]] = len(tagCount)+1
    return wordCount, tagCount, wordFreq

wordToIdx, tagToIdx, wordFreq = dictWCT(train_data)

wordToIdx['<PAD>'] = 0
wordFreq = {k: v for k, v in wordFreq.items() if v > 2}
wordToIdx['<UNK>'] = len(wordFreq)+1
tagToIdx['<UNK>'] = len(tagToIdx)+1
numUniqueWords = len(wordFreq)
numUniqueTags = len(tagToIdx)+1
idxToWord = {v: k for k, v in wordToIdx.items()}
idxToTag = {v: k for k, v in tagToIdx.items()}

# print("Number of unique words: ", numUniqueWords)
# print("Number of unique tags: ", numUniqueTags)

def getIndices(data, wordToIdx, tagToIdx, wordFreq, min_freq=2):
    wordIndices = []
    tagIndices = []
    mxTagForUnk = {}
    for sentence in data:
        wordIdx = []
        tagIdx = []
        for word in sentence:
            if word[0] not in wordFreq:
                wordIdx.append(wordToIdx['<UNK>'])
                tagIdx.append(tagToIdx['<UNK>'])
                if word[1] not in mxTagForUnk:
                    mxTagForUnk[word[1]] = 1
                else:
                    mxTagForUnk[word[1]] += 1
            else:
                wordIdx.append(wordToIdx[word[0]])
                tagIdx.append(tagToIdx[word[1]])
        wordIndices.append(wordIdx)
        tagIndices.append(tagIdx)

    mxTagForUnk = max(mxTagForUnk, key=mxTagForUnk.get)
    # tagIndices = [[tagToIdx[mxTagForUnk] if word == wordToIdx['<UNK>'] else tag for word, tag in zip(
    #     wordIndices[i], tagIndices[i])] for i in range(len(wordIndices))]
    return wordIndices, tagIndices

wordIndices, tagIndices = getIndices(train_data, wordToIdx, tagToIdx, wordFreq)
wordIndicesValid, tagIndicesValid = getIndices(
    valid_data, wordToIdx, tagToIdx, wordFreq)
wordIndicesTest, tagIndicesTest = getIndices(
    test_data, wordToIdx, tagToIdx, wordFreq)

def padWCT(wordIndices, tagIndices):
    max_len = 0
    for i in range(len(wordIndices)):
        if len(wordIndices[i]) > max_len:
            max_len = len(wordIndices[i])

    wordIndices = np.array([np.pad(wordIndices[i], (0, max_len -
                           len(wordIndices[i])), 'constant') for i in range(len(wordIndices))])
    tagIndices = np.array([np.pad(tagIndices[i], (0, max_len -
                          len(tagIndices[i])), 'constant') for i in range(len(tagIndices))])

    return wordIndices, tagIndices

wordIndices, tagIndices = padWCT(wordIndices, tagIndices)
wordIndicesValid, tagIndicesValid = padWCT(wordIndicesValid, tagIndicesValid)
wordIndicesTest, tagIndicesTest = padWCT(wordIndicesTest, tagIndicesTest)

# GLOBAL VARIABLES
WORD_EMBEDDING_DIM = 100
LSTM_HIDDEN_DIM_WORD = 256
DROPOUT_RATE = 0.5
BATCH_SIZE = 128
NUM_EPOCHS = 10

use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
# print(device)

# Create the model

class BiLSTM(nn.Module):
    def __init__(self, numUniqueWords, numUniqueTags, wordEmbeddingDim, lstmHiddenDimWord, dropoutRate, embedding):
        super(BiLSTM, self).__init__()
        self.wordEmbedding = nn.Embedding.from_pretrained(
            torch.from_numpy(embedding).float(), freeze=False, padding_idx=0)
        self.lstmWord = nn.LSTM(wordEmbeddingDim,
                                lstmHiddenDimWord, bidirectional=True, batch_first=True)
        self.dropout = nn.Dropout(dropoutRate)
        self.linear = nn.Linear(2*lstmHiddenDimWord, numUniqueTags)

    def forward(self, word_indices):
        word_embeddings = self.wordEmbedding(word_indices)
        lstm_word_out = self.lstmWord(word_embeddings)[0]
        lstm_word_out = self.dropout(lstm_word_out)
        linear_out = self.linear(lstm_word_out)
        tag_scores = F.log_softmax(linear_out, dim=2)

        return tag_scores

embeddings_matrix = np.load('embeddings_matrix.npy')
model = BiLSTM(numUniqueWords, numUniqueTags, WORD_EMBEDDING_DIM,
               LSTM_HIDDEN_DIM_WORD, DROPOUT_RATE, embeddings_matrix)
if torch.cuda.is_available():
    model.cuda()

# create tensors
def convertToTensors(wordIndices, tagIndices):
    wordTensor = [torch.LongTensor(word) for word in wordIndices]
    tagTensor = [torch.LongTensor(tag) for tag in tagIndices]
    return wordTensor, tagTensor

wordTensor, tagTensor = convertToTensors(wordIndices, tagIndices)
wordTensorValid, tagTensorValid = convertToTensors(
    wordIndicesValid, tagIndicesValid)
wordTensorTest, tagTensorTest = convertToTensors(
    wordIndicesTest, tagIndicesTest)

# training model
optimizer = optim.Adam(model.parameters(), lr=0.01)
criterion = nn.NLLLoss(ignore_index=0)
torch.autograd.set_detect_anomaly(True)

def train(model, optimizer, criterion, wordTensor, tagTensor):
    model.train()
    total_loss = 0
    tot_correct = 0
    total_pad = 0
    for i in range(0, len(wordIndices), BATCH_SIZE):
        optimizer.zero_grad()
        word_tensor = wordTensor[i:i+BATCH_SIZE]
        tag_tensor = tagTensor[i:i+BATCH_SIZE]
        word_tensor = torch.stack(word_tensor).to(device)
        tag_tensor = torch.stack(tag_tensor).to(device)
        tag_tensor = tag_tensor.view(-1)
        output = model(word_tensor)
        output = output.view(-1, output.shape[2])
        loss = criterion(output, tag_tensor)
        loss.backward()
        optimizer.step()
        total_loss += loss.item()
        pred = torch.argmax(output, dim=1)
        tot_correct += torch.sum((tag_tensor != 0) &
                                 (tag_tensor == pred)).item()

        total_pad += torch.sum(tag_tensor == 0).item()

    return total_loss, tot_correct/(len(wordIndices)*len(wordIndices[0]) - total_pad)

def evaluate(model, wordTensor, tagTensor):
    model.eval()
    total_loss = 0
    tot_correct = 0
    total_pad = 0
    wordTagList = []
    with torch.no_grad():
        for i in range(0, len(wordTensor), BATCH_SIZE):
            word_tensor = wordTensor[i:i+BATCH_SIZE]
            tag_tensor = tagTensor[i:i+BATCH_SIZE]
            word_tensor = torch.stack(word_tensor).to(device)
            tag_tensor = torch.stack(tag_tensor).to(device)
            tag_tensor = tag_tensor.view(-1)
            output = model(word_tensor)
            output = output.view(-1, output.shape[2])
            loss = criterion(output, tag_tensor)
            total_loss += loss.item()
            pred = torch.argmax(output, dim=1)
            tot_correct += torch.sum((tag_tensor != 0) &
                                     (tag_tensor == pred)).item()
            total_pad += torch.sum(tag_tensor == 0).item()
            # take those only which are not padded
            wordTagList.extend(
                list(zip(word_tensor.view(-1).tolist(), pred.view(-1).tolist())))
    return total_loss, tot_correct/(len(wordTensor)*len(wordTensor[0]) - total_pad), wordTagList

def EpochRun(model, optimizer, criterion, wordTensor, tagTensor, wordTensorValid, tagTensorValid):
    loss_list = []
    accuracy_list = []
    validaton_loss_list = []
    validaton_accuracy_list = []
    for epoch in range(NUM_EPOCHS):
        loss, totalCorrect = train(
            model, optimizer, criterion, wordTensor, tagTensor)
        lossValid, totalCorrectValid, _ = evaluate(
            model, wordTensorValid, tagTensorValid)
        print("Epoch: {0} \t Training Loss: {1} \t Training Accuracy: {2} \t Validation Loss: {3} \t Validation Accuracy: {4}".format(
            epoch+1, loss, totalCorrect, lossValid, totalCorrectValid))
        loss_list.append(loss)
        accuracy_list.append(totalCorrect)
        validaton_loss_list.append(lossValid)
        validaton_accuracy_list.append(totalCorrectValid)
    return loss_list, accuracy_list, validaton_loss_list, validaton_accuracy_list

# loss_list, accuracy_list, validaton_loss_list, validaton_accuracy_list = EpochRun(
#     model, optimizer, criterion, wordTensor, tagTensor, wordTensorValid, tagTensorValid)

def genSentence(model, wordFreq, sentence):
    sentence = sentence.lower()
    print(sentence)
    sentence = sentence.split(" ")
    word_indice = []
    for s in sentence:
        if s not in wordFreq or wordFreq[s] <= 2:
            word_indice.append(wordToIdx['<UNK>'])
        else:
            word_indice.append(wordToIdx[s])
    wordTen = torch.LongTensor(word_indice).to(device)
    wordTen = wordTen.view(1, -1)
    # predict the tag
    with torch.no_grad():
        model.eval()
        output = model(wordTen)
        _, predicted = torch.max(output, 2)
        # print(predicted.shape)
        for i in range(len(predicted[0])):
            print(sentence[i],"\t", idxToTag[predicted[0][i].item()])

# input sentence
import sys

input_sentence = input("Enter a sentence: ")

# load the model

model.load_state_dict(torch.load('modelFinal.pt'))
genSentence(model, wordFreq, input_sentence)