# -*- coding: utf-8 -*-
"""
Created on Sat Mar  6 11:18:14 2021

@author: Mikko
"""

from tkinter import *
from tkinter import Frame, Canvas, Label, Button, Message, LEFT, RIGHT, BOTTOM, ALL, Tk, Entry, BOTH, S, HORIZONTAL
from tkinter import filedialog
from random import randint
import tkinter as tk
from tkinter import ttk

import time

from PIL import Image, ImageTk
import PIL

from sklearn.cluster import KMeans

import numpy as np
import os

def to_array(photo):
    I = np.asarray(photo)
    return(I)

def to_image(arr, path):
    im = PIL.Image.fromarray(np.uint8(arr))
    #render = ImageTk.PhotoImage(im)
    im.save((path))

def row_to_image(row, check=True):
    l = len(row)/3
    l = int(l)
    if check:
        if not l==np.floor(l):
            raise ValueError("Column number has to be divisible by 3")
        if not np.sqrt(l)==np.floor(np.sqrt(l)):
            raise ValueError("Column number divided by 3 has to have a whole number squareroot, not {}".format(np.sqrt(l)))

    sl = np.sqrt(l).astype(int)

    r = row[:l].reshape((sl,sl))
    g = row[l:(2*l)].reshape((sl,sl))
    b = row[2*l:].reshape((sl,sl))

    pic = np.array([color for color in (r,g,b)])
    pic = np.moveaxis(pic, 0, -1)

    return(pic)

def distance(vec1,vec2):
    return(np.sqrt(np.sum((vec1-vec2)**2)))

def get_filename_dialog(root, initdir="/"):
    root.filename =  filedialog.askopenfilename(initialdir=initdir,title="Select file",filetypes=(("jpeg files","*.jpg"),("all files","*.*")))
    return(root.filename)

def get_foldername_dialog(root, initdir="/"):
    root.foldername =  filedialog.askdirectory(initialdir=initdir,title="Select file")
    return(root.foldername)

def get_files_dir():
    files_path = [os.path.abspath(x) for x in os.listdir()]
    return(files_path)

def get_files_dir_path(path):
    files_path = [os.path.abspath(x) for x in os.listdir()]
    return(files_path)

def kmeans_model(arr,axis=2,n_clust=20):

    nx,ny = arr.shape[:2]

    vectors = arr.reshape(nx*ny,3)
    pkmeans = KMeans(n_clusters=n_clust, random_state=0).fit(vectors)
    return(pkmeans)

def array_stats_(arr):
    stats = []

    meanvec = [np.mean(arr[:,:,i]) for i in range(3)]
    stats.append(meanvec)

    return stats

def array_stats(arr):
    stats = []
    vec = [np.bincount(arr[:,:,i].flatten()).argmax() for i in range(3)]
    stats.append(vec)

    return(stats)

def towards_mean(arr):
    meancolor = np.array([np.mean(arr[:,:,i]) for i in range(3)])

    arr2 = arr.copy()
    arr2.setflags(write=1)
    arr2 = arr2.astype(float)

    for i in range(3):
        arr2 += meancolor[i]
        arr2 /= 2

    return(arr2)

class Gui1:

    def __init__(self, root):
        self.root = root
        self.menubar = Menu(self.root)

        self.windowsize = root.size()
        self.fdx,self.fdy = self.windowsize

        "-- option menu widgets"

        self.menubar = Menu(self.root)
        self.filemenu = Menu(self.menubar, tearoff=0)
        self.menubar.add_cascade(label="File", menu=self.filemenu)

        self.filemenu.add_command(label ='New File', command = None)
        self.filemenu.add_command(label ='Open...', command = None)
        self.filemenu.add_command(label ='Save', command = None)
        self.filemenu.add_separator()
        self.filemenu.add_command(label ='Exit', command = root.destroy)

        self.root.config(menu = self.menubar)

        "-- gui variables "

        self.imagespath = StringVar()
        cf = os.getcwd()
        filename = cf+"\\"
        self.imagespath.set(filename)

        self.tileimage = StringVar()
        self.tileimage.set("")

        self.blend_original = IntVar()
        self.closer_to_clusters = IntVar()

        "-- first tab and sub widgets"

        self.tabs = ttk.Notebook(root, width=self.fdx, height=self.fdy)

        f1 = Frame(self.tabs)
        self.tabs.add(f1, text="Create Mosaic Image")

        f2 = Frame(f1)
        f2.pack()

        self.filename = Label(f2, textvariable=self.imagespath, width=100)
        self.filename.pack(side=RIGHT)

        bbtn1 = Button(f2, width=15, text="Add Folder", command=self.add_folder_to_scheme)
        bbtn1.pack(side=LEFT)

        f3 = Frame(f1)
        f3.pack()

        self.filename = Label(f3, textvariable=self.tileimage, width=100)
        self.filename.pack(side=RIGHT)

        bbtn2 = Button(f3, width=15, text="Tile picture", command=self.add_file_to_scheme)
        bbtn2.pack(side=LEFT)

        f4 = Frame(f1)
        f4.pack(anchor=W)

        self.nc = tk.Scale(f4, label="Number of Clusters", orient=tk.HORIZONTAL, length=200)
        self.nc.pack(anchor=W, padx=10)
        self.nc.set(25)

        self.cb1 = Checkbutton(f4, text="Blend with original", variable=self.blend_original, padx=20)
        self.cb1.pack(anchor=W)

        self.cb2 = Checkbutton(f4, text="Closer to cluster centers", variable=self.closer_to_clusters)
        self.cb2.pack()

        bbtn3 = Button(f4, width=15, text="Create mosaic", command=self.create_mosaic)
        bbtn3.pack(anchor=W)

        self.tabs.grid(row=0, column=0)

    def add_folder_to_scheme(self):
        file_name = get_foldername_dialog(self.root, initdir=self.imagespath)
        self.imagespath.set(file_name)

    def add_file_to_scheme(self, exts=["jpg","png","bmp","tif"]):
        file_name = get_filename_dialog(self.root, initdir=self.imagespath)
        if file_name.split(".")[-1] in exts:
            self.tileimage.set(file_name)

    def mosaic_folder(self):
        pass

    def create_mosaic(self):
        t = Tiles()
        print("initializing images")
        path = self.imagespath.get()
        #pathends = ["one","two","three","four","five"]

        t.get_image_set(path)

        path = self.tileimage.get()
        #path = "C://Users//Mikko//Documents//LiClipse Workspace//firebase//source2//data//xxx//meitsi.jpg"

        print("clustering images")
        t.tile_image(path)
        t.cluster_up(self.nc.get())

        print("mosaic creation bitches")

        t.create_mosaic_image(closer_to_clust=[False,True][self.closer_to_clusters.get()],
                              blend_original=[False,True][self.blend_original.get()],
                              blend_ratio=0.1)

        #path = os.path.abspath(__file__)
        path = os.getcwd()
        path = path+"//mosaic{}.jpg".format(int(time.time()))

        to_image(t.mosaic,path)
        print("done, check working directory")

class Tiles:

    def __init__(self, tilesize=(32,32)):

        self.arrays = []
        self.tilesize=tilesize

    def get_image_set(self, path):
        "folder iterating of self.get_pictures_from_folder"
        for e in os.scandir(path):
            if e.is_dir():
                self.get_pictures_from_folder(e.path,False)

    def get_pictures_from_array(self,array):
        "get mosaic image set from a np array color image set, for exmaple cifar-10"
        check = True
        for i in range(array.shape[0]):
            row = array[i,:]
            picarr = row_to_image(row, check=check)
            self.arrays.append(picarr)
            check = False

        n = len(self.arrays)
        self.n_arrays = n

        self.stats = np.zeros((n,6))
        self.stats[:,0] = [i for i in range(n)]
        self.set_means()

    def get_pictures_from_folder(self, path, clear_previous=True, exts=["jpg","jpeg","png","bmp","tif"]):
        "obtains mosaic images from a given folder"

        if clear_previous:
            del self.arrays
            self.arrays = []

        paths = [os.path.abspath(x.path) for x in os.scandir(path) if x.path.split(".")[-1] in exts]
        for path in paths:
            arr = Image.open(path).resize(self.tilesize)
            arr = to_array(arr)
            if arr.shape[2]==3:
                self.arrays.append(arr)

        n = len(self.arrays)
        self.n_arrays = n

        self.stats = np.zeros((n,6))
        self.stats[:,0] = [i for i in range(n)]
        self.set_means()

    def get_picture(self, get_filename=True):
        "appends a single picture to mosaic images from path"
        if get_filename:
            photo_name = get_filename_dialog(self.root)
        else:
            photo_name = os.getcwd()+"\\"
            photo_name += self.folder_picture_names[self.picture_index]

        self.pictures.append(Image.open(photo_name))

    def set_means(self):
        "mosaic image array set statistics... used in tiling"
        i = 0
        for arr in self.arrays:
            self.stats[i,1:4] = array_stats(arr)[0]
            i += 1

    def set_clusters(self):
        "mosaic images assigned to clusters of the model"
        i = 0
        for arr in self.arrays:
            pred = self.km.predict([self.stats[i,1:4]])[0]
            self.stats[i,-2] = pred
            self.stats[i,-1] = distance(self.km.cluster_centers_[pred,:],self.stats[i,1:4])
            i += 1

    def tile_image(self, path, resize=False, resize_ratio=0.50):
        "the image to be tiled is set"
        ti = Image.open(path)
        if resize:
            rx = int(resize_ratio*ti.size[0])
            ry = int(resize_ratio*ti.size[1])
            self.tile_image.resize((rx,ry))

        self.tile_image = ti

        self.tile_array = to_array(self.tile_image)
        self.shp = self.tile_array.shape

    def cluster_up(self, n_clusters=25):
        "tiles are assigned to self.cluster_array with clustering model predictions"

        self.km = kmeans_model(self.tile_array, n_clust=n_clusters)

        self.cluster_array = np.zeros((self.shp[0],self.shp[1]))

        for i in range(self.shp[0]):
            for j in range(self.shp[1]):
                self.cluster_array[i,j] = self.km.predict([self.tile_array[i,j,:]])[0]

        self.cluster_array = self.cluster_array.astype(int)
        self.set_clusters()

    def closer_to_clusters(self, i, ratio):
        "handicap #1 of the final product - tile array moved closed to cluster center"

        arr = self.arrays[i].copy()
        arr = arr.astype(float)
        center = self.km.cluster_centers_[int(self.stats[i,-2]),:]
        for z in range(3):
            arr[:,:,z]+=center[z]*ratio
            arr[:,:,z]/=2
        arr = arr.astype(int)
        return(arr)

    def create_mosaic_image(self, mode="distance", closer_to_clust=True, blend_original=True, blend_ratio=1.0):
        "mosaic image creation from tile images with cluster assignments"

        tilesize = self.tilesize
        self.mosaic = np.zeros((self.shp[0]*tilesize[0],self.shp[1]*tilesize[1],3))

        assignments = np.zeros(self.cluster_array.shape)
        ni,nj = self.cluster_array.shape

        br = max(0.0,blend_ratio)
        br = min(1.0,blend_ratio)

        for i in range(ni):
            for j in range(nj):
                assignments[i,j] = self.assign_tile(i,j,mode=mode)
        assignments = assignments.astype(int)

        ti, tj = self.tilesize

        for i in range(ni):
            for j in range(nj):
                im = int(ti*i)
                jm = int(tj*j)

                if closer_to_clust:
                    r = np.random.uniform(np.random.uniform(),1)
                    arr = self.closer_to_clusters(assignments[i,j],r)
                    #arr = towards_mean(arr)
                else:
                    arr  = self.arrays[assignments[i,j]]

                if blend_original:
                    "handicap #2: tile colors are moved closer to tile image pixels"
                    arr2 = arr.copy()
                    arr2.setflags(1)
                    arr2 = arr2.astype(float)

                    orig = [self.tile_array[i,j,c].astype(float) for c in range(3)]
                    for c in range(3):
                        arr2[:,:,c] += orig[c]*br
                        arr2[:,:,c] /= (1+br)

                    self.mosaic[im:(im+ti),jm:(jm+tj),:] = arr2.astype(int)
                else:
                    self.mosaic[im:(im+ti),jm:(jm+tj),:] = arr

    def assign_tile(self, i, j, mode):
        """mosaic tile is given an index representing a single tile from self.arrays"
        mode is deals with how tiles are picked from clusters"""

         #global rows
         #global r

        w = np.where(self.stats[:,4]==self.cluster_array[i,j])[0]
        rows = self.stats[w,:]

        if mode=="random":
            if not rows.shape[0]==0:
                r = np.random.randint(0,rows.shape[0])
                irow = rows[r,0]
            else:
                return(0)
            return(irow)

        if not rows.shape[0]==0:

            rows = rows[np.argsort(rows[:, -1])]
            r = np.random.triangular(0,0,rows.shape[0])
            r = np.round(r)
            r = int(r)
            r = min(r, rows.shape[0]-1)
            irow = rows[r,0]
        else:
            return(0)
        return(irow)

    def tile_stats(self):
        pass

def init_gui():
    w = Tk()
    lx,ly = 1000,500
    size = "{}x{}".format(lx,ly)
    w.geometry(size)
    w.resizable(1, 1)
    g = Gui1(w)
    w.mainloop()

#init_gui()


from dataset import MahjongGBDataset
from torch.utils.data import DataLoader
from model import CNNModel, Encoder
import torch.nn.functional as F
import torch
import os

splitRatio = 0.9
batchSize = 2000
lr = 5e-5
epoch = 15
warmup = 2
logdir = 'model/Data_51(214_235checkpoint'
file = "data/Data_51(214_235"
vocab_size = 214

if __name__ == '__main__':
    device = 'cuda'
    # Load dataset
    validateDataset = MahjongGBDataset(file, splitRatio, 1, False)
    vloader = DataLoader(dataset = validateDataset, batch_size = batchSize, shuffle = False)

    # Load model
    model = Encoder(vocab_size=vocab_size).to(device)
    model.load_state_dict(torch.load(f"{logdir}/7.pkl", map_location = torch.device('cuda')))

    print('Run validation:')
    correct = 0
    for i, d in enumerate(vloader):
        input_dict = {'is_training': False, 'obs': {'observation': d[0].cuda(), 'action_mask': d[1].cuda()}}
        with torch.no_grad():
            logits = model(input_dict)
            pred = logits.argmax(dim = 1)
            correct += torch.eq(pred, d[2].cuda()).sum().item()
    acc = correct / len(validateDataset)
    print('Validate acc:', acc)


#!/usr/bin/python
#-*-coding:utf-8-*-
import sys
from bson.json_util import dumps
sys.path.append('../')

from flask import Blueprint, render_template, session, request, make_response, redirect, url_for
from models import User, CreatedProjects
from definition import mongo
import os

samples = Blueprint('samples', __name__, template_folder='templates')

@samples.route('/samples', methods=['GET'])
@samples.route('/<language>/samples', methods=['GET'])
def samples_backend(language='en'):
    """
    Returns the sample page.
    :param language: Defines the language ('en' or 'cn') used for the template
    :return: The rendered samples.html template
              /samples?name=<project_name>
    """

    project_name = request.args.get('name', '')
    if not project_name or mongo.db.projects.find({"name": project_name}).count() <= 0 or \
        mongo.db.samples.find({"project_name": project_name}).count() <= 0:
        return redirect(url_for('projects.projects_backend', language=language))

    sample_list = mongo.db.samples.find({"project_name": project_name})
    if request.args.get("fields", ''):
        project_fields_name = request.args.getlist("fields")
    else:
        if mongo.db.fields.find({"project_name": project_name}).count() <= 0:
            return redirect(url_for('projects.projects_backend', language=language))
        project_fields_name = mongo.db.fields.find({"project_name": project_name})[0]["default_fields"]
    # to take the keys of one of the sample as heads of the sample table
    all_fields_name = dict(sample_list[0]).keys()
    all_fields_name.remove("_id")
    all_fields_name.remove("project_name")
    all_fields_name.sort()
    # get all the fields (for more fields)
    if mongo.db.fields.find({"project_name": project_name}).count() <= 0:
        return redirect(url_for('projects.projects_backend', language=language))
    fields_string_type = mongo.db.fields.find({"project_name": project_name})[0]["string_fields"]

    string_field_element = {}

    for sample in sample_list:
        for field in fields_string_type:
            if not string_field_element.has_key(field):
                string_field_element[field] = []
            string_field_element[field].append(sample[field])
    # to get possible value of keys

    filter = {"project_name": project_name}
    for field in project_fields_name:
        if field in fields_string_type:
            value = request.args.get(field, '')
            if value:
                filter[field] = value
        else:
            value = request.args.getlist(field)
            if len(value) > 1:
                try:
                    tmpeval = eval(str(value[1]))
                except:
                    continue
                if value[0] != 'no':
                    if value[0] == 'eq':
                        filter[field] = int(value[1])
                    elif value[0] == 'lt':
                        filter[field] = {'$lt': int(value[1])}
                    elif value[0] == 'gt':
                        filter[field] = {'$gt': int(value[1])}
    # to build the filter

    sample_list = mongo.db.samples.find(filter)
    sample_list_out =  list(mongo.db.samples.find(filter))

    for field in fields_string_type:
        string_field_element[field] = list(set(string_field_element[field]))
        string_field_element[field].sort()

    mapping = {}
    if language == 'cn':
        for field in all_fields_name:
            if mongo.db.mapping.find({'en': field}).count() > 0:
                mapping[field] = mongo.db.mapping.find({'en': field})[0]['cn']
            else:
                mapping[field] = field
    else:
        for field in all_fields_name:
            mapping[field] = field

    all_fields_name.remove('name')
    if 'username' in session:
        user = User.objects.get(username=session['username'])
        count = CreatedProjects.objects(username=user['username'], project_name=project_name).count()
        if count > 0 or user['admin'] == True:
            created_project = True
        else:
            created_project = False
        return render_template('samples.html', language=language, project_name=project_name,
                           sample_list=sample_list, project_fields_name=project_fields_name,
                           all_fields_name=all_fields_name, fields_string_type=fields_string_type,
                           string_field_element=string_field_element, mapping=mapping, user=user,
                           created_project=created_project)
    return render_template('samples.html', language=language, project_name=project_name,
                           sample_list=sample_list, project_fields_name=project_fields_name,
                           all_fields_name=all_fields_name, fields_string_type=fields_string_type,
                           string_field_element=string_field_element, mapping=mapping)

@samples.route('/download-sample', methods=['GET'])
def download_sample_backend():
    project_name = request.args.get('project_name', '')
    samples = mongo.db.samples.find({'project_name': project_name})
    json_string = ''
    samples = list(samples)
    for sample in samples:
        del sample['_id']
        json_string += dumps(sample, ensure_ascii=False) + '\n'
    response = make_response(json_string)
    response.headers['Content-Disposition'] = 'attachment; filename=' + project_name + '.json'
    return response


# -*- coding: utf-8 -*-
# @Filename : doubanTop250.py
# @Functions : 爬取豆瓣前250的图书信息->https://book.douban.com/top250
# @Time : 2022/12/24 10:41
# @Author : Mixue Lin
# @Email : m15528382598@163.com
import requests
from lxml import etree
import pandas as pd

def get_info(url,header,data):
    html = requests.get(url=url,headers=header)
    selector = etree.HTML(html.text)
    infos = selector.xpath('//tr[@class="item"]')
    for info in infos:
        name = info.xpath('td[2]/div[1]/a/@title')[0]
        url = info.xpath('td[2]/div[1]/a/@href')[0]
        book_infos = info.xpath('td[2]/p[1]/text()')[0]
        book_infos = book_infos.split('/')
        author = book_infos[0].strip()
        publisher = book_infos[-3].strip()
        date = book_infos[-2].strip()
        try:
            price = book_infos[-1].strip()
        except IndexError:
            print(book_infos)
            price = " "
        rank = info.xpath('td[2]/div[2]/span[2]/text()')[0]
        comments = info.xpath('td[2]/div[2]/span[3]/text()')[0].strip('(').strip(')').strip()
        try:
            quote = info.xpath('td[2]/p[@class="quote"]/span/text()')[0].strip()
        except IndexError:
            print(info.xpath('td[2]/p[@class="quote"]/span/text()'))
            quote = " "
        for key in data.keys():
            data[key].append(eval(key))

if __name__=="__main__":
    header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'}
    urls = ['https://book.douban.com/top250?start={}'.format(str(num)) for num in range(0,501,25)]
    cols = ["name","url","author","publisher","date","price","rank","comments","quote"]
    data = {}
    for col in cols:
        data[col] = []
    for url in urls:
        get_info(url,header,data)
    data = pd.DataFrame(data)
    data.to_csv("doubanTop250.csv")


from Crypto.Util.number import bytes_to_long, getPrime

def gen_keys(n_bits, p_bits, d_bits):
    q_bits = n_bits - p_bits
    p, q = getPrime(p_bits), getPrime(q_bits)
    N, phi = p*q, (p-1)*(q-1)
    while True:
        d = getPrime(d_bits)
        if d > N**0.292:
            break
    e = pow(d,-1,phi)
    return (N, e), (N, d)

def encrypt_flag(plaintext, pub):
    N, e = pub
    m = bytes_to_long(plaintext)
    return pow(m, e, N)

if __name__ == '__main__':
    with open("flag.txt", "rb") as f:
        flag = f.read().strip()

    n_bits = 1024
    p_bits = 256
    d_bits = 300
    pub, priv = gen_keys(n_bits, p_bits, d_bits)
    c = encrypt_flag(flag, pub)

    print(f"N = {hex(pub[0])}")
    print(f"e = {hex(pub[1])}")
    print(f"c = {hex(c)}")


from typing import Any, List
from src.libs.CrabadaWeb2Client.types.idleGameTypes import Game, Team
from src.strategies.loot.LootStrategy import LootStrategy
from src.helpers.general import firstOrNone

class LowestBp(LootStrategy):
    """
    Looting strategy that chooses the mine with the lowest
    defense points.

    Takes the list of attackable mines from the web2 endpoints,
    which means it is SLOW and will be unlikely to ever succeed
    """

    def query(self, team: Team) -> dict[str, Any]:
        return {
            "can_loot": 1,
            "status": "open",
            "looter_address": team["owner"],
            "limit": self.minesToFetch,
            "orderBy": "game_id",
            "order": "desc",
        }

    def mine(self, team: Team, list: List[Game]) -> Game:
        attackableMines = [m for m in list if m["defense_point"] < team["battle_point"]]
        if len(attackableMines) == 0:
            return None
        sortedAttackableMines = sorted(
            attackableMines, key=lambda m: (m["defense_point"], m["defense_mine_point"])
        )
        return firstOrNone(sortedAttackableMines)


import hashlib
import time
from prettytable import PrettyTable
from module.csrfScan import csrf_scan
from module.scopeScan import scope_scan
from module.openRedirect import opredirect_scan
from module.covertRedirect import coredirect_scan
from module.soRedirect import soredirect_scan
from db.db import Database
from db.logger import logger

import copy

logger = logger()
db = Database()

'''
def show_result(task):
	self.db.fetch_records('csrf','{"scanid":"%s"}' % (self.scanid))

def code_valid(task):
	if 'cookies' in self.target1 and self.target1['cookies'] != '':
		code_scan(task)
	else:
		print('[-] Code Scan need to provide cookies or just provide An Authoration Code with -code.')

def token_valid(task):
	if 'cookies' in self.target1 and self.target1['cookies'] != '':
		token_scan(task)
	else:
		print('[-] Token Scan need to provide cookies or just provide An Authoration Code with -code.')
'''

def fetch_data(col,data,table,i):
	#print(data)
	result = db.fetch_records(col,data)
	for res in result:
		i += 1
		table.add_row([logger.G+str(i)+logger.W,logger.G+res['type']+logger.W,logger.G+res['payload']+logger.W])
	return table,i

def show_result(scanid):
	data = '{"scanid":"%s"}' % str(scanid)
	col = ['csrf','Scope','Redirection']
	table = PrettyTable(['id','vul type','payload'])
	i = 0
	for _ in col:
		table,i = fetch_data(_,data,table,i)
		#print("%s##############################################################%s" % (logger.G,logger.W))
		#print("\n")
	print(table)

def coRedirect(task):
	print("[ ]Covert Redirection Scan starting.")
	res = coredirect_scan(task)
	if res:
		print("[ ]Covert Redirection Scan has been completed.")

def soRedirect(task):
	print("[ ] Same Origin Redirection Scan starting.")
	res = soredirect_scan(task)
	if res:
		print("[ ]Same Origin Redirection Scan has been completed.")

def openRedirect(task):
	print("[ ]Open Redirction Scan starting.")
	res = opredirect_scan(task)
	if res:
		print("[ ]Open Redirection Scan has been completed.")

def scopeScan(task):
	print("[ ]Scope Escalation Scan starting.")
	res = scope_scan(task)
	if res:
		print("[ ]Scope Escalation Scan has been completed.")

def csrfScan(task):
	print("[ ]CSRF Scan starting.")
	res = csrf_scan(task)
	if res:
		print("[ ]CSRF Scan has been completed.")

def generate_scanid():
	scanid = hashlib.md5(str(time.time()).encode("utf-8")).hexdigest()
	return scanid

def scantask(target1,target2):
	scanid = generate_scanid()
	task = {}
	task['scanid'] = scanid
	task['target1'] = target1
	task['target2'] = target2
	task1 = copy.deepcopy(task)
	task2 = copy.deepcopy(task)
	task3 = copy.deepcopy(task)
	task4 = copy.deepcopy(task)
	# Start Scanning.
	# Start CSRF Scanning.
	csrfScan(task)
	# Start Scope Scanning.
	scopeScan(task1)
	# Start Redirection Scanning.
	# Start Open Redirection Scanning.
	openRedirect(task2)
	# Start Same Origin Redirection Scanning.
	soRedirect(task3)
	# Start Covert Redirection Scanning.
	# coRedirect(task4)
	# Start Authorization Code Verification
	print("[ ]Scan has been completed. Show the result.")
	show_result(scanid)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 16 11:27:34 2020

@author: ryanoreilly
This script will create the final dataframes that represent the parameter for all years and all regions
HR04 was split into HR05, HR06 and HR02
"""

import pandas as pd ## necessary data analysis package
import numpy as np ## necessary data analysis package
import os
import openpyxl as xl
from decimal import Decimal
### Set options
#os.chdir('I:\Projekte\OpenEntrance - WV0173\Durchführungsphase\WP6\CS1\gitlab\datainputs') # set wd - change to server path - IMPORTANT
os.chdir('/Users/ryanoreilly/Desktop/Energy Institute/openEntrance/data')
## TEST TEST COMMMENT
# Display 6 columns for viewing purposes
pd.set_option('display.max_columns', 6)
# Reduce decimal points to 2
pd.options.display.float_format = '{:,.2f}'.format

# data that holds gils dissertaion assumptions about future varaibles
gils = pd.read_excel('/Users/ryanoreilly/Desktop/Energy Institute/openEntrance/data/openENTRANCE projection/gils projection assumptions.xlsx')
gils = gils.sort_values(by = "country")
gils = gils.reset_index()
gils = gils.drop('index', axis=1)

# regional household data for 2018 and 2019
dfnhh = pd.read_excel('/Users/ryanoreilly/Desktop/Energy Institute/openEntrance/data/number households/EUROSTAT Nuts 2 HH.xlsx')
# make adjustements for Croatia
temp = dfnhh[dfnhh['nutscode']=="HR04"]
dfnhh = dfnhh[dfnhh.nutscode != 'HR04']
data = {'nutscode':["HR05","HR06","HR02"],
        'nhh_2018':[float(temp['nhh_2018']/3),float(temp['nhh_2018']/3),float(temp['nhh_2018']/3)],
        'nhh_2019':[float(temp['nhh_2019']/3),float(temp['nhh_2019']/3),float(temp['nhh_2019']/3)],
        'rNat_NHH':[float(temp['rNat_NHH']/3),float(temp['rNat_NHH']/3),float(temp['rNat_NHH']/3)],
        'country': ["HR","HR","HR"]}
temp2 = pd.DataFrame(data)
dfnhh = dfnhh.append(temp2)
# drop old nuts regions
dfnhh = dfnhh[dfnhh.nutscode != 'HU10']
dfnhh = dfnhh[dfnhh.nutscode != 'IE01']
dfnhh = dfnhh[dfnhh.nutscode != 'IE02']
dfnhh = dfnhh[dfnhh.nutscode != 'SI01']
dfnhh = dfnhh[dfnhh.nutscode != 'SI02']
dfnhh = dfnhh[dfnhh.nutscode != 'UKI1']
dfnhh = dfnhh[dfnhh.nutscode != 'UKI2']
dfnhh = dfnhh[dfnhh.nutscode != 'UKM2']
dfnhh = dfnhh[dfnhh.nutscode != 'UKM3']
dfnhh = dfnhh[dfnhh.nutscode != 'LT00']
dfnhh = dfnhh.reset_index()
dfnhh = dfnhh.drop('index', axis=1)
#add norway and switzerland
norway     = pd.read_excel('./number households/norway households.xlsx')
switzerland= pd.read_excel('./number households/switzerland households.xlsx')
CHnutscode = pd.read_excel('./number households/CH code to nutscode.xlsx')
switzerland = switzerland.merge(CHnutscode,on = "Code")
switzerland = switzerland.groupby(['nutscode']).sum()
switzerland['nutscode']= switzerland.index

#concatinate df's
#add norway
frames=[dfnhh,norway]
dfnhh =pd.concat(frames,ignore_index=True)
#add Switzerland
frames=[dfnhh,switzerland]
dfnhh =pd.concat(frames,ignore_index=True)

#add country Acronym
dfnhh['country'] = ""
for row in range(0,len(dfnhh['nutscode'])):
    dfnhh['country'][row]=dfnhh['nutscode'][row][0:2]

del norway
del switzerland
del CHnutscode
dfnhh = dfnhh.sort_values(by = "nutscode")
dfnhh = dfnhh.reset_index()
dfnhh = dfnhh.drop('index', axis=1)
#############
# nflh ac
#############
#gils = gils.drop('index', axis=1)
temp = gils[['country', 'nflh_ac_2010','nflh_ac_2020','nflh_ac_2030','nflh_ac_2050']]
temp['r10']= (temp['nflh_ac_2020']-temp['nflh_ac_2010'])/10
temp['r20']= (temp['nflh_ac_2030']-temp['nflh_ac_2020'])/10
temp['r30']= (temp['nflh_ac_2050']-temp['nflh_ac_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
nflh_ac = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(nflh_ac)):
    nflh_ac['2020'][row] = temp['nflh_ac_2020'][row]
    nflh_ac['2030'][row] = temp['nflh_ac_2030'][row]
    nflh_ac['2050'][row] = temp['nflh_ac_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(nflh_ac)):
    for _10 in range(0,2): # all years before 2020
        nflh_ac[columns[_10]][row] = round(temp['nflh_ac_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),0)
    for _20 in range(3,12):#  2020 < years < 2030
        nflh_ac[columns[_20]][row] = round(temp['nflh_ac_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020)),0)
    for _30 in range(13,32):
        nflh_ac[columns[_30]][row] = round(temp['nflh_ac_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030)),0)

nflh_ac.to_csv(r'./openENTRANCE final data/nflh_ac.csv')
del nflh_ac

#############
# nflh cp
#############
temp = gils[['country', 'nflh_cp_2010','nflh_cp_2020','nflh_cp_2030','nflh_cp_2050']]
temp['r10']= (temp['nflh_cp_2020']-temp['nflh_cp_2010'])/10
temp['r20']= (temp['nflh_cp_2030']-temp['nflh_cp_2020'])/10
temp['r30']= (temp['nflh_cp_2050']-temp['nflh_cp_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
nflh_cp = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(nflh_cp)):
    nflh_cp['2020'][row] = temp['nflh_cp_2020'][row]
    nflh_cp['2030'][row] = temp['nflh_cp_2030'][row]
    nflh_cp['2050'][row] = temp['nflh_cp_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(nflh_cp)):
    for _10 in range(0,2): # all years before 2020
        nflh_cp[columns[_10]][row] = round(temp['nflh_cp_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),0)
    for _20 in range(3,12):#  2020 < years < 2030
        nflh_cp[columns[_20]][row] = round(temp['nflh_cp_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020)),0)
    for _30 in range(13,32):
        nflh_cp[columns[_30]][row] = round(temp['nflh_cp_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030)),0)

nflh_cp.to_csv(r'./openENTRANCE final data/nflh_cp.csv')
del nflh_cp

#############
#Begin wunit
#############

#############
#rf_fr
#############
temp = gils[['country',  'Wunit_rf_fr_2010', 'Wunit_rf_fr_2020', 'Wunit_rf_fr_2030', 'Wunit_rf_fr_2050']]
temp['r10']= (temp['Wunit_rf_fr_2020']-temp['Wunit_rf_fr_2010'])/10
temp['r20']= (temp['Wunit_rf_fr_2030']-temp['Wunit_rf_fr_2020'])/10
temp['r30']= (temp['Wunit_rf_fr_2050']-temp['Wunit_rf_fr_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
Wunit_rf_fr = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(Wunit_rf_fr)):
    Wunit_rf_fr['2020'][row] = temp['Wunit_rf_fr_2020'][row]
    Wunit_rf_fr['2030'][row] = temp['Wunit_rf_fr_2030'][row]
    Wunit_rf_fr['2050'][row] = temp['Wunit_rf_fr_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(Wunit_rf_fr)):
    for _10 in range(0,2): # all years before 2020
        Wunit_rf_fr[columns[_10]][row] = round(temp['Wunit_rf_fr_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),0)
    for _20 in range(3,12):#  2020 < years < 2030
        Wunit_rf_fr[columns[_20]][row] = round(temp['Wunit_rf_fr_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020)),0)
    for _30 in range(13,32):
        Wunit_rf_fr[columns[_30]][row] = round(temp['Wunit_rf_fr_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030)),0)

Wunit_rf_fr.to_csv(r'./openENTRANCE final data/Wunit_rf_fr.csv')
del Wunit_rf_fr

#############
#Begin Pcycle
#############

#############
#Pcycle wm
#############
temp = gils[['country',  'Pcycle_wm_2010', 'Pcycle_wm_2020', 'Pcycle_wm_2030', 'Pcycle_wm_2050']]
temp['r10']= (temp['Pcycle_wm_2020']-temp['Pcycle_wm_2010'])/10
temp['r20']= (temp['Pcycle_wm_2030']-temp['Pcycle_wm_2020'])/10
temp['r30']= (temp['Pcycle_wm_2050']-temp['Pcycle_wm_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
Pcycle_wm = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(Pcycle_wm)):
    Pcycle_wm['2020'][row] = temp['Pcycle_wm_2020'][row]
    Pcycle_wm['2030'][row] = temp['Pcycle_wm_2030'][row]
    Pcycle_wm['2050'][row] = temp['Pcycle_wm_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(Pcycle_wm)):
    for _10 in range(0,2): # all years before 2020
        Pcycle_wm[columns[_10]][row] = round(temp['Pcycle_wm_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)
    for _20 in range(3,12):#  2020 < years < 2030
        Pcycle_wm[columns[_20]][row] = round(temp['Pcycle_wm_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020)),2)
    for _30 in range(13,32):
        Pcycle_wm[columns[_30]][row] = round(temp['Pcycle_wm_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030)),2)

Pcycle_wm.to_csv(r'./openENTRANCE final data/Pcycle_wm.csv')
del Pcycle_wm

#############
#Pcycle td
#############
temp = gils[['country',  'Pcycle_td_2010', 'Pcycle_td_2020', 'Pcycle_td_2030', 'Pcycle_td_2050']]
temp['r10']= (temp['Pcycle_td_2020']-temp['Pcycle_td_2010'])/10
temp['r20']= (temp['Pcycle_td_2030']-temp['Pcycle_td_2020'])/10
temp['r30']= (temp['Pcycle_td_2050']-temp['Pcycle_td_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
Pcycle_td = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(Pcycle_td)):
    Pcycle_td['2020'][row] = temp['Pcycle_td_2020'][row]
    Pcycle_td['2030'][row] = temp['Pcycle_td_2030'][row]
    Pcycle_td['2050'][row] = temp['Pcycle_td_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(Pcycle_td)):
    for _10 in range(0,2): # all years before 2020
        Pcycle_td[columns[_10]][row] = round(temp['Pcycle_td_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)
    for _20 in range(3,12):#  2020 < years < 2030
        Pcycle_td[columns[_20]][row] = round(temp['Pcycle_td_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020)),2)
    for _30 in range(13,32):
        Pcycle_td[columns[_30]][row] = round(temp['Pcycle_td_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030)),2)

Pcycle_td.to_csv(r'./openENTRANCE final data/Pcycle_td.csv')
del Pcycle_td

#############
#Pcycle dw
#############
temp = gils[['country',  'Pcycle_dw_2010', 'Pcycle_dw_2020', 'Pcycle_dw_2030', 'Pcycle_dw_2050']]
temp['r10']= (temp['Pcycle_dw_2020']-temp['Pcycle_dw_2010'])/10
temp['r20']= (temp['Pcycle_dw_2030']-temp['Pcycle_dw_2020'])/10
temp['r30']= (temp['Pcycle_dw_2050']-temp['Pcycle_dw_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
Pcycle_dw = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(Pcycle_dw)):
    Pcycle_dw['2020'][row] = temp['Pcycle_dw_2020'][row]
    Pcycle_dw['2030'][row] = temp['Pcycle_dw_2030'][row]
    Pcycle_dw['2050'][row] = temp['Pcycle_dw_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(Pcycle_dw)):
    for _10 in range(0,2): # all years before 2020
        Pcycle_dw[columns[_10]][row] = round(temp['Pcycle_dw_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)
    for _20 in range(3,12):#  2020 < years < 2030
        Pcycle_dw[columns[_20]][row] = round(temp['Pcycle_dw_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020)),2)
    for _30 in range(13,32):
        Pcycle_dw[columns[_30]][row] = round(temp['Pcycle_dw_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030)),2)

Pcycle_dw.to_csv(r'./openENTRANCE final data/Pcycle_dw.csv')
del Pcycle_dw

#############
# Begin Punit
#############

#############
# Punit ac
#############
temp = gils[['country',  'Punit_ac_2010', 'Punit_ac_2020', 'Punit_ac_2030', 'Punit_ac_2050']]
temp['r10']= (temp['Punit_ac_2020']-temp['Punit_ac_2010'])/10
temp['r20']= (temp['Punit_ac_2030']-temp['Punit_ac_2020'])/10
temp['r30']= (temp['Punit_ac_2050']-temp['Punit_ac_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
Punit_ac = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(Punit_ac)):
    Punit_ac['2020'][row] = temp['Punit_ac_2020'][row]
    Punit_ac['2030'][row] = temp['Punit_ac_2030'][row]
    Punit_ac['2050'][row] = temp['Punit_ac_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(Punit_ac)):
    for _10 in range(0,2): # all years before 2020
        Punit_ac[columns[_10]][row] = round(temp['Punit_ac_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)
    for _20 in range(3,12):#  2020 < years < 2030
        Punit_ac[columns[_20]][row] = round(temp['Punit_ac_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020)),2)
    for _30 in range(13,32):
        Punit_ac[columns[_30]][row] = round(temp['Punit_ac_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030)),2)

Punit_ac.to_csv(r'./openENTRANCE final data/Punit_ac.csv')
del Punit_ac

#############
# Punit cp
#############
temp = gils[['country',  'Punit_cp_2010', 'Punit_cp_2020', 'Punit_cp_2030', 'Punit_cp_2050']]
temp['r10']= (temp['Punit_cp_2020']-temp['Punit_cp_2010'])/10
temp['r20']= (temp['Punit_cp_2030']-temp['Punit_cp_2020'])/10
temp['r30']= (temp['Punit_cp_2050']-temp['Punit_cp_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
Punit_cp = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(Punit_cp)):
    Punit_cp['2020'][row] = temp['Punit_cp_2020'][row]
    Punit_cp['2030'][row] = temp['Punit_cp_2030'][row]
    Punit_cp['2050'][row] = temp['Punit_cp_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(Punit_cp)):
    for _10 in range(0,2): # all years before 2020
        Punit_cp[columns[_10]][row] = round(temp['Punit_cp_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)
    for _20 in range(3,12):#  2020 < years < 2030
        Punit_cp[columns[_20]][row] = round(temp['Punit_cp_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020)),2)
    for _30 in range(13,32):
        Punit_cp[columns[_30]][row] = round(temp['Punit_cp_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030)),2)

Punit_cp.to_csv(r'./openENTRANCE final data/Punit_cp.csv')
del Punit_cp

#############
# Begin rates of ownership
#############

#############
# rfr
#############
temp = gils[['country',  'rfr_2010', 'rfr_2050']]
temp['r10']= (temp['rfr_2050']-temp['rfr_2010'])/40

columns = []
for i in range(2018,2051):
    columns.append(str(i))
rfr = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(rfr)):
    rfr['2050'][row] = temp['rfr_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(rfr)):
    for _10 in range(0,len(columns)): # 2018-2050
        rfr[columns[_10]][row] = round(temp['rfr_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)

rfr.to_csv(r'./openENTRANCE final data/rfr.csv')
del rfr

#############
# rrf
#############
temp = gils[['country',  'rrf_2010', 'rrf_2050']]
temp['r10']= (temp['rrf_2050']-temp['rrf_2010'])/40

columns = []
for i in range(2018,2051):
    columns.append(str(i))
rrf = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(rrf)):
    rrf['2050'][row] = temp['rrf_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(rrf)):
    for _10 in range(0,len(columns)): # 2018-2050
        rrf[columns[_10]][row] = round(temp['rrf_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)

rrf.to_csv(r'./openENTRANCE final data/rrf.csv')
del rrf

#############
# rwm
#############
temp = gils[['country',  'rwm_2010', 'rwm_2050']]
temp['r10']= (temp['rwm_2050']-temp['rwm_2010'])/40

columns = []
for i in range(2018,2051):
    columns.append(str(i))
rwm = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(rwm)):
    rwm['2050'][row] = temp['rwm_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(rwm)):
    for _10 in range(0,len(columns)): # 2018-2050
        rwm[columns[_10]][row] = round(temp['rwm_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)

rwm.to_csv(r'./openENTRANCE final data/rwm.csv')
del rwm

#############
# rtd
#############
temp = gils[['country',  'rtd_2010', 'rtd_2050']]
temp['r10']= (temp['rtd_2050']-temp['rtd_2010'])/40

columns = []
for i in range(2018,2051):
    columns.append(str(i))
rtd = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(rtd)):
    rtd['2050'][row] = temp['rtd_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(rtd)):
    for _10 in range(0,len(columns)): # 2018-2050
        rtd[columns[_10]][row] = round(temp['rtd_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)

rtd.to_csv(r'./openENTRANCE final data/rtd.csv')
del rtd

#############
# rdw
#############
temp = gils[['country',  'rdw_2010', 'rdw_2050']]
temp['r10']= (temp['rdw_2050']-temp['rdw_2010'])/40

columns = []
for i in range(2018,2051):
    columns.append(str(i))
rdw = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(rdw)):
    rdw['2050'][row] = temp['rdw_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(rdw)):
    for _10 in range(0,len(columns)): # 2018-2050
        rdw[columns[_10]][row] = round(temp['rdw_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)

rdw.to_csv(r'./openENTRANCE final data/rdw.csv')
del rdw

#############
# rac
#############
temp = gils[['country',  'rac_2010', 'rac_2050']]
temp['r10']= (temp['rac_2050']-temp['rac_2010'])/40

columns = []
for i in range(2018,2051):
    columns.append(str(i))
rac = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(rac)):
    rac['2050'][row] = temp['rac_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(rac)):
    for _10 in range(0,len(columns)): # 2018-2050
        rac[columns[_10]][row] = round(temp['rac_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)

rac.to_csv(r'./openENTRANCE final data/rac.csv')
del rac

#############
# rwh
#############
temp = gils[['country',  'rwh_2010', 'rwh_2050']]
temp['r10']= (temp['rwh_2050']-temp['rwh_2010'])/40

columns = []
for i in range(2018,2051):
    columns.append(str(i))
rwh = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(rwh)):
    rwh['2050'][row] = temp['rwh_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(rwh)):
    for _10 in range(0,len(columns)): # 2018-2050
        rwh[columns[_10]][row] = round(temp['rwh_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)

rwh.to_csv(r'./openENTRANCE final data/rwh.csv')
del rwh

#############
# rcp
#############
temp = gils[['country',  'rcp_2010', 'rcp_2050']]
temp['r10']= (temp['rcp_2050']-temp['rcp_2010'])/40

columns = []
for i in range(2018,2051):
    columns.append(str(i))
rcp = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(rcp)):
    rcp['2050'][row] = temp['rcp_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(rcp)):
    for _10 in range(0,len(columns)): # 2018-2050
        rcp[columns[_10]][row] = round(temp['rcp_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)

rcp.to_csv(r'./openENTRANCE final data/rcp.csv')
del rcp

#############
# rsh
#############
temp = gils[['country',  'rsh_2010', 'rsh_2050']]
temp['r10']= (temp['rsh_2050']-temp['rsh_2010'])/40

columns = []
for i in range(2018,2051):
    columns.append(str(i))
rsh = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(rsh)):
    rsh['2050'][row] = temp['rsh_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(rsh)):
    for _10 in range(0,len(columns)): # 2018-2050
        rsh[columns[_10]][row] = round(temp['rsh_2010'][row]+(temp['r10'][row]*(int(columns[_10])-2010)),2)

rsh.to_csv(r'./openENTRANCE final data/rsh.csv')
del rsh

#############
# sreduction & sincrease
#############

#############
# sred_wm
#############
temp = gils[['country',  'sred_wm_2020', 'sred_wm_2030','sred_wm_2050']]
temp['r20']= (temp['sred_wm_2030']-temp['sred_wm_2020'])/10
temp['r30']= (temp['sred_wm_2050']-temp['sred_wm_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
sred_wm = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(sred_wm)):
    sred_wm['2020'][row] = temp['sred_wm_2020'][row]
    sred_wm['2030'][row] = temp['sred_wm_2030'][row]
    sred_wm['2050'][row] = temp['sred_wm_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(sred_wm)):
    for _10 in range(1,-1,-1): # all years before 2020
        sred_wm[columns[_10]][row] = temp['sred_wm_2020'][row]+(temp['r20'][row]*(int(columns[_10])-2020))
    for _20 in range(3,12):#  2020 < years < 2030
        sred_wm[columns[_20]][row] = temp['sred_wm_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020))
    for _30 in range(13,32):
        sred_wm[columns[_30]][row] = temp['sred_wm_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030))

sred_wm.to_csv(r'./openENTRANCE final data/sred_wm.csv')
del sred_wm

#############
# sinc_wm
#############
temp = gils[['country',  'sinc_wm_2020', 'sinc_wm_2030','sinc_wm_2050']]
temp['r20']= (temp['sinc_wm_2030']-temp['sinc_wm_2020'])/10
temp['r30']= (temp['sinc_wm_2050']-temp['sinc_wm_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
sinc_wm = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(sinc_wm)):
    sinc_wm['2020'][row] = temp['sinc_wm_2020'][row]
    sinc_wm['2030'][row] = temp['sinc_wm_2030'][row]
    sinc_wm['2050'][row] = temp['sinc_wm_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(sinc_wm)):
    for _10 in range(1,-1,-1): # all years before 2020
        sinc_wm[columns[_10]][row] = temp['sinc_wm_2020'][row]+(temp['r20'][row]*(int(columns[_10])-2020))
    for _20 in range(3,12):#  2020 < years < 2030
        sinc_wm[columns[_20]][row] = temp['sinc_wm_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020))
    for _30 in range(13,32):
        sinc_wm[columns[_30]][row] = temp['sinc_wm_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030))

sinc_wm.to_csv(r'./openENTRANCE final data/sinc_wm.csv')
del sinc_wm

#############
# sred_td
#############
temp = gils[['country',  'sred_td_2020', 'sred_td_2030','sred_td_2050']]
temp['r20']= (temp['sred_td_2030']-temp['sred_td_2020'])/10
temp['r30']= (temp['sred_td_2050']-temp['sred_td_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
sred_td = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(sred_td)):
    sred_td['2020'][row] = temp['sred_td_2020'][row]
    sred_td['2030'][row] = temp['sred_td_2030'][row]
    sred_td['2050'][row] = temp['sred_td_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(sred_td)):
    for _10 in range(1,-1,-1): # all years before 2020
        sred_td[columns[_10]][row] = temp['sred_td_2020'][row]+(temp['r20'][row]*(int(columns[_10])-2020))
    for _20 in range(3,12):#  2020 < years < 2030
        sred_td[columns[_20]][row] = temp['sred_td_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020))
    for _30 in range(13,32):
        sred_td[columns[_30]][row] = temp['sred_td_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030))

sred_td.to_csv(r'./openENTRANCE final data/sred_td.csv')
del sred_td

#############
# sinc_td
#############
temp = gils[['country',  'sinc_td_2020', 'sinc_td_2030','sinc_td_2050']]
temp['r20']= (temp['sinc_td_2030']-temp['sinc_td_2020'])/10
temp['r30']= (temp['sinc_td_2050']-temp['sinc_td_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
sinc_td = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(sinc_td)):
    sinc_td['2020'][row] = temp['sinc_td_2020'][row]
    sinc_td['2030'][row] = temp['sinc_td_2030'][row]
    sinc_td['2050'][row] = temp['sinc_td_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(sinc_td)):
    for _10 in range(1,-1,-1): # all years before 2020
        sinc_td[columns[_10]][row] = temp['sinc_td_2020'][row]+(temp['r20'][row]*(int(columns[_10])-2020))
    for _20 in range(3,12):#  2020 < years < 2030
        sinc_td[columns[_20]][row] = temp['sinc_td_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020))
    for _30 in range(13,32):
        sinc_td[columns[_30]][row] = temp['sinc_td_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030))

sinc_td.to_csv(r'./openENTRANCE final data/sinc_td.csv')
del sinc_td

#############
# sred_dw
#############
temp = gils[['country',  'sred_dw_2020', 'sred_dw_2030','sred_dw_2050']]
temp['r20']= (temp['sred_dw_2030']-temp['sred_dw_2020'])/10
temp['r30']= (temp['sred_dw_2050']-temp['sred_dw_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
sred_dw = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(sred_dw)):
    sred_dw['2020'][row] = temp['sred_dw_2020'][row]
    sred_dw['2030'][row] = temp['sred_dw_2030'][row]
    sred_dw['2050'][row] = temp['sred_dw_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(sred_dw)):
    for _10 in range(1,-1,-1): # all years before 2020
        sred_dw[columns[_10]][row] = temp['sred_dw_2020'][row]+(temp['r20'][row]*(int(columns[_10])-2020))
    for _20 in range(3,12):#  2020 < years < 2030
        sred_dw[columns[_20]][row] = temp['sred_dw_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020))
    for _30 in range(13,32):
        sred_dw[columns[_30]][row] = temp['sred_dw_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030))

sred_dw.to_csv(r'./openENTRANCE final data/sred_dw.csv')
del sred_dw

#############
# sinc_dw
#############
temp = gils[['country',  'sinc_dw_2020', 'sinc_dw_2030','sinc_dw_2050']]
temp['r20']= (temp['sinc_dw_2030']-temp['sinc_dw_2020'])/10
temp['r30']= (temp['sinc_dw_2050']-temp['sinc_dw_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
sinc_dw = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(sinc_dw)):
    sinc_dw['2020'][row] = temp['sinc_dw_2020'][row]
    sinc_dw['2030'][row] = temp['sinc_dw_2030'][row]
    sinc_dw['2050'][row] = temp['sinc_dw_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(sinc_dw)):
    for _10 in range(1,-1,-1): # all years before 2020
        sinc_dw[columns[_10]][row] = temp['sinc_dw_2020'][row]+(temp['r20'][row]*(int(columns[_10])-2020))
    for _20 in range(3,12):#  2020 < years < 2030
        sinc_dw[columns[_20]][row] = temp['sinc_dw_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020))
    for _30 in range(13,32):
        sinc_dw[columns[_30]][row] = temp['sinc_dw_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030))

sinc_dw.to_csv(r'./openENTRANCE final data/sinc_dw.csv')
del sinc_dw

#############
# sred_ac
#############
temp = gils[['country',  'sred_ac_2020', 'sred_ac_2030','sred_ac_2050']]
temp['r20']= (temp['sred_ac_2030']-temp['sred_ac_2020'])/10
temp['r30']= (temp['sred_ac_2050']-temp['sred_ac_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
sred_ac = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(sred_ac)):
    sred_ac['2020'][row] = temp['sred_ac_2020'][row]
    sred_ac['2030'][row] = temp['sred_ac_2030'][row]
    sred_ac['2050'][row] = temp['sred_ac_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(sred_ac)):
    for _10 in range(1,-1,-1): # all years before 2020
        sred_ac[columns[_10]][row] = temp['sred_ac_2020'][row]+(temp['r20'][row]*(int(columns[_10])-2020))
    for _20 in range(3,12):#  2020 < years < 2030
        sred_ac[columns[_20]][row] = temp['sred_ac_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020))
    for _30 in range(13,32):
        sred_ac[columns[_30]][row] = temp['sred_ac_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030))

sred_ac.to_csv(r'./openENTRANCE final data/sred_ac.csv')
del sred_ac

#############
# sred_cp
#############
temp = gils[['country',  'sred_cp_2020', 'sred_cp_2030','sred_cp_2050']]
temp['r20']= (temp['sred_cp_2030']-temp['sred_cp_2020'])/10
temp['r30']= (temp['sred_cp_2050']-temp['sred_cp_2030'])/20

columns = []
for i in range(2018,2051):
    columns.append(str(i))
sred_cp = pd.DataFrame(index = gils['country'], columns = columns)

#fill in data for known years
for row in range(0,len(sred_cp)):
    sred_cp['2020'][row] = temp['sred_cp_2020'][row]
    sred_cp['2030'][row] = temp['sred_cp_2030'][row]
    sred_cp['2050'][row] = temp['sred_cp_2050'][row]

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(sred_cp)):
    for _10 in range(1,-1,-1): # all years before 2020
        sred_cp[columns[_10]][row] = temp['sred_cp_2020'][row]+(temp['r20'][row]*(int(columns[_10])-2020))
    for _20 in range(3,12):#  2020 < years < 2030
        sred_cp[columns[_20]][row] = temp['sred_cp_2020'][row]+(temp['r20'][row]*(int(columns[_20])-2020))
    for _30 in range(13,32):
        sred_cp[columns[_30]][row] = temp['sred_cp_2030'][row]+(temp['r30'][row]*(int(columns[_30])-2030))

sred_cp.to_csv(r'./openENTRANCE final data/sred_cp.csv')
del sred_cp

#############
# NHH to regional NHH projections
#############

#############
# nhh
#############
# gils is missing Serbia, Masedonia, Montenegro while Household data file does not have lichtenstein
# I need to merge the two to get the final list of nuts 2 regions to be used in the loop below
temp1 = dfnhh
temp2 = gils[['country','nhh_2010','nhh_2020', 'nhh_2030', 'nhh_2050']]
temp3 = temp2.merge(temp1)
temp3 = temp3.reset_index()
temp3 = temp3.drop('index', axis=1)
temp3['nhh30'] =temp3['nhh_2030']*temp3['rNat_NHH']*10**6
temp3['nhh50'] =temp3['nhh_2050']*temp3['rNat_NHH']*10**6
del temp1
del temp2

columns = []
for i in range(2018,2051):
    columns.append(str(i))
nhh = pd.DataFrame(index = temp3['nutscode'], columns = columns)

country_list =np.unique(temp3['country']) #countries used in the analysis

cntry_nuts = {} # countries nuts codes or regions
for i in country_list:
    temp_df = temp3[temp3['country']== i]
    cntry_nuts[i] = np.unique(temp_df['nutscode'])

#fill in data for known years
for row in range(0,len(nhh)):
    nhh['2018'][row] = temp3['nhh_2018'][row]
    nhh['2019'][row] = temp3['nhh_2019'][row]
    nhh['2030'][row] = temp3['nhh30'][row]
    nhh['2050'][row] = temp3['nhh50'][row]

temp3['r19']= ((temp3['nhh30']-temp3['nhh_2019'])/11)
temp3['r30']= ((temp3['nhh50']-temp3['nhh30'])/20)

# use linear growth function to find data for unknown years pv+growth*(t present - t beginning))
for row in range(0,len(temp3)):
    for _20 in range(2,12):#  2020 < years < 2030
        nhh[columns[_20]][row] = nhh['2019'][row]+(temp3['r19'][row]*(int(columns[_20])-2019))
    for _30 in range(13,32):
        nhh[columns[_30]][row] = nhh['2030'][row]+(temp3['r30'][row]*(int(columns[_30])-2030))

nhh.to_csv(r'./openENTRANCE final data/nhh.csv')


from operator import itemgetter

class BasePacketMeta(type):

    registry = {}

    def __new__(cls, name, bases, dic):
        token = dic['token']
        if 'fields' not in dic:
            numargs = 0
            dic['__slots__'] = ()
        else:
            numargs = len(dic['fields'])
            dic['__slots__'] = tuple(map(itemgetter(0), dic['fields']))
        self = super().__new__(cls, name, bases, dic)
        if token is not None:
            cls.registry[token.encode('ascii'), numargs] = self
        return self

class BasePacket(metaclass=BasePacketMeta):
    token = None
    fields = []

    def __init__(self, *args):
        assert len(args) == len(self.fields), (args, self.fields)
        for (k, _), v in zip(self.fields, args):
            setattr(self, k, v)

    def __eq__(self, other):
        return (self.__class__ == other.__class__ and
                all(getattr(self, a) == getattr(other, a)
                    for a, _ in self.fields))

    def __repr__(self):
        return '<{} {}>'.format(self.__class__.__name__,
            ', '.join('{}={}'.format(a, getattr(self, a))
                      for a, _ in self.fields))

class Inserted(BasePacket):
    token = "INSERTED"
    fields = [('job_id', int)]

class Buried(BasePacket):
    token = "BURIED"

class Using(BasePacket):
    token = "USING"
    fields = [('tube', str)]

class Reserved(BasePacket):
    token = "RESERVED"
    fields = [('job_id', int), ('data', bytes)]

class Deleted(BasePacket):
    token = "DELETED"

class Released(BasePacket):
    token = "RELEASED"

class Touched(BasePacket):
    token = "TOUCHED"

class Watching(BasePacket):
    token = "WATCHING"
    fields = [('count', int)]

class Found(BasePacket):
    token = "FOUND"
    fields = [('job_id', int), ('data', bytes)]

class Kicked(BasePacket):
    token = "KICKED"

class Ok(BasePacket):
    token = "OK"
    fields = [('data', bytes)]

class Paused(BasePacket):
    token = "PAUSED"


from typing import Optional

import numpy as np
import torch
from torchtyping import TensorType

from gflownet.envs.tree import Tree
from gflownet.proxy.base import Proxy

class TreeProxy(Proxy):
    """
    Simple decision tree proxy that uses empirical frequency of correct predictions for
    computing likelihood, and the number of nodes in the tree for computing the prior.
    """

    def __init__(self, use_prior: bool = True, beta: float = 1.0, **kwargs):
        """
        Parameters
        ----------
        use_prior : bool
            Whether to use -likelihood * prior for energy computation or just the
            -likelihood.
        beta : float
            Beta coefficient in `prior = np.exp(-self.beta * n_nodes)`. Note that
            this is temporary prior implementation that was used for debugging,
            in combination with reward_func="boltzmann" it doesn't make much sense.
        """
        super().__init__(**kwargs)

        self.use_prior = use_prior
        self.beta = beta
        self.X = None
        self.y = None

    def setup(self, env: Optional[Tree] = None):
        self.X = env.X_train
        self.y = env.y_train

    def __call__(self, states: TensorType["batch", "state_dim"]) -> TensorType["batch"]:
        energies = []

        for state in states:
            predictions = []
            for x in self.X:
                predictions.append(Tree.predict(state, x))
            likelihood = (np.array(predictions) == self.y).mean()

            if self.use_prior:
                n_nodes = Tree.get_n_nodes(state)
                prior = np.exp(-self.beta * n_nodes)
            else:
                prior = 1

            energies.append(-likelihood * prior)

        return torch.Tensor(energies)


#! -*- coding:utf-8 -*-
# 句子对分类任务，LCQMC数据集
# val_acc: 0.887071, test_acc: 0.870320

import numpy as np
import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

from keras.callbacks import EarlyStopping, ModelCheckpoint

from path import BASE_CONFIG_NAME, BASE_CKPT_NAME, BASE_MODEL_DIR, train_file_path_Similarity, val_file_path_Similarity, \
    test_file_path_Similarity, MODEL_TYPE, weights_path, event_type
from utils.backend import keras, set_gelu, K
from utils.tokenizers import Tokenizer
from utils.models import build_transformer_model
from utils.optimizers import Adam, extend_with_piecewise_linear_lr
from utils.snippets import sequence_padding, DataGenerator
from utils.snippets import open
from keras.layers import Dropout, Dense

set_gelu('tanh')  # 切换gelu版本

# albert tiny

maxlen = 128
batch_size = 256
bert_layers = 3
config_path = BASE_CONFIG_NAME
checkpoint_path = BASE_CKPT_NAME
dict_path = '{}/vocab.txt'.format(BASE_MODEL_DIR)

def load_data(filename):
    """加载数据
    单条格式：(文本1, 文本2, 标签id)
    """
    D = []
    with open(filename, encoding = 'utf-8') as f:
        for l in f:
            text1, text2, label = l.strip().split('\t')
            D.append((text1, text2, int(label)))
    return D

# 加载数据集
train_data = load_data(train_file_path_Similarity)
valid_data = load_data(val_file_path_Similarity)
test_data = load_data(test_file_path_Similarity)

# 建立分词器
tokenizer = Tokenizer(dict_path, do_lower_case = True)

class data_generator(DataGenerator):
    """数据生成器
    """

    def __iter__(self, random = False):
        batch_token_ids, batch_segment_ids, batch_labels = [], [], []
        for is_end, (text1, text2, label) in self.sample(random):
            token_ids, segment_ids = tokenizer.encode(
                text1, text2, maxlen = maxlen
            )
            batch_token_ids.append(token_ids)
            batch_segment_ids.append(segment_ids)
            batch_labels.append([label])
            if len(batch_token_ids) == self.batch_size or is_end:
                batch_token_ids = sequence_padding(batch_token_ids)
                batch_segment_ids = sequence_padding(batch_segment_ids)
                batch_labels = sequence_padding(batch_labels)
                yield [batch_token_ids, batch_segment_ids], batch_labels
                batch_token_ids, batch_segment_ids, batch_labels = [], [], []

# 加载预训练模型
bert = build_transformer_model(
    config_path = config_path,
    checkpoint_path = checkpoint_path,
    model = MODEL_TYPE,
    with_pool = True,
    return_keras_model = False,
)
# output_layer = 'Transformer-FeedForward-Norm'
# output = bert.get_layer(output_layer).get_output_at(bert_layers)

output = Dropout(rate = 0.1)(bert.model.output)
output = Dense(
    units = 2, activation = 'softmax', kernel_initializer = bert.initializer
)(output)

model = keras.models.Model(bert.model.input, output)
model.summary()

AdamLR = extend_with_piecewise_linear_lr(Adam, name = 'AdamLR')

model.compile(
    loss = 'sparse_categorical_crossentropy',
    optimizer = AdamLR(lr = 1e-4, lr_schedule = {
        1000: 1,
        2000: 0.1
    }),
    metrics = ['accuracy'],
)

# 转换数据集
train_generator = data_generator(train_data, batch_size)
valid_generator = data_generator(valid_data, batch_size)
test_generator = data_generator(test_data, batch_size)

def evaluate(data):
    total, right = 0., 0.
    for x_true, y_true in data:
        y_pred = model.predict(x_true).argmax(axis = 1)
        y_true = y_true[:, 0]
        total += len(y_true)
        right += (y_true == y_pred).sum()
    return right / total

class Evaluator(keras.callbacks.Callback):
    """评估与保存
    """

    def __init__(self):
        self.best_val_acc = 0.

    def on_epoch_end(self, epoch, logs = None):
        save_file_path = "{}/{}_similarity_max_acc.h5".format(weights_path, MODEL_TYPE)
        val_acc = evaluate(valid_generator)
        if val_acc > self.best_val_acc:
            self.best_val_acc = val_acc
            model.save_weights(save_file_path)
        test_acc = evaluate(test_generator)
        print(
            u'val_acc: %.5f, best_val_acc: %.5f, test_acc: %.5f\n' %
            (val_acc, self.best_val_acc, test_acc)
        )

if __name__ == '__main__':

    save_file_path = "{}/{}_similarity_min_loss.h5".format(weights_path, MODEL_TYPE)

    evaluator = Evaluator()
    early_stopping = EarlyStopping(monitor = 'val_loss', patience = 10, verbose = 1)  # 提前结束
    save_model = ModelCheckpoint(save_file_path, monitor = 'val_loss', verbose = 0, save_best_only = True,
                                 mode = 'min')

    for i, item in enumerate(train_generator):
        print("\nbatch_token_ids shape:", item[0][0].shape)
        print("batch_segment_ids shape:", item[0][1].shape)
        print("batch_labels shape:", item[1].shape)
        if i == 4:
            break

    # batch_token_ids shape: (256, 65)
    # batch_segment_ids shape: (256, 65)
    # batch_labels shape: (256, 1)

    # batch_token_ids shape: (256, 71)
    # batch_segment_ids shape: (256, 71)
    # batch_labels shape: (256, 1)

    # batch_token_ids shape: (256, 69)
    # batch_segment_ids shape: (256, 69)
    # batch_labels shape: (256, 1)

    model.fit(
        train_generator.forfit(),
        steps_per_epoch = len(train_generator),
        validation_data = valid_generator.forfit(),
        validation_steps = len(valid_generator),
        epochs = 100,
        callbacks = [evaluator, early_stopping, save_model]
    )

    model.load_weights(save_file_path)
    print(u'final min loss test acc: %05f\n' % (evaluate(test_generator)))

    save_file_path = "{}/{}_similarity_max_acc.h5".format(weights_path, MODEL_TYPE)
    model.load_weights(save_file_path)
    print(u'final max acc test acc: %05f\n' % (evaluate(test_generator)))

else:
    save_file_path = "{}/{}_similarity_min_loss.h5".format(weights_path, MODEL_TYPE)
    model.load_weights(save_file_path)


import sys
from s3 import S3
from botocore.exceptions import ClientError

from logger import get_pystacks_log
log = get_pystacks_log()

class s3tasks:

    def __init__(self, session):
        self.s3 = S3(session)

    def checkForFile(self, bucket, key):
        try:
            response = self.s3.checkForFile(
                Bucket=bucket,
                Delimiter=key,
            )
            if len(response.Contents) > 0:
                print("found")
            else:
                print("not found")
        except ClientError as err:
            logging.warn(err)
            sys.exit(1)

    def checkForFileV2(self, bucket, filename):
        try:
            response = self.s3.list_objects_v2(
                Bucket=bucket,
                Prefix=filename
            )
            if len(response.Contents) > 0:
                return True
            else:
                return False
        except ClientError as err:
            logging.warn(err)
            sys.exit(1)

    def uploadFile(self, filename, bucket, key):
        try:
            self.s3.uploadFile(filename=filename, bucket=bucket, key=key)
        except ClientError as err:
            logging.warn(err)
            sys.exit(1)

    def bucket_exists(self, bucket_name):
        return bucket_name in [bucket['Name'] for bucket in self.s3.list_buckets()['Buckets']]

    def get_bucket(self, bucket_name, region, create=False):
        """Returns Bucket object.  Create the bucket if it doesn't exist using create=True"""

        # ClientError if problem with the bucket name, If this is slow consdider only checking AFTER failure
        if create and not self.bucket_exists(bucket_name):
            self.s3.create_bucket(bucket_name, region)
        return self.s3.resource.Bucket(bucket_name)

def upload_object(bucket, dest_filename, binary_contents, kms_key=None, session=None):
    """ if bucket is string type, a session must be provided.  """

    specs = {
        'Key': dest_filename,
        'Body': binary_contents,
    }

    if kms_key:
        specs['ServerSideEncryption'] = 'aws:kms'
        specs['SSEKMSKeyId'] = kms_key

    if isinstance(bucket, basestring):
        if not session:
            msg = "A session must be provided with bucket as a string.  Alternatively use s3tasks.get_bucket"
            log.error(msg)
            raise Exception(msg)

        s3t = s3tasks(session)
        bucket = s3t.get_bucket(bucket, session.region_name)

    try:
        res = bucket.put_object(**specs)
    except ClientError as ex:
        log.error("Bucket '{}' may not exist.  Consider using 'sanitise_bucket_name' and 'get_bucket'".format(bucket.name))
        raise ex

    return res

def sanitise_bucket_name(seed):
    """Sanitize the seed according to the s3 rules [here]
        (http://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html)
        """

    # if 3 > len(seed)
    # if len(seed) > 63
    # if consecutive periods
    # if dashes adjacent to periods
    # if ends with dash

    return seed.lower().replace('_', '-')


import os
import numpy as np

from shutil import copyfile

def organize_data(train_data_ratio,intermedia_data_root,output_data_root):

	input_roi_dir = os.path.join(intermedia_data_root,"roi")

	number_of_series = len(os.listdir(input_roi_dir))
	series_indices = np.arange(number_of_series)

	#ransomly chosse train/test data
	np.random.shuffle(series_indices)

	train_series_number = int(number_of_series * train_data_ratio)
	input_plain_dir = os.path.join(intermedia_data_root,"plain")

	#orgainize train data
	print("Copy train-roi-data:")
	output_train_roi_dir = os.path.join(output_data_root,"train","roi")
	copy_to_one_dir(series_indices[0:train_series_number],input_roi_dir,output_train_roi_dir)

	print("Copy train-plain-data:")
	output_train_plain_dir = os.path.join(output_data_root,"train","plain")
	copy_to_one_dir(series_indices[0:train_series_number],input_plain_dir,output_train_plain_dir)

	#orginize test_data
	print("Copy test-roi-data:")
	output_test_roi_dir = os.path.join(output_data_root,"test","roi")
	copy_to_one_dir(series_indices[train_series_number:],input_roi_dir,output_test_roi_dir)

	print("Copy test-plain-data:")
	output_test_plain_dir = os.path.join(output_data_root,"test","plain")
	copy_to_one_dir(series_indices[train_series_number:],input_plain_dir,output_test_plain_dir)

def path_check(path):
	'''Create a new path if not existed
	'''
	if not os.path.exists(path):
		os.makedirs(path)

def copy_to_one_dir(series_indices,src_dir,dst_dir):
	'''Copy all these files in sepicified sub-series directories to one directory.

	If path dst_dir not existed, a new path will be created.
	'''
	path_check(dst_dir)

	print("Copy files from ",src_dir,"\n to ",dst_dir)
	print("Related series: ",series_indices)

	series_names = os.listdir(src_dir)
	img_id = 0
	for index in series_indices:
		tmp_series_dir = os.path.join(src_dir,series_names[index])

		for filename in os.listdir(tmp_series_dir):
			src_fp = os.path.join(tmp_series_dir,filename)

			#use img_id as the uniform filename and align it to the right
			#with a fill char '0'
			dst_fp = os.path.join(dst_dir,"{:0>4}".format(img_id)+ ".npy")
			copyfile(src_fp,dst_fp)

			img_id +=1
'''
intermedia_data_root = "/media/charles/study/Nodules"
output_data_root = "/media/charles/study/Organized_Nodules"

organize_data(0.9,intermedia_data_root,output_data_root)
'''


#!/usr/bin/env python
# vim: set filencoding=utf8
"""
Mercurial Shared SSH Login Validator

@author: Mike Crute (mcrute@gmail.com)
@organization: SoftGroup Hosting
@date: February 23, 2011
"""

import re
import os
import repolib

from mercurial import demandimport; demandimport.enable()
from mercurial import dispatch

def parse_path():
    cmd = os.environ.get('SSH_ORIGINAL_COMMAND', '')
    path = re.match("hg -R (\S+) serve --stdio", cmd)

    if path:
        return path.groups()[0]

    return None

def main(argv):
    log = repolib.get_logger('validate-login')
    user = argv[-1]
    path = parse_path()

    log.info("Validating login for %r on %r", user, path)

    if path:
        repo = repolib.Repository(path)
        repo.repo_path = os.getcwd()
    else:
        log.error("Invalid command")
        return 1

    if not repo.exists:
        log.error("Repo %s does not exist", repo.full_path)
        return 1

    try:
        repo.load_from_hgrc()
    except IOError:
        log.error("Could not read repo config")
        return 1

    if not repo.can_be_read_by(user):
        log.error("You can not read this repository")
        return 1

    os.environ['SSH_HG_USER'] = user
    os.environ['SSH_HG_REPO'] = repo.full_path

    log.info("All checks passed, serving.")
    dispatch.dispatch(['-R', path, 'serve', '--stdio'])

if __name__ == "__main__":
    import sys
    sys.exit(main(sys.argv[1:]))


'''
ÁRVORE BINÁRIA DE BUSCA - EXCLUSÃO
O NÓ A SER APAGADO TEM APENAS UM FILHO

* O Nó tem apenas duas conexões: com seu pai e com seu único filho
* Deseja-se "cortar" o nó dessa sequência conectando o pai dele diretamente ao filho dele

'''
class No:
    def __init__(self, valor):
        self.valor = valor
        self.esquerda = None
        self.direita = None

class ArvoreBuscaBinaria:
    def __init__(self):
        self.raiz = None
        self.ligacoes = []

    def insere(self, valor):
        novo = No(valor)
        if self.raiz == None:
            self.raiz = novo
        else:
            atual = self.raiz
            while True:
                pai = atual
                if valor < atual.valor:
                    atual = atual.esquerda
                    if atual == None:
                        pai.esquerda = novo
                        self.ligacoes.append(str(pai.valor) + '->' + str(valor))
                        return
                else:
                    atual = atual.direita
                    if atual == None:
                        pai.direita = novo
                        self.ligacoes.append(str(pai.valor) + '->' + str(valor))
                        return

    def pesquisa(self, valor):
        atual = self.raiz
        while valor != atual.valor:
            if valor < atual.valor:
                atual = atual.esquerda
            else:
                atual = atual.direita
            if atual == None:
                return None
        return valor

    # raiz, esquerda, direita
    def pre_ordem(self, no):
        if no != None:
            print(no.valor)
            self.pre_ordem(no.esquerda)
            self.pre_ordem(no.direita)

    # esquerda, raiz, direita
    def em_ordem(self, no):
        if no != None:
            self.em_ordem(no.esquerda)
            print(no.valor)
            self.em_ordem(no.direita)

    # esquerda, direita, raiz
    def pos_ordem(self, no):
        if no != None:
            self.pos_ordem(no.esquerda)
            self.pos_ordem(no.direita)
            print(no.valor)

    def excluir(self, valor):
        if self.raiz == None:
            print('Árvore está vazia')
            return

        atual = self.raiz
        pai = self.raiz
        e_esquerda = True

        while valor != atual.valor:
            pai = atual
            if valor < atual.valor:
                atual = atual.esquerda
                e_esquerda = True
            else:
                atual = atual.direita
                e_esquerda = False
            if atual == None:
                return False

        # Verifica se o nó (atual) é uma folha
        if atual.esquerda == None and atual.direita == None:
            if atual == self.raiz:
                self.raiz = None
            elif e_esquerda:
                pai.esquerda = None
            else:
                pai.direita = None

        # nó com filho único
        # O nó a ser apagado não possui filho na direita (nó a ser APAGADO é o ESQUERDO)
        elif atual.direita == None:
            if atual == self.raiz:
               self.raiz = atual.esquerda
            elif e_esquerda:
                pai.esquerda = atual.esquerda
            else:
                pai.direita = atual.esquerda
        # O nó a ser apagado não possui filho na esquerda (nó a ser APAGADO é o DIREITO)
        elif atual.esquerda == None:
            if atual == self.raiz:
                self.raiz = atual.direita
            elif e_esquerda:
                pai.esquerda = atual.direita
            else:
                pai.direita = atual.direita

arvore = ArvoreBuscaBinaria()
arvore.insere(53)
arvore.insere(30)
arvore.insere(14)
arvore.insere(39)
arvore.insere(9)
arvore.insere(23)
arvore.insere(34)
arvore.insere(49)
arvore.insere(72)
arvore.insere(61)
arvore.insere(84)
arvore.insere(79)

arvore.excluir(84)
arvore.excluir(9)
arvore.excluir(14)
print('a')


import json

from dojo.models import Finding

class WhispersParser(object):
    """
    Identify hardcoded secrets in static structured text
    """

    SEVERITY_MAP = {
        "BLOCKER": "Critical",
        "CRITICAL": "High",
        "MAJOR": "Medium",
        "MINOR": "Low",
        "INFO": "Info",
    }

    @staticmethod
    def _mask(text, n_plain=4):
        length = len(text)
        if length <= n_plain:
            n_plain = 0

        return text[:n_plain] + ("*" * (length - n_plain))

    def get_scan_types(self):
        return ["Whispers Scan"]

    def get_label_for_scan_types(self, scan_type):
        return "Whispers Scan"

    def get_description_for_scan_types(self, scan_type):
        return "Whispers report file can be imported in JSON format (option --json)."

    def get_findings(self, file, test):
        tree = json.load(file)
        findings = []
        for vuln in tree:
            summary = (
                f'Hardcoded {vuln.get("message")} "{vuln.get("key")}" '
                f'in {vuln.get("file")}:{vuln.get("line")}'
            )
            description = f'{summary} `{self._mask(vuln.get("value"))}`'
            findings.append(
                Finding(
                    title=summary,
                    description=description,
                    mitigation=(
                        "Replace hardcoded secret with a placeholder (ie: ENV-VAR). "
                        "Invalidate the leaked secret and generate a new one. "
                        "Supply the new secret through a placeholder to avoid disclosing "
                        "sensitive information in code."
                    ),
                    references="https://cwe.mitre.org/data/definitions/798.html",
                    cwe=798,
                    severity=self.SEVERITY_MAP.get(
                        vuln.get("severity"), "Info"
                    ),
                    file_path=vuln.get("file"),
                    line=int(vuln.get("line")),
                    vuln_id_from_tool=vuln.get("message"),
                    static_finding=True,
                    dynamic_finding=False,
                    test=test,
                )
            )

        return findings


from bleak import BleakScanner

# Returns scanned sensors
async def scanSensors(addressNames, ready_to_stream, auto_run):
    # List of sensors for program and converted version
    sensors = [[], ([""]*len(addressNames))]

    # Scanning for Dots
    print("Scanning for Dots!")
    ble_devices = await BleakScanner.discover()

    # DEBUG Uncomment for scan results
    # print(ble_devices)

    # Checking for Dots in ble_devices
    for ble_device in ble_devices:

        # Fixes issue with a ble device with no name
        if(ble_device.name is None):
            continue

        if "xsens" in ble_device.name.lower():
            sensors[0].append(ble_device)

            # Placing the sensors in the correct order post scan. Only for GUI
            for ii in range(len(addressNames)):
                if(str(ble_device) == str(addressNames[ii][0])):
                    sensors[1][(addressNames[ii][1]) - 1] = addressNames[ii][2]
    if len(sensors[0]) == len(addressNames):
        ready_to_stream["stream_status"] = True
        print("All sensors found!")
    else:
        auto_run["scan"] = True
    print("Finished scanning!")
    return sensors


from flask import Flask, redirect, request
import json

app = Flask(__name__)

api_online = {}
api_online['online'] = True

with open('files/status/api.json', 'w') as outfile:
    json.dump(api_online, outfile)

@app.route('/')
def home():
    return redirect('http://10.0.1.124/accses-denied/api')

@app.route('/kill')
def quit():
    func = request.environ.get('werkzeug.server.shutdown')
    func()
    print("Web Server Crashed")
    api_online['online'] = False
    with open('status/api.json', 'w') as outfile:
        json.dump(api_online, outfile)
    return "Shut API Server Down"

@app.errorhandler(404)
def err_404():
    return redirect('http://10.0.1.124/accses-denied/api')

print("Booting API Server")
app.run(debug=True, host="0.0.0.0", port=81)
print("API Server Chrashed")

api_online['online'] = False

with open('status/api.json', 'w') as outfile:
    json.dump(api_online, outfile)