import requests
from config import API_KEY
# from flask import jsonify

def get_data(query):

    # query = "Bread"
    print(query)

    url = "https://api.nal.usda.gov/fdc/v1/foods/search?"

    query_url = f"{url}api_key={API_KEY}&query={query}"

    x = requests.get(query_url)
    # x = requests.get('https://api.nal.usda.gov/fdc/v1/foods/search?api_key=DEMO_KEY&query=Bread')
    # x = requests.get('https://api.nal.usda.gov/fdc/v1/foods/list?api_key=DEMO_KEY')
    # x = requests.get('https://api.nal.usda.gov/fdc/v1/json-spec?api_key=DEMO_KEY')

    foods_list = x.json()

    print(foods_list)

    return foods_list


import json

MANDATORY_SETTINGS = ('FunctionName', 'Handler', 'Role', 'Runtime')

def load_settings(filepath):
    with open(filepath, 'r') as f:
        settings = json.loads(f.read())
    for key in MANDATORY_SETTINGS:
        try:
            assert key in settings
        except AssertionError:
            raise KeyError('{} not found in the settings JSON.'.format(key))
    return settings


# 출력층 설계하기
# 기계학습 문제는 분류와 회구로 나뉩는다 분류는 데이타가 어느 클래스 속 하느냐 문제

# 항등 함수와 소프트맥스 함수 구현하기
# 항등 함수는 입력을 대로 출력
# 입력과 출력이 항상 같다는 뜻
# 소프트맥스 함수
# 소프트맥스의 출력은 모든 입력 신호로부터 화살표를 받는다
# 출력층의 각 뉴런이 모든 입력 신호에서 영향을 받기 대문이다

import numpy as np
a = np.array([0.3, 2.9, 4.0])

def softmax(a):
    exp_a = np.exp(a) # 지수 함수
    sum_exp_a = np.sum(exp_a) # 지수 함수의 합
    y = exp_a / sum_exp_a
    return y

print(softmax(a)) # [0.01821127 0.24519181 0.73659691]


from requests import request
from json import loads

emails = [
    ("hello.world", "failure"),
    ("hello.world@company.com", "success"),
    ("hello.world@", "failure"),
    ("hello.world@.com", "failure"),
    ("hello.world@company.gov.in", "success"),
    ("hello.world@company.edu", "success")
    ]

for email in emails:
    url = f"https://api.eva.pingutil.com/email?email={email[0]}"

    # returns a Request object
    response = request("GET", url)

    # getting json string from response object
    json_text = response.text

    # deserilising the json string
    rsp = loads(json_text)

    # actual status getting from the response
    actual_status = rsp['status']

    if actual_status == email[1]:
        print("PASS")
    else:
        print("FAIL")


import pymongo
from datetime import datetime, timedelta
from db import DBHandler
from tinkof import FetchPrices, quotation2float
import asyncio
import os

connString = os.getenv('MONGODB_CONNSTRING')
dbName = os.getenv('MONGODB_DATABASE')

client = pymongo.MongoClient(connString)
db2 = client[dbName]

def CalcProfit(bond):
    daysLeft = (bond['maturity_date']-datetime.now()).days
    marketDiff = bond['nominal']-bond['market_price']-bond['aci_value']
    if bond['market_price'] == 0:
        return (0, 0)

    if 'coupons' not in bond.keys():
        return (marketDiff/bond['market_price']/daysLeft*365, 0)

    if bond['coupons'] == []:
        return (marketDiff/bond['market_price']/daysLeft*365, 0)

    if bond['coupons'][-1]['coupon_type'] == 'COUPON_TYPE_CONSTANT':
        c = sum([i['pay_one_bond'] for i in bond['coupons']])/bond['market_price']/daysLeft*365
        m = marketDiff/bond['market_price']/daysLeft*365
        return (c+m, c)

    elif bond['coupons'][-1]['coupon_type'] == 'COUPON_TYPE_VARIABLE':
        c = bond['coupons'][-1]['pay_one_bond']*len(bond['coupons'])/bond['market_price']/daysLeft*365
        m = marketDiff/bond['market_price']/daysLeft*365
        return (c+m, c)

    else:
        couponSum = sum([i['pay_one_bond'] for i in bond['coupons']])
        fp = (couponSum+marketDiff)/bond['market_price']/daysLeft*365
        cp = couponSum/bond['market_price']/daysLeft*365
        return (cp, fp)

async def main():
    db = DBHandler()
    bonds = [i for i in (await db.GetAllBonds())['bonds']]
    bond_figis = [i['figi'] for i in bonds]
    prices = await FetchPrices(bond_figis, 't.CbJu2z3-n0MfU9Dbtbk9kxlGvnml00A7upkA6WvXDjQcpwmtqQyyJ4z00oS17cMfVFO_twNOZ5OcdHvMLyHbwg')
    price_dict = {}
    for i in prices:
        price_dict[i.figi] = i.price
    for bond in bonds:
        bond['market_price'] = quotation2float(
            price_dict[bond['figi']])*bond['nominal']/100
        fp, cp = CalcProfit(bond)
        bond['profitability'] = fp
        bond['coupon_profitability'] = cp
    for bond in bonds:
        db2['bonds'].replace_one({"_id": bond["_id"]}, bond, upsert=True)

asyncio.run(main())


import random
import string
import input_processing as w

def gen_LD(word, dist):
    word_l = list(word)

    if dist > len(word):
        new_word = ''
        for _ in range(dist):
            new_letter = random.choice(string.ascii_lowercase)
            while new_letter in word_l:
                new_letter = random.choice(string.ascii_lowercase)
            new_word += new_letter
        return new_word

    used_indexes = []
    changes = [0] * len(word)
    new_word = list(word)

    for _ in range(dist):
        choice = random.randint(0, 1)

        new_pos = random.randint(0, len(new_word) - 1)
        if choice == 0: #sostituzione
            while changes[new_pos] == 1 or new_pos in used_indexes:
                new_pos = random.randint(0, len(new_word) - 1)
            changes[new_pos] = 1
            if new_pos not in used_indexes:
                used_indexes.append(new_pos)
            old_letter = new_word[new_pos]
            new_letter = random.choice(string.ascii_lowercase)
            while new_letter == old_letter or new_letter in word_l:
                new_letter = random.choice(string.ascii_lowercase)
            new_word[new_pos] = new_letter

        if choice == 1: #inserimento
            changes.insert(new_pos, 1)
            word_l.insert(new_pos,0)
            if new_pos not in used_indexes:
                used_indexes.append(new_pos)
            new_letter = random.choice(string.ascii_lowercase)
            while new_letter in word_l:
                new_letter = random.choice(string.ascii_lowercase)
            new_word.insert(new_pos, new_letter)

        if choice == 2: #cancellazione
            while changes[new_pos] == 1 or new_pos in used_indexes:
                new_pos = random.randint(0, len(new_word) - 1)
            del new_word[new_pos]
            del changes[new_pos]

        #print(choice)
        #print(changes)
        #print(used_indexes)

    new_word = ''.join(new_word)
    return new_word

#testo lo script
prec = 0
test = 500
for i in range(test):
    w_len = random.randint(4, 8)
    word = ''
    for _ in range(w_len):
        word += random.choice(string.ascii_lowercase)

    dist = random.randint(1, len(word) + 3)
    new_word = gen_LD(word,dist)
    if w.LD(word,new_word) == dist:
        prec += 1
    else:
        print(word + " - " + str(dist))
        print(new_word)
    print(i)

print("precision: " + str((prec/test)*100) + "%")

'''
print(word + " - " + str(len(word)))
print(new_word)
print("needed dist: " + str(dist))
print("obtained: " + str(w.LD(word,new_word)))
print(w.LD(word,new_word) == dist)
'''


import os
from subprocess import run
from chimera_app.config import BIN_PATH

POWER_TOOL_PATH = os.path.join(BIN_PATH, 'power-tool')

DEVICE_DB = {
    # Aya Neo 2021
    'AMD Ryzen 5 4500U with Radeon Graphics' : {
        'min_tdp'   : 5,
        'max_tdp'   : 28,
        'max_boost' : 2,
    },
    # Aya Neo Air
    'AMD Ryzen 5 5560U with Radeon Graphics' : {
        'min_tdp'   : 3,
        'max_tdp'   : 28,
        'max_boost' : 2,
    },
    # ONEXPLAYER Mini AMD
    'AMD Ryzen 7 5800U with Radeon Graphics' : {
        'min_tdp'   : 5,
        'max_tdp'   : 33,
        'max_boost' : 5,
    },
    # Aya Neo 2
    # AOKZOE A1
    # Aya Neo Air Plus
    'AMD Ryzen 7 6800U with Radeon Graphics' : {
        'min_tdp'   : 5,
        'max_tdp'   : 33,
        'max_boost' : 5,
    },
}

def get_device_info():
    results = run([ 'bash', '-c', 'lscpu | grep "Model name" | cut -d: -f2' ], capture_output=True, text=True)
    key = results.stdout.strip()

    if not key in DEVICE_DB:
        return None

    return DEVICE_DB[key]

def get_tdp():
    device = get_device_info()
    if not device:
        return

    results = run([ 'sudo', '--non-interactive', POWER_TOOL_PATH, 'get-tdp' ], capture_output=True, text=True)
    lines = results.stdout.splitlines()

    tdp = None
    if len(lines) > 0:
        tdp = results.stdout.splitlines()[0]

    if not tdp:
        return

    return int(tdp)

def set_tdp(new_tdp):
    if type(new_tdp) != int:
        return

    device = get_device_info()
    if not device:
        return

    if new_tdp < device['min_tdp'] or new_tdp > device['max_tdp']:
        return

    run([ 'sudo', '--non-interactive', POWER_TOOL_PATH, 'set-tdp', str(new_tdp), str(device['max_boost']) ])


__author__ = "Hao Qin"
__email__ = "awww797877@gmail.com"

import numpy as np
from data import dummy
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

BATCH_SIZE = 32

def get_y(poses, index_pose1, index_pose2, num_batch_images, max_transform, max_rotation):
    transform = np.square(
        poses[index_pose1:index_pose1 + num_batch_images, 0] - poses[index_pose2:index_pose2 + num_batch_images,
                                                                   0])
    transform += np.square(
        poses[index_pose1:index_pose1 + num_batch_images, 1] - poses[index_pose2:index_pose2 + num_batch_images,
                                                                   1])
    # print(transform.shape)
    transform = np.sqrt(transform)
    rotation = np.absolute(
        poses[index_pose1:index_pose1 + num_batch_images, 2] - poses[index_pose2:index_pose2 + num_batch_images,
                                                                   2])
    y = 20 * (transform + rotation) / (max_transform + max_rotation) # 1 / (1 + np.exp(- (transform + rotation)))
    y = (1 / (1 + np.exp(y)) - 0.5) * 2
    return y

if __name__ == '__main__':
    poses = dummy.load_poses('/home/hao/others/data/CNN_SLAM/2012-04-06-11-15-29_part1_floor2.gt.laser.poses')

    max_pose = np.amax(poses, axis=0)
    min_pose = np.amin(poses, axis=0)
    max_transform = np.sqrt(np.square(max_pose[0] - min_pose[0]) + np.square(max_pose[1] - min_pose[1]))
    max_rotation = max_pose[2] - min_pose[2]
    print(max_transform, max_rotation)

    z = np.array([])
    for offset1 in range(0, BATCH_SIZE, BATCH_SIZE):
        temp = np.array([])
        for offset2 in range(0, len(poses[0:1216]), BATCH_SIZE):
            t = get_y(poses, offset1, offset2, BATCH_SIZE, max_transform, max_rotation)
            print(t)
            temp = np.append(temp, t)
        z = np.vstack((z, temp)) if z.size else temp


from __future__ import unicode_literals

import frappe
from frappe.utils.make_random import how_many, can_make
from frappe.desk import query_report
from frappe.utils import cstr
from erpnext_demo import settings

def run_manufacturing(current_date):
	from erpnext.projects.doctype.time_log.time_log import NotSubmittedError, OverlapError

	ppt = frappe.get_doc("Production Planning Tool", "Production Planning Tool")
	ppt.company = settings.company
	ppt.use_multi_level_bom = 1
	ppt.purchase_request_for_warehouse = "Stores - {}".format(settings.company_abbr)
	ppt.run_method("get_open_sales_orders")
	ppt.run_method("get_items")
	ppt.run_method("raise_production_orders")
	ppt.run_method("raise_material_requests")
	frappe.db.commit()

	# submit production orders
	for pro in frappe.db.get_values("Production Order", {"docstatus": 0}, "name"):
		b = frappe.get_doc("Production Order", pro[0])
		b.wip_warehouse = "Work in Progress - WP"
		b.submit()
		frappe.db.commit()

	# submit material requests
	for pro in frappe.db.get_values("Material Request", {"docstatus": 0}, "name"):
		b = frappe.get_doc("Material Request", pro[0])
		b.submit()
		frappe.db.commit()

	# stores -> wip
	if can_make("Stock Entry for WIP"):
		for pro in query_report.run("Open Production Orders")["result"][:how_many("Stock Entry for WIP")]:
			make_stock_entry_from_pro(pro[0], "Material Transfer for Manufacture", current_date)

	# wip -> fg
	if can_make("Stock Entry for FG"):
		for pro in query_report.run("Production Orders in Progress")["result"][:how_many("Stock Entry for FG")]:
			make_stock_entry_from_pro(pro[0], "Manufacture", current_date)

	# submit time logs
	for time_log in frappe.get_all("Time Log", ["name"], {"docstatus": 0,
		"production_order": ("!=", ""), "to_time": ("<", current_date)}):
		time_log = frappe.get_doc("Time Log", time_log.name)
		try:
			time_log.submit()
			frappe.db.commit()
		except OverlapError:
			pass

def make_stock_entry_from_pro(pro_id, purpose, current_date):
	from erpnext.manufacturing.doctype.production_order.production_order import make_stock_entry
	from erpnext.stock.stock_ledger import NegativeStockError
	from erpnext.stock.doctype.stock_entry.stock_entry import IncorrectValuationRateError, \
		DuplicateEntryForProductionOrderError, OperationsNotCompleteError

	try:
		st = frappe.get_doc(make_stock_entry(pro_id, purpose))
		st.posting_date = current_date
		st.fiscal_year = cstr(current_date.year)
		for d in st.get("items"):
			d.cost_center = "Main - " + settings.company_abbr
		st.insert()
		frappe.db.commit()
		st.submit()
		frappe.db.commit()
	except (NegativeStockError, IncorrectValuationRateError, DuplicateEntryForProductionOrderError,
		OperationsNotCompleteError):
		frappe.db.rollback()


import pickle
import unittest

from cupy.cuda import cutensor

@unittest.skipUnless(cutensor.available, 'cuTensor is unavailable')
class TestExceptionPicklable(unittest.TestCase):

    def test(self):
        e1 = cutensor.CuTensorError(1)
        e2 = pickle.loads(pickle.dumps(e1))
        assert e1.args == e2.args
        assert str(e1) == str(e2)


# import the packages
from getAnimePics.items import GetanimepicsItem
import datetime
import scrapy
from scrapy.exceptions import CloseSpider

pageNumber = 0

class CoverSpider(scrapy.Spider):
    name = "gelbooruSearch"
    allowed_domains = ['gelbooru.com']
    start_urls = ["https://gelbooru.com/index.php?page=post&s=list&tags=okita_souji_%28fate%29+solo+rating%3asafe"]
    custom_settings = {
        "ITEM_PIPELINES": {'scrapy.pipelines.images.ImagesPipeline': 1},
        "IMAGES_STORE": 'D:\Cute anime girls\Okita'
    }

    def request(self, url, callback):
        request = scrapy.Request(url=url, callback=callback)
        request.cookies['resize-original'] = "1"
        request.cookies['resize-notification'] = "1"
        request.headers['User-Agent'] = ('Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, '
            'like Gecko) Chrome/45.0.2454.85 Safari/537.36')
        return request

    def parse(self, response):
        #Starts on first page and increases the page number once every image is downloaded
        imagesPerPage = 42
        #For every thumbnail, extract the image link from under the span a attribute
        url = response.css("span a::attr(href)").extract()
        urlList = zip(url)
        #Gets every url in the list and then makes them into a string
        for item in urlList:
            imageLink = ''.join(item)
            imageLink = "https:" + imageLink
            #Once the link is converted into a string, it uses it as the URL and calls parse_images
            yield self.request(imageLink, callback=self.parse_images)

        nextUrl = response.url.split("&pid=")
        global pageNumber
        pageNumber += 1
        nextPage = nextUrl[0] + "&pid=" + str(pageNumber * imagesPerPage)
        if len(urlList) == 42:
            yield self.request(nextPage, callback=self.parse)

    def parse_images(self, response):
        #this is real bad
        imageUrl = response.css("img::attr(src)").extract_first()
        realUrl = ''.join(imageUrl)
        #if the following pic is not in original quality
        if "samples" in realUrl:
            #hard code found the xpath on gelbooru seems to be the same for every image though
            list = response.xpath("/html/body/div[4]/div[5]/script[2]/text()").extract()
            tempString = ''.join(list)
            #extracts part of the hd image url, not sample
            tempString = tempString[(tempString.index('\'img\'')):(tempString.index('\', \'base_dir'))]
            image = tempString.split('\'img\':\'')
            image = ''.join(image[1])
            #replaces the imageurl if it is a sample
            realUrl = realUrl.replace("samples", "images")
            realUrl = realUrl.split("sample_")
            realUrl = realUrl[0] + image

        #Converts the URL into a string and places it in images_urls, which is used to download the image
        yield GetanimepicsItem(image_urls=[realUrl])


import json
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score, f1_score

def train_model(features, target, train_params):
    mt = train_params.model_type
    if train_params.model_type == 'RandomForestClassifier':
        model = RandomForestClassifier(n_estimators=train_params.model_params.RandomForestClassifier.n_estimators,
                                       random_state=train_params.model_params.RandomForestClassifier.random_state,
                                       min_samples_leaf=train_params.model_params.RandomForestClassifier.min_samples_leaf,
                                       max_depth=train_params.model_params.RandomForestClassifier.forest_max_depth)
    elif train_params.model_type == 'DecisionTreeClassifier':
        model = DecisionTreeClassifier(random_state=train_params.model_params.DecisionTreeClassifier.random_state,
                                       min_samples_leaf=train_params.model_params.DecisionTreeClassifier.min_samples_leaf,
                                       max_depth=train_params.model_params.DecisionTreeClassifier.max_depth)
    model.fit(features, target)
    return model

def predict_model(model, features):
    predicts = model.predict(features)
    return predicts

def evaluate_model(predicts, target):
    f1 = round(f1_score(target, predicts), 3)
    accuracy = round(accuracy_score(target, predicts), 3)
    return {
        'accuracy': accuracy,
        'f1_score': f1
    }

def write_csv_data(data, path):
    data = pd.DataFrame(data, columns=['y'])
    data.to_csv(path, index=False)

def write_json_data(data, path):
    with open(path, 'w') as file:
        json.dump(data, file)


from core.models import Habit, DailyRecord, User
from rest_framework import serializers

class DailyRecordSerializer(serializers.ModelSerializer):
    class Meta:
        model = DailyRecord
        fields = ("date", "note",)

class HabitSerializer(serializers.ModelSerializer):
    daily_records = DailyRecordSerializer(many=True, read_only=False)

    class Meta:
        model = Habit
        fields = (
            "pk",
            "title",
            "goal",
            "created_at",
            "daily_records",
        )

class UserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = ['url', 'username', 'email', 'is_staff']

# class RecordSerializer(serializers.ModelSerializer):
#     habit = HabitSerializer(read_only=)

#     class Meta:
#         model = DailyRecord
#         fields = ("habit", "date", "note",)


import numpy as np
from tqdm import tqdm
import random
import matplotlib.pyplot as plt
from simplenn.layers import *

class Model:

    def __init__(self, layers, verbose=False):
        self.layers = layers

    def set_verbosity(self, verbose):
        for l in self.layers:
            if hasattr(l, 'verbose'):
                l.verbose=verbose

    def __call__(self, x:np.array):
        for l in self.layers:
            x = l(x)
        return x

    def backward(self, grad_output):
        for l in reversed(self.layers):
            grad_output = l.backward(grad_output)

def run_test(layers, epochs, num_data, batch, lr, pbar=True):
    def make_model(layers, ref=False):
        model = []
        for i, num_output in enumerate(layers):
            if i == 0:
                continue
            num_input = layers[i-1]
            layer = Linear(num_input, num_output)
            if ref:
                layer.w = np.random.uniform(-10, 10, size=(num_input, num_output))
                layer.b = np.random.uniform(-20, 20, size=(num_output, ))
            model.append(layer)

            if i + 1 < len(layers):
                model.append(Sigmoid())
        return Model(model)

    ref_model = make_model(layers, True)
    model = make_model(layers, False)
    model.set_verbosity(False)
    num_batch = (num_data + batch - 1) // batch

    fig, ax = plt.subplots()
    (ln,) = ax.plot([-1], [0], animated=True)
    ax.set_xlim(0, epochs)
    plt.show(block=False)
    bg = fig.canvas.copy_from_bbox(fig.bbox)
    fig.canvas.blit(fig.bbox)

    X = [np.random.normal(size=(1, layers[0], )) for _ in range(num_data)]
    Y = [ref_model(x) for x in X]
    data_idx = list(range(len(X)))
    loss_history = []
    with tqdm(total=epochs, disable=not pbar) as t:
        for e in range(epochs):
            random.shuffle(data_idx)
            grads = []
            for i in range(0, len(data_idx), batch):
                x_batch = [X[j] for j in data_idx[i:(i+batch)]]
                y_ref_batch = [Y[j] for j in data_idx[i:(i+batch)]]
                batch_size = len(x_batch)
                x_batch = np.concatenate(x_batch).reshape(batch_size, -1)
                y_ref_batch = np.concatenate(y_ref_batch).reshape(batch_size, -1)
                y_pred_batch = model(x_batch)
                y_pred_batch = y_pred_batch.reshape(y_ref_batch.shape)
                grad = y_pred_batch - y_ref_batch
                grads.append(grad.copy())
                d = lr * grad
                model.backward(d)
            t.update()
            grads = np.concatenate(grads)
            loss = np.mean(grad**2)
            loss_history.append(loss)
            fig.canvas.restore_region(bg)
            ax.set_ylim(0, max(loss_history))
            ln.set_xdata(np.arange(len(loss_history)))
            ln.set_ydata(loss_history)
            ax.draw_artist(ln)
            fig.canvas.blit(fig.bbox)
            fig.canvas.flush_events()
            t.set_description(f'Epoch: {e:6} Loss: {float(loss):8.2f}')

run_test([256, 128, 128, 64, 64, 32, 16, 4, 1], 100, 10000, 100, 0.001)


import time
import numpy as np
import scipy.io as sio
from datetime import datetime
import cPickle as pickle
import sys

# local modules
import wtahash as wh
import cluster
import utils

class Evaluation:
    ''' Class for evaluating the speed and storing the rankings of a dataset
        using WTAHash.
    '''
    def __init__(self, dataset_path):
        self.dataset = dataset_path
        self.log = ""

    def run(self):
        # Get the options
        s = "Enter the number of classes from the dataset to be used "\
            "(0 for all).\n"
        self.n_classes = input(s)
        s = "Choose an option:\n"\
            "- [0] Calculate WTA Hash.\n"\
            "- [1] Load a stored WTA Hash.\n"\
            "- [2] Calculate WTA Hash and store it.\n"
        opt_load = input(s)
        k = 16
        w = 2
        n = 1200

        LOAD_HASH = 1
        STORE_HASH = 2

        s = "Enter the ranking size you want to use (from 1 to 100).\n"
        ranking_size = input(s)
        if ranking_size > 100:
            ranking_size = 100
        # Percentage of the data that will be used for training, the rest is
        # testing
        train_perc = 80
        self.log = ""

        starting_time = datetime.now()
        self.log += "Starting time {0}\n".format(starting_time)
        self.log += "Using the first {0} classes of the dataset\n".format(
            self.n_classes
        )

        # Training
        #-----------------------------------------------------------------------
        train_data, train_labels = self.read_descriptors(train_perc, "training")
        if opt_load == LOAD_HASH:
            hash_filename = "results/wtahash_{0}.obj".format(self.n_classes)
            print("Loading hash from file {0}...".format(hash_filename))
            wta_hash = pickle.load(open(hash_filename, "rb"))
        else:
            wta_hash = self.create_hash(train_data, n, k, w)
            if opt_load == STORE_HASH:
                self.store_hash(wta_hash)

        # Testing
        #-----------------------------------------------------------------------
        test_data, test_labels = self.read_descriptors(train_perc, "testing")
        ranking_size = min((len(train_data), ranking_size))
        rankings = self.get_rankings(test_data, wta_hash, ranking_size)
        self.store_rankings(rankings)
        self.store_labels(train_labels, test_labels)

        # Dot products
        #-----------------------------------------------------------------------
        sorted_prods, prods = self.dot_products(
            train_data, test_data, rankings
        )
        self.store_products(sorted_prods, prods)

        # Precision metrics
        #-----------------------------------------------------------------------
        # Generate relevance rankings
        self.metrics(rankings, train_labels, test_labels, sorted_prods)
        end_time = datetime.now()
        self.log += "Ending time {0}\n".format(end_time)
        # Write times in a text file
        log_filename = "results/log_{0}.txt".format(self.n_classes)
        with open(log_filename, "w") as f:
            f.write(self.log)

    def read_descriptors(self, training_percentage, set_name):
        ###                  Load training information matrix                ###
        ###------------------------------------------------------------------###

        print ("Reading {0} instances ...".format(set_name))
        start = time.time()
        data, labels = cluster.load_classes(
            training_percentage, self.dataset, set_name, self.n_classes
        )
        end = time.time()
        self.log += "{0} matrix of shape {1}\n".format(set_name, data.shape)
        elapsed_time = utils.humanize_time(end - start)
        s = "Elapsed time reading the {0} files: {1}".format(
            set_name, elapsed_time
        )
        self.log += s + "\n"
        print (s)

        return data, labels

    def create_hash(self, train_data, n, k, w):
        ###                        Use WTAHash on it                         ###
        ###------------------------------------------------------------------###

        print ("Starting to generate hash table ...")
        start = time.time()
        wta_hash = wh.WTAHash(train_data, n, k, w)
        end = time.time()
        elapsed_time = utils.humanize_time(end - start)
        s = "Elapsed time on generation of hash table: {0}".format(elapsed_time)
        self.log += s + "\n"
        print (s)

        return wta_hash

    def get_rankings(self, test_data, wta_hash, ranking_size):
        ###                   Get the rankings for the test set              ###
        ###------------------------------------------------------------------###

        print ("Generating ranking matrix for the test set ...")
        start = time.time()
        rankings = wta_hash.best_classifiers(test_data, ranking_size)
        end = time.time()
        elapsed_time = utils.humanize_time(end - start)
        s = "Elapsed time generating ranking matrix: {0}".format(elapsed_time)
        self.log += s + "\n"
        print (s)

        return rankings

    def dot_products(self, train_data, test_data, rankings):
        ''' Calculates the dot product for each element in the test set with
            every element of the train set. Returns a matrix with two columns
            matrix. The first column is the index of the object in the train set
            and the second column is the value of the dot product of that object
            with the test object with index equal to the number of the row. Then
            the number of rows is the number of objects in the test set.

        Args:
            train_data (np matrix of floats): Each row is the vector of an
                object in the train set.
            test_data (np matrix of floats): Each row is the vector of an object
                in the test set.
            rankings (list of lists int): The ranking created for each object
                in the test set.

        Returns:
            list of list of tuples {
                e.g.:
                         0             ranking_size
                         |                  |
                0 - [[(21, 0.91), (3, 0.87), ...],
                     [(10, 0.83), (0, 0.72), ...],
                             ...
   len(test_data) -  [                      ... ]]

                int: Index of the object in the train set that should be ranked
                    in the i-th position where i is the number of the row,
                float: The value of the dot product between the object in the
                    train set and the object in the test set in the i-th
                    position where i is the number of the row.
            },
            numpy array of arrays of floats: Dot products where the [i-th, j-th]
                element is the product between the i-th object of the testing
                set and the j-th object of the training set.
        '''
        ###                Calculate dot product on the variables            ###
        ###------------------------------------------------------------------###

        print ("Calculating dot products on the rankings ...")
        start = time.time()
        # products is the matrix that stores the dot product of each testing
        # vector with each training vector
        sorted_prods = []
        products = []
        ranking_size = len(rankings[0])
        step = (len(test_data) * 5) / 100
        train_norm = [utils.normalize(train_vec) for train_vec in train_data]
        train_norm = np.array(train_norm)
        for i in range(len(test_data)):
            # y is the current testing vector
            y = test_data[i]
            y_norm = utils.normalize(y)
            current_tuples = []
            products.append([])
            for j in range(len(train_data)):
                # vector is the training object ranked in the current position
                vector_norm = train_norm[j]
                prod = np.dot(y_norm, vector_norm)
                if j < ranking_size:
                    products[i].append(prod)
                current_tuples.append( (j, prod) )
            current_tuples.sort(key=lambda x: x[1], reverse=True)
            sorted_prods.append(current_tuples[:ranking_size])
            if i % step == 0:
                percentage = (i * 100) / len(test_data)
                print (
                    "Vector number {0} of {1} ({2}%) multiplied".format(
                        i, len(test_data), percentage
                    )
                )
        end = time.time()
        elapsed_time = utils.humanize_time(end - start)
        s = "Elapsed time calculating dot products: {0}".format(elapsed_time)
        self.log += s + "\n"
        print (s)
        return sorted_prods, np.array(products)

    def metrics(self, rankings, train_labels, test_labels, sorted_prods):
        ###    Calculates classification and products set and position mAP   ###
        ###------------------------------------------------------------------###

        print("Starting to calculate metrics ...")
        start = time.time()
        rel_ranks = []
        for i in range(len(rankings)):
            rel_ranks.append(
                utils.relevance_ranking(
                    rankings[i], train_labels, test_labels[i]
                )
            )

        # Classification mAP
        #-----------------------------------------------------------------------
        class_ap = [utils.class_ap(rel_rk) for rel_rk in rel_ranks]
        class_ap_filename = "results/class_avg_precs_{0}.txt".format(
            self.n_classes
        )
        utils.write_list(class_ap, class_ap_filename)
        class_map = np.mean(class_ap)
        self.log += "ranking size = {0}".format(len(rankings[0])) + "\n"
        s = "classification mean average precision = {0}".format(class_map)
        self.log += s + "\n"
        print(s)

        # Dot products average precision
        #-----------------------------------------------------------------------
        # Set
        set_prec = []
        for i in range(len(rankings)):
            indices = [prods[0] for prods in sorted_prods[i]]
            precision = utils.prod_set_prec(indices, rankings[i])
            set_prec.append(precision)
        set_ap_filename = "results/set_avg_precs_{0}.txt".format(
            self.n_classes
        )
        utils.write_list(set_prec, set_ap_filename)
        set_map = np.mean(set_prec)
        s = "set mean average precision = {0}".format(set_map)
        self.log += s + "\n"
        print(s)

        # Position
        pos_prec = []
        for i in range(len(rankings)):
            indices = [prods[0] for prods in sorted_prods[i]]
            precision = utils.prod_pos_prec(indices, rankings[i])
            pos_prec.append(precision)
        pos_ap_filename = "results/pos_avg_precs_{0}.txt".format(
            self.n_classes
        )
        utils.write_list(pos_prec, pos_ap_filename)
        pos_map = np.mean(pos_prec)
        s = "position mean average precision = {0}".format(pos_map)
        self.log += s + "\n"
        print(s)

        end = time.time()
        elapsed_time = utils.humanize_time(end - start)
        s = "Elapsed time calculating metrics: {0}".format(elapsed_time)
        self.log += s + "\n"
        print (s)
    ############################################################################
    ####                    Functions for storing values                    ####
    ############################################################################
    def store_hash(self, wta_hash):
        ## Store the hash in a binary file
        print("Storing the hash in a file ...")
        start = time.time()
        hash_filename = "results/wtahash_{0}.obj".format(self.n_classes)
        pickle.dump(wta_hash, open(hash_filename, "wb"), protocol=2)
        end = time.time()
        elapsed_time = utils.humanize_time(end - start)
        s = "Elapsed time storing the hash {0}".format(elapsed_time)
        self.log += s + "\n"
        print(s)

    def store_rankings(self, rankings):
        ## Store the rankings in a mat file
        print("Storing rankings in a mat file ...")
        start = time.time()
        rankings_filename = "results/rankings_{0}.mat".format(self.n_classes)
        data = {"stored": rankings}
        sio.savemat(rankings_filename, data)
        end = time.time()
        s = "Elapsed time storing the rankings {0} secs.".format(end - start)
        self.log += s + "\n"
        print(s)

    def store_labels(self, train_labels, test_labels):
        ## Store the labels in a text file
        print("Storing the labels in text files...")
        start = time.time()
        train_labels_fn = "results/train_labels_{0}.txt".format(self.n_classes)
        test_labels_fn = "results/test_labels_{0}.txt".format(self.n_classes)
        utils.write_list(train_labels, train_labels_fn)
        utils.write_list(test_labels, test_labels_fn)
        end = time.time()
        s = "Elapsed time storing the labels {0} secs.".format(end - start)
        self.log += s + "\n"
        print(s)

    def store_products(self, sorted_prods, products):
        # Write products in a mat file
        print("Storing products in a mat file ...")
        start = time.time()
        prods_filename = "results/products_{0}.mat".format(self.n_classes)
        sio.savemat(prods_filename, {"stored": products})
        # e.g. elem = [(1, 0.94), (12, 0.83), (4, 0.6), ...]
        #   indices = [1, 12, 4, ...]
        indices = [ [prod[0] for prod in row] for row in sorted_prods]
        ids_filename = "results/indices_{0}.mat".format(self.n_classes)
        sio.savemat(ids_filename, {"stored": indices})
        end = time.time()
        print("Elapsed time storing the products {0} secs.".format(end - start))


from __future__ import print_function
import time
import numpy as np
import matplotlib.pyplot as plt
from stats232a.classifiers.fc_net import *
from stats232a.data_utils import *
from stats232a.gradient_check import eval_numerical_gradient, eval_numerical_gradient_array
from stats232a.solver import Solver
from stats232a.layers import *

def rel_error(x, y):
    """ returns relative error """
    return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))

# load CIFAR-10 dataset
# from stats232a.data_utils import get_CIFAR10_data
# data = get_CIFAR10_data()
# for k, v in list(data.items()):
#     print(('%s: ' % k, v.shape))

class Test(object):
    def __init__(self):
        # Load the (preprocessed) MNIST data.
        # The second dimension of images indicated the number of channel. For black and white images in MNIST, channel=1.
        # Load the raw MNIST data
        num_training = 59000
        num_validation = 1000
        num_test = 1000
        subtract_mean = True

        mnist_dir = '/home/parallels/PycharmProjects/Courses/232A/project2/stats232a/datasets'
        X_train, y_train = load_mnist(dataset="training", path=mnist_dir, size=num_training + num_validation)
        X_test, y_test = load_mnist(dataset="testing", path=mnist_dir, size=num_test)

        # Subsample the data
        mask = list(range(num_training, num_training + num_validation))
        X_val = X_train[mask]
        y_val = y_train[mask]
        mask = list(range(num_training))
        X_train = X_train[mask]
        y_train = y_train[mask]

        # Normalize the data: subtract the mean image
        if subtract_mean:
            mean_image = np.mean(X_train, axis=0)
            X_train -= mean_image
            X_val -= mean_image
            X_test -= mean_image

        # Transpose so that channels come first
        X_train = X_train.transpose(0, 3, 1, 2).copy()
        X_val = X_val.transpose(0, 3, 1, 2).copy()
        X_test = X_test.transpose(0, 3, 1, 2).copy()

        # Package data into a dictionary
        self.data = {
            'X_train': X_train, 'y_train': y_train,
            'X_val': X_val, 'y_val': y_val,
            'X_test': X_test, 'y_test': y_test,
        }

    #
    # Test1 - forward
    def test1(self):
        num_inputs = 2
        input_shape = (4, 5, 6)
        output_dim = 3

        input_size = num_inputs * np.prod(input_shape)
        weight_size = output_dim * np.prod(input_shape)

        x = np.linspace(-0.1, 0.5, num=input_size).reshape(num_inputs, *input_shape)
        w = np.linspace(-0.2, 0.3, num=weight_size).reshape(np.prod(input_shape), output_dim)
        b = np.linspace(-0.3, 0.1, num=output_dim)

        out, _ = fc_forward(x, w, b)
        correct_out = np.array([[1.49834967, 1.70660132, 1.91485297],
                                [3.25553199, 3.5141327, 3.77273342]])

        # Compare your output with ours. The error should be around 1e-9.
        print('Testing fc_forward function:')
        print('difference: ', rel_error(out, correct_out))

    #
    # Test2 - backprop
    def test2(self):
        np.random.seed(231)
        x = np.random.randn(10, 2, 3)
        w = np.random.randn(6, 5)
        b = np.random.randn(5)
        dout = np.random.randn(10, 5)

        dx_num = eval_numerical_gradient_array(lambda x: fc_forward(x, w, b)[0], x, dout)
        dw_num = eval_numerical_gradient_array(lambda w: fc_forward(x, w, b)[0], w, dout)
        db_num = eval_numerical_gradient_array(lambda b: fc_forward(x, w, b)[0], b, dout)

        _, cache = fc_forward(x, w, b)
        dx, dw, db = fc_backward(dout, cache)

        # The error should be around 1e-10
        print('Testing fc_backward function:')
        print('dx error: ', rel_error(dx_num, dx))
        print('dw error: ', rel_error(dw_num, dw))
        print('db error: ', rel_error(db_num, db))

    #
    # Test the relu_forward function
    def test3(self):
        x = np.linspace(-0.5, 0.5, num=12).reshape(3, 4)

        out, _ = relu_forward(x)
        correct_out = np.array([[0., 0., 0., 0., ],
                                [0., 0., 0.04545455, 0.13636364, ],
                                [0.22727273, 0.31818182, 0.40909091, 0.5, ]])

        # Compare your output with ours. The error should be around 5e-8
        print('Testing relu_forward function:')
        print('difference: ', rel_error(out, correct_out))

    #
    # ReLU layer: backward
    def test4(self):
        np.random.seed(231)
        x = np.random.randn(10, 10)
        dout = np.random.randn(*x.shape)

        dx_num = eval_numerical_gradient_array(lambda x: relu_forward(x)[0], x, dout)

        _, cache = relu_forward(x)
        dx = relu_backward(dout, cache)

        # The error should be around 3e-12
        print('Testing relu_backward function:')
        print('dx error: ', rel_error(dx_num, dx))

    #
    # Test: "Sandwich" layers
    def test5(self):

        from stats232a.layer_utils import fc_relu_forward, fc_relu_backward
        np.random.seed(231)
        x = np.random.randn(2, 3, 4)
        w = np.random.randn(12, 10)
        b = np.random.randn(10)
        dout = np.random.randn(2, 10)

        out, cache = fc_relu_forward(x, w, b)
        dx, dw, db = fc_relu_backward(dout, cache)

        dx_num = eval_numerical_gradient_array(lambda x: fc_relu_forward(x, w, b)[0], x, dout)
        dw_num = eval_numerical_gradient_array(lambda w: fc_relu_forward(x, w, b)[0], w, dout)
        db_num = eval_numerical_gradient_array(lambda b: fc_relu_forward(x, w, b)[0], b, dout)

        print('Testing affine_relu_forward:')
        print('dx error: ', rel_error(dx_num, dx))
        print('dw error: ', rel_error(dw_num, dw))
        print('db error: ', rel_error(db_num, db))

    #
    # Loss layers: Softmax
    def test6(self):
        np.random.seed(231)
        num_classes, num_inputs = 10, 50
        x = 0.001 * np.random.randn(num_inputs, num_classes)
        y = np.random.randint(num_classes, size=num_inputs)

        dx_num = eval_numerical_gradient(lambda x: softmax_loss(x, y)[0], x, verbose=False)
        loss, dx = softmax_loss(x, y)

        # Test softmax_loss function. Loss should be 2.3 and dx error should be 1e-8
        print('\nTesting softmax_loss:')
        print('loss: ', loss)
        print('dx error: ', rel_error(dx_num, dx))

    #
    # Test: Two-layer network
    def test7(self):
        np.random.seed(231)
        N, D, H, C = 3, 5, 50, 7
        X = np.random.randn(N, D)
        y = np.random.randint(C, size=N)

        std = 1e-3
        model = TwoLayerNet(input_dim=D, hidden_dim=H, num_classes=C)

        print('Testing test-time forward pass ... ')
        model.params['W1'] = np.linspace(-0.7, 0.3, num=D * H).reshape(D, H)
        model.params['b1'] = np.linspace(-0.1, 0.9, num=H)
        model.params['W2'] = np.linspace(-0.3, 0.4, num=H * C).reshape(H, C)
        model.params['b2'] = np.linspace(-0.9, 0.1, num=C)
        X = np.linspace(-5.5, 4.5, num=N * D).reshape(D, N).T
        scores = model.loss(X)
        correct_scores = np.asarray(
            [[11.53165108, 12.2917344, 13.05181771, 13.81190102, 14.57198434, 15.33206765, 16.09215096],
             [12.05769098, 12.74614105, 13.43459113, 14.1230412, 14.81149128, 15.49994135, 16.18839143],
             [12.58373087, 13.20054771, 13.81736455, 14.43418138, 15.05099822, 15.66781506, 16.2846319]])
        scores_diff = np.abs(scores - correct_scores).sum()
        assert scores_diff < 1e-6, 'Problem with test-time forward pass'

        print('Testing training loss (no regularization)')
        y = np.asarray([0, 5, 1])
        loss, grads = model.loss(X, y)
        correct_loss = 3.4702243556
        assert abs(loss - correct_loss) < 1e-10, 'Problem with training-time loss'

        model.reg = 1.0
        loss, grads = model.loss(X, y)
        correct_loss = 26.5948426952
        assert abs(loss - correct_loss) < 1e-10, 'Problem with regularization loss'

        for reg in [0.0, 0.7]:
            print('Running numeric gradient check with reg = ', reg)
            model.reg = 0
            loss, grads = model.loss(X, y)

            for name in sorted(grads):
                f = lambda _: model.loss(X, y)[0]
                grad_num = eval_numerical_gradient(f, model.params[name], verbose=False)
                print('%s relative error: %.2e' % (name, rel_error(grad_num, grads[name])))

    #
    # Test: Solver
    def test8(self):
        model = TwoLayerNet()

        ##############################################################################
        # TODO: Use a Solver instance to train a TwoLayerNet that achieves at least  #
        # 96% accuracy on the validation set.                                        #
        ##############################################################################
        solver = Solver(model, self.data,
                        update_rule='sgd',
                        optim_config={
                            'learning_rate': 1e-3,
                        },
                        lr_decay=0.95,
                        num_epochs=9, batch_size=200,
                        print_every=100)
        solver.train()
        ##############################################################################
        #                             END OF YOUR CODE                               #
        ##############################################################################

        # Run this cell to visualize training loss and train / val accuracy

        plt.subplot(2, 1, 1)
        plt.title('Training loss')
        plt.plot(solver.loss_history, 'o')
        plt.xlabel('Iteration')

        plt.subplot(2, 1, 2)
        plt.title('Accuracy')
        plt.plot(solver.train_acc_history, '-o', label='train')
        plt.plot(solver.val_acc_history, '-o', label='val')
        plt.plot([0.5] * len(solver.val_acc_history), 'k--')
        plt.xlabel('Epoch')
        plt.legend(loc='lower right')
        plt.gcf().set_size_inches(15, 12)
        plt.show()

    #
    # Test: Multiple Layers
    def test9(self):
        # TODO: Use a three-layer Net to overfit 50 training examples.
        # You will need to tweak the learning rate and initialization scale

        num_train = 50
        small_data = {
            'X_train': self.data['X_train'][:num_train],
            'y_train': self.data['y_train'][:num_train],
            'X_val': self.data['X_val'],
            'y_val': self.data['y_val'],
        }

        weight_scale = 1e-1
        learning_rate = 5e-4
        model = FullyConnectedNet([100, 100],
                                  weight_scale=weight_scale, dtype=np.float64)

        solver = Solver(model, small_data,
                        print_every=10, num_epochs=20, batch_size=25,
                        update_rule='sgd',
                        optim_config={
                            'learning_rate': learning_rate,
                        }
                        )
        solver.train()

        plt.plot(solver.loss_history, 'o')
        plt.title('Training loss history')
        plt.xlabel('Iteration')
        plt.ylabel('Training loss')
        plt.show()

    def test10(self):
        # TODO: Use a five-layer Net to overfit 50 training examples.
        # You will have to adjust the learning rate and weight initialization,

        num_train = 50
        small_data = {
            'X_train': self.data['X_train'][:num_train],
            'y_train': self.data['y_train'][:num_train],
            'X_val': self.data['X_val'],
            'y_val': self.data['y_val'],
        }

        learning_rate = 5e-4
        weight_scale = 1e-1
        model = FullyConnectedNet([100, 100, 100, 100],
                                  weight_scale=weight_scale, dtype=np.float64)
        solver = Solver(model, small_data,
                        print_every=10, num_epochs=20, batch_size=25,
                        update_rule='sgd',
                        optim_config={
                            'learning_rate': learning_rate,
                        }
                        )
        solver.train()

        plt.plot(solver.loss_history, 'o')
        plt.title('Training loss history')
        plt.xlabel('Iteration')
        plt.ylabel('Training loss')
        plt.show()

    #
    # SGD+Momentum test
    def test11(self):
        from stats232a.optim import sgd_momentum

        N, D = 4, 5
        w = np.linspace(-0.4, 0.6, num=N * D).reshape(N, D)
        dw = np.linspace(-0.6, 0.4, num=N * D).reshape(N, D)
        v = np.linspace(0.6, 0.9, num=N * D).reshape(N, D)

        config = {'learning_rate': 1e-3, 'velocity': v}
        next_w, _ = sgd_momentum(w, dw, config=config)

        expected_next_w = np.asarray([
            [0.1406, 0.20738947, 0.27417895, 0.34096842, 0.40775789],
            [0.47454737, 0.54133684, 0.60812632, 0.67491579, 0.74170526],
            [0.80849474, 0.87528421, 0.94207368, 1.00886316, 1.07565263],
            [1.14244211, 1.20923158, 1.27602105, 1.34281053, 1.4096]])
        expected_velocity = np.asarray([
            [0.5406, 0.55475789, 0.56891579, 0.58307368, 0.59723158],
            [0.61138947, 0.62554737, 0.63970526, 0.65386316, 0.66802105],
            [0.68217895, 0.69633684, 0.71049474, 0.72465263, 0.73881053],
            [0.75296842, 0.76712632, 0.78128421, 0.79544211, 0.8096]])

        print('next_w error: ', rel_error(next_w, expected_next_w))
        print('velocity error: ', rel_error(expected_velocity, config['velocity']))

    def test12(self):
        # six-layer network with both SGD and SGD+momentum. You should see the SGD+momentum update rule converge faster.
        num_train = 4000
        small_data = {
            'X_train': self.data['X_train'][:num_train],
            'y_train': self.data['y_train'][:num_train],
            'X_val': self.data['X_val'],
            'y_val': self.data['y_val'],
        }

        solvers = {}

        for update_rule in ['sgd', 'sgd_momentum']:
            print('running with ', update_rule)
            model = FullyConnectedNet([100, 100, 100, 100, 100], weight_scale=5e-2)

            solver = Solver(model, small_data,
                            num_epochs=5, batch_size=100,
                            update_rule=update_rule,
                            optim_config={
                                'learning_rate': 1e-2,
                            },
                            verbose=True)
            solvers[update_rule] = solver
            solver.train()
            print()

        plt.subplot(3, 1, 1)
        plt.title('Training loss')
        plt.xlabel('Iteration')

        plt.subplot(3, 1, 2)
        plt.title('Training accuracy')
        plt.xlabel('Epoch')

        plt.subplot(3, 1, 3)
        plt.title('Validation accuracy')
        plt.xlabel('Epoch')

        for update_rule, solver in list(solvers.items()):
            plt.subplot(3, 1, 1)
            plt.plot(solver.loss_history, 'o', label=update_rule)

            plt.subplot(3, 1, 2)
            plt.plot(solver.train_acc_history, '-o', label=update_rule)

            plt.subplot(3, 1, 3)
            plt.plot(solver.val_acc_history, '-o', label=update_rule)

        for i in [1, 2, 3]:
            plt.subplot(3, 1, i)
            plt.legend(loc='upper center', ncol=4)
        plt.gcf().set_size_inches(15, 15)
        plt.show()

    #
    # RMSProp
    def test13(self):
        # Test RMSProp implementation; you should see errors less than 1e-7
        from stats232a.optim import rmsprop

        N, D = 4, 5
        w = np.linspace(-0.4, 0.6, num=N * D).reshape(N, D)
        dw = np.linspace(-0.6, 0.4, num=N * D).reshape(N, D)
        cache = np.linspace(0.6, 0.9, num=N * D).reshape(N, D)

        config = {'learning_rate': 1e-2, 'cache': cache}
        next_w, _ = rmsprop(w, dw, config=config)

        expected_next_w = np.asarray([
            [-0.39223849, -0.34037513, -0.28849239, -0.23659121, -0.18467247],
            [-0.132737, -0.08078555, -0.02881884, 0.02316247, 0.07515774],
            [0.12716641, 0.17918792, 0.23122175, 0.28326742, 0.33532447],
            [0.38739248, 0.43947102, 0.49155973, 0.54365823, 0.59576619]])
        expected_cache = np.asarray([
            [0.5976, 0.6126277, 0.6277108, 0.64284931, 0.65804321],
            [0.67329252, 0.68859723, 0.70395734, 0.71937285, 0.73484377],
            [0.75037008, 0.7659518, 0.78158892, 0.79728144, 0.81302936],
            [0.82883269, 0.84469141, 0.86060554, 0.87657507, 0.8926]])

        print('next_w error: ', rel_error(expected_next_w, next_w))
        print('cache error: ', rel_error(expected_cache, config['cache']))

    #
    # Adam
    def test14(self):
        # Test Adam implementation; you should see errors around 1e-7 or less
        from stats232a.optim import adam

        N, D = 4, 5
        w = np.linspace(-0.4, 0.6, num=N * D).reshape(N, D)
        dw = np.linspace(-0.6, 0.4, num=N * D).reshape(N, D)
        m = np.linspace(0.6, 0.9, num=N * D).reshape(N, D)
        v = np.linspace(0.7, 0.5, num=N * D).reshape(N, D)

        config = {'learning_rate': 1e-2, 'm': m, 'v': v, 't': 5}
        next_w, _ = adam(w, dw, config=config)

        expected_next_w = np.asarray([
            [-0.40094747, -0.34836187, -0.29577703, -0.24319299, -0.19060977],
            [-0.1380274, -0.08544591, -0.03286534, 0.01971428, 0.0722929],
            [0.1248705, 0.17744702, 0.23002243, 0.28259667, 0.33516969],
            [0.38774145, 0.44031188, 0.49288093, 0.54544852, 0.59801459]])
        expected_v = np.asarray([
            [0.69966, 0.68908382, 0.67851319, 0.66794809, 0.65738853, ],
            [0.64683452, 0.63628604, 0.6257431, 0.61520571, 0.60467385, ],
            [0.59414753, 0.58362676, 0.57311152, 0.56260183, 0.55209767, ],
            [0.54159906, 0.53110598, 0.52061845, 0.51013645, 0.49966, ]])
        expected_m = np.asarray([
            [0.48, 0.49947368, 0.51894737, 0.53842105, 0.55789474],
            [0.57736842, 0.59684211, 0.61631579, 0.63578947, 0.65526316],
            [0.67473684, 0.69421053, 0.71368421, 0.73315789, 0.75263158],
            [0.77210526, 0.79157895, 0.81105263, 0.83052632, 0.85]])

        print('next_w error: ', rel_error(expected_next_w, next_w))
        print('v error: ', rel_error(expected_v, config['v']))
        print('m error: ', rel_error(expected_m, config['m']))

    #
    # Train with RMSProp and Adam
    def test15(self):
        num_train = 4000
        small_data = {
            'X_train': self.data['X_train'][:num_train],
            'y_train': self.data['y_train'][:num_train],
            'X_val': self.data['X_val'],
            'y_val': self.data['y_val'],
        }
        solvers = {}
        learning_rates = {'rmsprop': 1e-4, 'adam': 1e-3}
        for update_rule in ['adam', 'rmsprop']:
            print('running with ', update_rule)
            model = FullyConnectedNet([100, 100, 100, 100, 100], weight_scale=5e-2)

            solver = Solver(model, small_data,
                            num_epochs=5, batch_size=100,
                            update_rule=update_rule,
                            optim_config={
                                'learning_rate': learning_rates[update_rule]
                            },
                            verbose=True)
            solvers[update_rule] = solver
            solver.train()
            print()

        plt.subplot(3, 1, 1)
        plt.title('Training loss')
        plt.xlabel('Iteration')

        plt.subplot(3, 1, 2)
        plt.title('Training accuracy')
        plt.xlabel('Epoch')

        plt.subplot(3, 1, 3)
        plt.title('Validation accuracy')
        plt.xlabel('Epoch')

        for update_rule, solver in list(solvers.items()):
            plt.subplot(3, 1, 1)
            plt.plot(solver.loss_history, 'o', label=update_rule)

            plt.subplot(3, 1, 2)
            plt.plot(solver.train_acc_history, '-o', label=update_rule)

            plt.subplot(3, 1, 3)
            plt.plot(solver.val_acc_history, '-o', label=update_rule)

        for i in [1, 2, 3]:
            plt.subplot(3, 1, i)
            plt.legend(loc='upper center', ncol=4)
        plt.gcf().set_size_inches(15, 15)
        plt.show()

    #
    # Batch Normalization: Forward in training mode
    def test16(self):
        # Check the training-time forward pass by checking means and variances
        # of features both before and after batch normalization

        # Simulate the forward pass for a two-layer network
        N, D1, D2, D3 = 200, 50, 60, 3
        X = np.random.randn(N, D1)
        W1 = np.random.randn(D1, D2)
        W2 = np.random.randn(D2, D3)
        a = np.maximum(0, X.dot(W1)).dot(W2)

        print('Before batch normalization:')
        print('  means: ', a.mean(axis=0))
        print('  stds: ', a.std(axis=0))

        # Means should be close to zero and stds close to one
        print('After batch normalization (gamma=1, beta=0)')
        a_norm, _ = batchnorm_forward(a, np.ones(D3), np.zeros(D3), {'mode': 'train'})
        print('  mean: ', a_norm.mean(axis=0))
        print('  std: ', a_norm.std(axis=0))

        # Now means should be close to beta and stds close to gamma
        gamma = np.asarray([1.0, 2.0, 3.0])
        beta = np.asarray([11.0, 12.0, 13.0])
        a_norm, _ = batchnorm_forward(a, gamma, beta, {'mode': 'train'})
        print('After batch normalization (nontrivial gamma, beta)')
        print('  means: ', a_norm.mean(axis=0))
        print('  stds: ', a_norm.std(axis=0))

    #
    # Batch Normalization: Forward in test mode
    def test17(self):
        # Check the test-time forward pass by running the training-time
        # forward pass many times to warm up the running averages, and then
        # checking the means and variances of activations after a test-time
        # forward pass.

        N, D1, D2, D3 = 200, 50, 60, 3
        W1 = np.random.randn(D1, D2)
        W2 = np.random.randn(D2, D3)

        bn_param = {'mode': 'train'}
        gamma = np.ones(D3)
        beta = np.zeros(D3)
        for t in range(100):
            X = np.random.randn(N, D1)
            a = np.maximum(0, X.dot(W1)).dot(W2)
            batchnorm_forward(a, gamma, beta, bn_param)
        bn_param['mode'] = 'test'
        X = np.random.randn(N, D1)
        a = np.maximum(0, X.dot(W1)).dot(W2)
        a_norm, _ = batchnorm_forward(a, gamma, beta, bn_param)

        # Means should be close to zero and stds close to one, but will be
        # noisier than training-time forward passes.
        print('After batch normalization (test-time):')
        print('  means: ', a_norm.mean(axis=0))
        print('  stds: ', a_norm.std(axis=0))

    #
    # Batch Normalization: backward
    def test18(self):
        # Gradient check batchnorm backward pass

        N, D = 2, 3
        x = 5 * np.random.randn(N, D) + 12
        gamma = np.random.randn(D)
        beta = np.random.randn(D)
        dout = np.random.randn(N, D)

        bn_param = {'mode': 'train'}
        fx = lambda x: batchnorm_forward(x, gamma, beta, bn_param)[0]
        fg = lambda a: batchnorm_forward(x, gamma, beta, bn_param)[0]
        fb = lambda b: batchnorm_forward(x, gamma, beta, bn_param)[0]

        dx_num = eval_numerical_gradient_array(fx, x, dout)
        da_num = eval_numerical_gradient_array(fg, gamma, dout)
        db_num = eval_numerical_gradient_array(fb, beta, dout)
        dmean_num = np.sum(dx_num, axis=0) * N
        _, cache = batchnorm_forward(x, gamma, beta, bn_param)
        dx, dgamma, dbeta = batchnorm_backward(dout, cache)

        # print('dmean error: ', rel_error(dmean_num, dmean), dmean, dmean_num)
        print('dx error: ', rel_error(dx_num, dx))
        print('dgamma error: ', rel_error(da_num, dgamma))
        print('dbeta error: ', rel_error(db_num, dbeta))

    #
    # Batch Normalization: Fully Connected Net
    def test19(self):
        N, D, H1, H2, C = 2, 15, 20, 30, 10
        X = np.random.randn(N, D)
        y = np.random.randint(C, size=(N,))

        for reg in [0, 3.14]:
            print('Running check with reg = ', reg)
            model = FullyConnectedNet([H1, H2], input_dim=D, num_classes=C,
                                      reg=reg, weight_scale=5e-2, dtype=np.float64,
                                      use_batchnorm=True)
            loss, grads = model.loss(X, y)
            print('Initial loss: ', loss)
            grads.pop('dx')
            for name in sorted(grads):
                f = lambda _: model.loss(X, y)[0]
                grad_num = eval_numerical_gradient(f, model.params[name], verbose=False, h=1e-5)
                print('%s relative error: %.2e' % (name, rel_error(grad_num, grads[name])))
            if reg == 0:
                print()

    #
    # CIFAR-10
    def finalTest(self):
        """
        Load the CIFAR-10 dataset from disk and perform preprocessing to prepare
        it for classifiers. These are the same steps as we used for the SVM, but
        condensed to a single function.
        """
        # Load the raw CIFAR-10 data
        num_training = 49000
        num_validation = 1000
        num_test = 1000
        subtract_mean = True

        cifar10_dir = '/home/parallels/PycharmProjects/Courses/232A/project2/stats232a/datasets/cifar-10-batches-py'
        X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)

        # Subsample the data
        mask = list(range(num_training, num_training + num_validation))
        X_val = X_train[mask]
        y_val = y_train[mask]
        mask = list(range(num_training))
        X_train = X_train[mask]
        y_train = y_train[mask]
        mask = list(range(num_test))
        X_test = X_test[mask]
        y_test = y_test[mask]

        # Normalize the data: subtract the mean image
        if subtract_mean:
            mean_image = np.mean(X_train, axis=0)
            X_train -= mean_image
            X_val -= mean_image
            X_test -= mean_image

        # Transpose so that channels come first
        X_train = X_train.transpose(0, 3, 1, 2)
        X_val = X_val.transpose(0, 3, 1, 2)
        X_test = X_test.transpose(0, 3, 1, 2)

        # Package data into a dictionary
        data = {
            'X_train': X_train, 'y_train': y_train,
            'X_val': X_val, 'y_val': y_val,
            'X_test': X_test, 'y_test': y_test,
        }

        # Training
        weight_scale = 0
        learning_rate = 5e-4
        model = FullyConnectedNet([200, 200, 200, 200, 200], input_dim=3 * 32 * 32, reg=0.1,
                weight_scale=weight_scale, dtype=np.float64, use_batchnorm=True)

        solver = Solver(model, data,
                print_every=50, num_epochs=20, batch_size=256,
                update_rule='adam',
                optim_config={
                    'learning_rate': learning_rate
                },
                verbose=True)

        solver.train()

        plt.subplot(3, 1, 1)
        plt.title('Training loss')
        plt.xlabel('Iteration')

        plt.subplot(3, 1, 2)
        plt.title('Training accuracy')
        plt.xlabel('Epoch')

        plt.subplot(3, 1, 3)
        plt.title('Validation accuracy')
        plt.xlabel('Epoch')

        plt.subplot(3, 1, 1)
        plt.plot(solver.loss_history, 'o', label='Adam')

        plt.subplot(3, 1, 2)
        plt.plot(solver.train_acc_history, '-o', label='Adam')

        plt.subplot(3, 1, 3)
        plt.plot(solver.val_acc_history, '-o', label='Adam')

        plt.gcf().set_size_inches(15, 15)
        plt.show()

        best_model = solver.model

        y_test_pred = np.argmax(best_model.loss(data['X_test']), axis=1)
        y_val_pred = np.argmax(best_model.loss(data['X_val']), axis=1)
        print('Validation set accuracy: ', (y_val_pred == data['y_val']).mean())
        print('Test set accuracy: ', (y_test_pred == data['y_test']).mean())

#
# Implement test
#
test = Test()
test.test9()


import json
import logging
import os
import pickle
import random
import multiprocessing
from copy import deepcopy

import torch
from torch.utils.data import DataLoader, Dataset
from tqdm import tqdm
from ipdb import set_trace
from transformers import AutoTokenizer, BertTokenizer

class MemExamples:
    def __init__(self):
        pass

class LpBERTDataset(Dataset):
    def __init__(self, max_len, triplets, cui2concepts, cui2desc, tokenizer):
        """
        将数据集转变为MLM和NSP任务所需要的格式
        :param triplets：这里data是原始的三元组
        :param cui2concepts：这是cui对应的所有同义词
        :param cui2desc：这是给cui的详细解释
        """

        self.triplets = triplets
        self.cui2desc = cui2desc
        self.cui2concepts = cui2concepts
        self.nums = len(triplets)
        self.max_len = max_len  # 这是每句话的最大长度
        self.tokenizer = tokenizer

    def __len__(self):
        return self.nums
    def collate_fn(self,data):
        return data
    def __getitem__(self, index):

        """
        每取一个数据，处理一个数据,这个时候需要保证data是一个两列的数据....
        这里将数据转变为MLM和NSP所需要的格式....
        :param index:
        :return:
        """
        triple = self.triplets[index].strip().split('\t')

        head_cui, rel, tail_cui = triple
        rel = rel.replace("_", " ")
        head_desc = ""
        if head_cui in self.cui2desc:
            head_desc = self.cui2desc[head_cui]
        tail_desc = ""
        if tail_cui in self.cui2desc:
            tail_desc = self.cui2desc[tail_cui]
        try:
            head_ent = random.choice(self.cui2concepts[head_cui])
            tail_ent = random.choice(self.cui2concepts[tail_cui])
        except:
            return None

        rel_tokens = self.tokenizer.tokenize(rel)
        rel_token_ids = self.tokenizer.convert_tokens_to_ids(rel_tokens)

        # rel mrm + mlm
        rel_tokens_ids, rel_mrm_label, head_ent_desc_ids, head_ent_desc_label, tail_ent_desc_ids, tail_ent_desc_label = self.rel_mlm(
            head_ent, head_desc, tail_ent, tail_desc, rel)
        rel_mrm_input_ids = [self.tokenizer.convert_tokens_to_ids('[CLS]')] + head_ent_desc_ids + [
            self.tokenizer.convert_tokens_to_ids('[SEP]')] + rel_tokens_ids + [
                                self.tokenizer.convert_tokens_to_ids('[SEP]')] + tail_ent_desc_ids + [
                                self.tokenizer.convert_tokens_to_ids('[SEP]')]
        rel_mrm_attention_mask = [1] * len(rel_mrm_input_ids)
        rel_mrm_token_type_ids = [0] * (2 + len(head_ent_desc_ids))
        rel_mrm_token_type_ids += [1] * (1 + len(rel_token_ids))
        rel_mrm_token_type_ids += [2] * (1 + len(tail_ent_desc_ids))
        rel_mrm_label_ids = [-100] + head_ent_desc_label + [-100] + rel_mrm_label + [-100] + tail_ent_desc_label + [
            -100]
        assert len(rel_mrm_input_ids) == len(rel_mrm_attention_mask) == len(rel_mrm_token_type_ids) == len(
            rel_mrm_label_ids), "mrm 数据生成不一致"

        if len(rel_mrm_input_ids) > self.max_len:
            # todo:需要从desc进行下手裁剪
            # 直接跳过这个数据
            return
        else:
            pad_len = self.max_len - len(rel_mrm_label_ids)
            rel_mrm_input_ids += [0] * pad_len
            rel_mrm_attention_mask += [0] * pad_len
            rel_mrm_token_type_ids += [0] * pad_len
            rel_mrm_label_ids += [-100] * pad_len

        head_tokens_ids, head_mem_label = self.head_entity_mlm(head_ent)

        # 这是head mem任务+mlm任务
        head_desc_token_ids = self.tokenizer.encode(head_desc)[1:-1]
        head_mem_input_ids = [self.tokenizer.convert_tokens_to_ids('[CLS]')] + head_tokens_ids + head_desc_token_ids + [
            self.tokenizer.convert_tokens_to_ids('[SEP]')] + rel_token_ids + [
                                 self.tokenizer.convert_tokens_to_ids('[SEP]')] + tail_ent_desc_ids + [
                                 self.tokenizer.convert_tokens_to_ids('[SEP]')]
        head_mem_attention_mask = [1] * len(head_mem_input_ids)
        head_mem_token_type_ids = [0] * (2 + len(head_tokens_ids) + len(head_desc_token_ids))
        head_mem_token_type_ids += [1] * (1 + len(rel_token_ids))
        head_mem_token_type_ids += [2] * (1 + len(tail_ent_desc_ids))
        head_mem_label_ids = [-100] + head_mem_label + [-100] * len(head_desc_token_ids) + [-100] + [-100] * len(
            rel_token_ids) + [-100] + tail_ent_desc_label + [-100]
        # todo: 最长长度进行限制
        assert len(head_mem_input_ids) == len(head_mem_attention_mask) == len(head_mem_attention_mask) == len(
            head_mem_label_ids)

        if len(head_mem_input_ids) > self.max_len:
            # todo:需要从desc进行下手裁剪
            pass
        else:
            pad_len = self.max_len - len(head_mem_input_ids)
            head_mem_input_ids += [0] * pad_len
            head_mem_attention_mask += [0] * pad_len
            head_mem_token_type_ids += [0] * pad_len
            head_mem_label_ids += [-100] * pad_len

        # tail mem + mlm
        tail_tokens_ids, tail_mem_label = self.tail_entity_mlm(tail_ent)
        tail_desc_token_ids = self.tokenizer.encode(tail_desc)[1:-1]
        tail_mem_input_ids = [self.tokenizer.convert_tokens_to_ids('[CLS]')] + head_ent_desc_ids + [
            self.tokenizer.convert_tokens_to_ids('[SEP]')] + rel_token_ids + [self.tokenizer.convert_tokens_to_ids(
            '[SEP]')] + tail_tokens_ids + tail_desc_token_ids + [self.tokenizer.convert_tokens_to_ids('[SEP]')]
        tail_mem_attention_mask = [1] * len(tail_mem_input_ids)
        tail_mem_token_type_ids = [0] * (2 + len(head_ent_desc_ids))
        tail_mem_token_type_ids += [1] * (1 + len(rel_token_ids))
        tail_mem_token_type_ids += [2] * (1 + len(tail_tokens_ids) + len(tail_desc_token_ids))
        tail_mem_label_ids = [-100] + head_ent_desc_label + [-100] + [-100] * len(rel_token_ids) + [
            -100] + tail_mem_label + [-100] * len(tail_desc_token_ids) + [-100]

        assert len(tail_mem_input_ids) == len(tail_mem_attention_mask) == len(tail_mem_token_type_ids) == len(
            tail_mem_label_ids), "tail mem 数据生成错误"

        if len(tail_mem_input_ids) > self.max_len:
            # todo:需要从desc进行下手裁剪
            pass
        else:
            pad_len = self.max_len - len(tail_mem_input_ids)
            tail_mem_input_ids += [0] * pad_len
            tail_mem_attention_mask += [0] * pad_len
            tail_mem_token_type_ids += [0] * pad_len
            tail_mem_label_ids += [-100] * pad_len

        rel_mrm_input_ids = torch.tensor(rel_mrm_input_ids).long()
        rel_mrm_attention_mask = torch.tensor(rel_mrm_attention_mask).bool()
        rel_mrm_token_type_ids = torch.tensor(rel_mrm_token_type_ids).long()
        rel_mrm_label_ids = torch.tensor(rel_mrm_label_ids).long()
        mrm_res = {
            'input_ids': rel_mrm_input_ids,
            'attention_mask': rel_mrm_attention_mask,
            'token_type_ids': rel_mrm_token_type_ids,
            'label_ids': rel_mrm_label_ids
        }
        head_mem_input_ids = torch.tensor(head_mem_input_ids).long()
        head_mem_attention_mask = torch.tensor(head_mem_attention_mask).bool()
        head_mem_token_type_ids = torch.tensor(head_mem_token_type_ids).long()
        head_mem_label_ids = torch.tensor(head_mem_label_ids).long()
        head_mem_res = {
            'input_ids': head_mem_input_ids,
            'attention_mask': head_mem_attention_mask,
            'token_type_ids': head_mem_token_type_ids,
            'label_ids': head_mem_label_ids
        }
        tail_mem_input_ids = torch.tensor(tail_mem_input_ids).long()
        tail_mem_attention_mask = torch.tensor(tail_mem_attention_mask).bool()
        tail_mem_token_type_ids = torch.tensor(tail_mem_token_type_ids).long()
        tail_mem_label_ids = torch.tensor(tail_mem_label_ids).long()
        tail_mem_res = {
            'input_ids': tail_mem_input_ids,
            'attention_mask': tail_mem_attention_mask,
            'token_type_ids': tail_mem_token_type_ids,
            'label_ids': tail_mem_label_ids
        }
        return head_mem_res, tail_mem_res, mrm_res

    def head_entity_mlm(self, head_ent):
        """

        这是对head entity的随机mask
        对head ent进行完全mask，对tail ent和tail desc进行mlm任务
        :param head_ent
        :param head_desc
        :return:
        """

        # head_token_ids = self.tokenizer.encode(head_ent)[1:-1]
        # head_desc_ids = self.tokenizer.encode(head_desc)[1:-1]
        # rel_ids = self.tokenizer.encode(rel)[1:-1]

        # head_labels =deepcopy(head_token_ids)

        head_tokens = self.tokenizer.tokenize(head_ent)
        head_tokens_ids = self.tokenizer.convert_tokens_to_ids(head_tokens)
        # 对head ent进行完全mask
        head_mem_label = deepcopy(head_tokens_ids)
        head_tokens_ids = [self.tokenizer.convert_tokens_to_ids('[MASK]')] * len(head_tokens)

        return head_tokens_ids, head_mem_label

    def tail_entity_mlm(self, tail_ent):
        """

        这是对head entity的随机mask
        对head ent进行完全mask，对tail ent和tail desc进行mlm任务
        :param head_ent
        :param head_desc
        :return:
        """

        tail_tokens = self.tokenizer.tokenize(tail_ent)
        tail_tokens_ids = self.tokenizer.convert_tokens_to_ids(tail_tokens)
        # 对tao; ent进行完全mask
        tail_mem_label = deepcopy(tail_tokens_ids)
        tail_tokens_ids = [self.tokenizer.convert_tokens_to_ids('[MASK]')] * len(tail_tokens)

        return tail_tokens_ids, tail_mem_label

    def rel_mlm(self, head_ent, head_desc, tail_ent, tail_desc, rel):

        rel_tokens = self.tokenizer.tokenize(rel)
        rel_tokens_ids = self.tokenizer.convert_tokens_to_ids(rel_tokens)
        # rel 进行完全mask
        rel_mrm_label = deepcopy(rel_tokens_ids)
        rel_tokens_ids = [self.tokenizer.convert_tokens_to_ids('[MASK]')] * len(rel_tokens_ids)

        head_ent_desc = head_ent + " " + head_desc
        tokens = self.tokenizer.tokenize(head_ent_desc)
        head_ent_desc_ids = self.tokenizer.convert_tokens_to_ids(tokens)
        head_ent_mlm_label = [-100] * len(head_ent_desc_ids)

        for i, token in enumerate(tokens):
            if i == 0 or i == len(tokens) - 1:
                continue
            prob = random.random()
            if prob < 0.15:
                prob /= 0.15
                # 80% randomly change token to mask token
                if prob < 0.9:
                    # 这是whole word mask的中间subword
                    if len(tokens[i]) > 1 and tokens[i][0] == '#':
                        self.label_mlm(head_ent_desc_ids, head_ent_mlm_label, tokens, i, special_token='[MASK]')
                    else:
                        if i + 1 < len(tokens) and len(tokens[i + 1]) > 1 and tokens[i + 1][0] == '#':
                            # 这个情况是，一个word是由多个subword组成，这里正好选到了word的第一个subword，因此需要对整个mask
                            self.label_mlm(head_ent_desc_ids, head_ent_mlm_label, tokens, i + 1, special_token='[MASK]')
                        else:
                            head_ent_desc_ids[i] = self.tokenizer.convert_tokens_to_ids('[MASK]')
                            head_ent_mlm_label[i] = self.tokenizer.convert_tokens_to_ids(token)

            else:  # 85%的概率不修改
                pass
        # 对tail_ent+tail_desc进行MLM
        tail_ent_desc = tail_ent + " " + tail_desc
        tokens = self.tokenizer.tokenize(tail_ent_desc)
        tail_ent_desc_ids = self.tokenizer.convert_tokens_to_ids(tokens)

        tail_mlm_label = [-100] * len(tail_ent_desc_ids)

        for i, token in enumerate(tokens):
            if i == 0 or i == len(tokens) - 1:
                continue
            prob = random.random()
            if prob < 0.15:
                prob /= 0.15
                # 80% randomly change token to mask token
                if prob < 0.9:
                    # 这是whole word mask的中间subword
                    if len(tokens[i]) > 1 and tokens[i][0] == '#':
                        self.label_mlm(tail_ent_desc_ids, tail_mlm_label, tokens, i, special_token='[MASK]')
                    else:
                        if i + 1 < len(tokens) and len(tokens[i + 1]) > 1 and tokens[i + 1][0] == '#':
                            # 这个情况是，一个word是由多个subword组成，这里正好选到了word的第一个subword，因此需要对整个mask
                            self.label_mlm(tail_ent_desc_ids, tail_mlm_label, tokens, i + 1, special_token='[MASK]')
                        else:
                            tail_ent_desc_ids[i] = self.tokenizer.convert_tokens_to_ids('[MASK]')
                            tail_mlm_label[i] = self.tokenizer.convert_tokens_to_ids(token)

            else:  # 85%的概率不修改
                pass
        return rel_tokens_ids, rel_mrm_label, head_ent_desc_ids, head_ent_mlm_label, tail_ent_desc_ids, tail_mlm_label

    def label_mlm(self, input_ids, mlm_label, tokens, cur_pos, special_token='[MASK]'):
        """
        这是开始以cur_pos为中心，向前后进行试探，然后对whole word进行mask
        :param mlm_label:
        :param tokens:
        :param cur_pos:
        :return:
        """
        # 从当前到后 [cur_pos:]
        index_ = cur_pos
        lens = len(mlm_label)
        if special_token == '[MASK]':
            while index_ < lens:  # 这是从当前向后查找
                if len(tokens[index_]) > 1 and tokens[index_][0] == '#':
                    input_ids[index_] = self.tokenizer.convert_tokens_to_ids(special_token)
                    mlm_label[index_] = self.tokenizer.convert_tokens_to_ids(tokens[index_])
                else:
                    break
                index_ += 1
            index_ = cur_pos - 1

            while index_ >= 0:  # 这是从当前向前查找
                if len(tokens[index_]) > 1 and tokens[index_][0] == '#':

                    input_ids[index_] = self.tokenizer.convert_tokens_to_ids(special_token)
                    mlm_label[index_] = self.tokenizer.convert_tokens_to_ids(tokens[index_])

                else:
                    input_ids[index_] = self.tokenizer.convert_tokens_to_ids(special_token)
                    mlm_label[index_] = self.tokenizer.convert_tokens_to_ids(tokens[index_])
                    break
                index_ -= 1
            # 注意这里，在向前查找的时候，第一个单词是没有#，这个时候别忘了处理

        else:
            raise NotImplementedError

def generate_lp_dataset(tokenizer, max_len=512):
    '''
    将tokenize之后的数据convert_to_ids
    :param tokenizer:
    :param file_path:
    :param target_file_path:
    :param max_len:
    :return:
    '''

    with open("../../umls/umls_triplets.txt", 'r', encoding='utf-8') as f:
        triplets = f.readlines()
    print("读取三元组完成")
    with open("../../umls/cui2desc.json", 'r', encoding='utf-8') as f:
        cui2desc = json.load(f)
    print("读取cui2desc完成")
    with open("../../umls/cui2concept.json", 'r', encoding='utf-8') as f:
        cui2concepts = json.load(f)
    print("读取cui2concept完成")

    train_dataset = LpBERTDataset(max_len, triplets, cui2concepts, cui2desc, tokenizer)
    train_loader = DataLoader(dataset=train_dataset, num_workers=0, batch_size=1,collate_fn=train_dataset.collate_fn)
    new_line = []
    file_idx = 1
    for step, batch_data in tqdm(enumerate(train_loader), total=len(train_dataset)):

        if batch_data[0] is None:
            continue

        head_mem_res, tail_mem_res, mrm_res = batch_data[0]

        for key in ['input_ids','attention_mask','token_type_ids','label_ids']:
            head_mem_res[key] = head_mem_res[key].squeeze().numpy().tolist()
            tail_mem_res[key] = tail_mem_res[key].squeeze().numpy().tolist()
            mrm_res[key] = mrm_res[key].squeeze().numpy().tolist()

        new_line.append((head_mem_res, tail_mem_res, mrm_res))
        if len(new_line) >150000:
            file_path = '../../Lpcorpus/{}_150000.pk'.format(file_idx)
            print("当前保存到文件", file_path)
            with open(file_path, 'wb') as f:
                pickle.dump(new_line, f)
            new_line = []
            file_idx += 1

    # print("文件存储到:{}".format(target_file_path))

if __name__ == '__main__':
    import time

    max_len = 512

    tokenizer_file_path = '../../../embedding/scibert_scivocab_uncased'
    try:
        tokenizer = AutoTokenizer.from_pretrained(tokenizer_file_path)
    except:
        tokenizer = BertTokenizer.from_pretrained(tokenizer_file_path)
    generate_lp_dataset(tokenizer, max_len=max_len)


import backtrader as bt
from datetime import datetime
import pandas as pd
import matplotlib.pyplot as plt
plt.rcParams["figure.figsize"] = (15,8)
import csv

cerebro = bt.Cerebro(stdstats=False)
cerebro.addobserver(bt.observers.BuySell)
cerebro.addobserver(bt.observers.Broker)

data = bt.feeds.GenericCSVData(
    dataname='dataMT5/EURUSDM1_220301.csv',
    nullvalue=0.0,
    timeframe=bt.TimeFrame.Minutes,
    compression=1,
    fromdate=datetime(2023, 3, 1, 9, 45, 00),
    todate=datetime(2023, 4, 22, 00, 00, 00),
    dtformat=('%Y.%m.%d %H:%M'),

    open=1,
    high=2,
    low=3,
    close=4,
    volume=5,
    openinterest=-1
)

cerebro.adddata(data)

class SeizeVolatilityStrategy(bt.Strategy):
    params = (('price_base', 1.0300),
              ('price_unit', 0.0020),
              ('value_unit', 600),
              ('max_unit', 35))

    def __init__(self):
        self.logfile = None
        self.csvfile = None
        self.csvwriter = None
        self.order = None
        self.order_time = None
        self.units = 0
        self.new_units = 0
        self.price_position = self.p.price_base - self.p.price_unit * self.units # <0: sell
        self.count = 0
        self.prev_close = None

    def notify_order(self, order):
        order_info = '\n----------ORDER BEGIN----------\n%s\n----------ORDER END----------' % order
        self.log(order_info)
        if order.status in [order.Submitted, order.Accepted]:
            return

        if order.status in [order.Completed]:
            self.units = self.new_units
            self.price_position = self.p.price_base - self.p.price_unit * self.units # <0: sell
            position = self.broker.getposition(self.data0)
            if order.isbuy():
                self.log(
                    'BUY EXECUTED, Price: %.4f, Cost: %.4f, Position: %.4f' %
                    (order.executed.price,
                     order.executed.value,
                     position.size))

            else:  # Sell
                self.log('SELL EXECUTED, Price: %.4f, Cost: %.4f, Position: %.4f' %
                         (order.executed.price,
                          order.executed.value,
                          position.size))

        elif order.status in [order.Canceled, order.Margin, order.Rejected]:
            self.log('Order Canceled/Margin/Rejected')

        # Write down: no pending order
        self.order = None

    def next(self):
        if self.order:
            if (datetime.now() - self.order_time).total_seconds() > 60: # order time out
                self.log('Order Timeout')
                # position = self.broker.getserverposition(self.data0, update_latest=True)
                position = self.broker.getposition(self.data0)
                self.log('Position size: %d, price: %.4f' % (position.size, position.price))
                self.order = None
                # cheat as order.completed
                self.units = self.new_units
                self.price_position = self.p.price_base - self.p.price_unit * self.units # <0: sell
            return

        self.order_time = datetime.now()

        str_close = '%.4f' % (self.data0.close[0])
        if str_close != self.prev_close:
            self.log('%s --- %d' % (str_close, self.new_units))
            self.prev_close = str_close
        diff_units = -1 * int((self.data0.close[0] - self.price_position)/self.p.price_unit)
        if diff_units != 0 and abs(self.units+diff_units) <= self.p.max_unit:
            self.count += 1
            self.new_units = self.units + diff_units
            position = self.broker.getposition(self.data0)
            value = self.broker.getvalue()
            cash = self.broker.getcash()
            self.log('count: %d, price: %.4f, unit: %d, value: %.2f, cash: %.2f, posi_size: %d, posi_price: %.4f' % \
                     (self.count, self.data0.close[0], diff_units, value, cash, position.size, position.price))
            self.csvwriter.writerow([self.datetime.datetime().strftime('%Y-%m-%d %H:%M:%S'), \
                                   '%d' % self.count, \
                                   '%.4f' % self.data0.close[0], \
                                   '%d' % diff_units, \
                                   '%.2f' % value, \
                                   '%.2f' % cash, \
                                   '%d' % position.size, \
                                    '%.4f' % position.price])
            self.csvfile.flush()
            if diff_units < 0:
                self.order = self.sell(size=self.p.value_unit*abs(diff_units), price=self.data0.close[0])
            else:
                self.order = self.buy(size=self.p.value_unit*abs(diff_units), price=self.data0.close[0])

    def start(self):
        self.logfile = open('access.log', 'a')
        self.csvfile = open('OandaActivity.csv', 'w')
        self.csvwriter = csv.writer(self.csvfile)
        self.csvwriter.writerow((['datetime', 'count', 'price', 'unit', 'value', 'cash', 'posi_value', 'posi_price']))
        self.done = False
        position = self.broker.getposition(self.data0)
        self.units = int(position.size / self.p.value_unit)
        self.new_units = self.units
        self.price_position = self.p.price_base - self.p.price_unit * self.units # <0: sell
        self.log('Initialization, Position: %d, %.4f, uints: %d' % (position.size, position.price, self.units),
                 dt=datetime.now())

    def stop(self):
        print('Data length: %d' % (len(self.data0)))
        self.logfile.close()
        self.csvfile.close()

    def log(self, txt, dt=None):
        dt = dt or self.data0.datetime.datetime()
        dtstr = dt.strftime('%Y-%m-%d %H:%M:%S')
        self.logfile.write('%s, %s\n' % (dtstr, txt))
        self.logfile.flush()

cerebro.broker.setcash(1066.0)
cerebro.broker.setcommission(commission=0.0, margin=0.02)
cerebro.broker.set_slippage_perc(perc=0.005)

cerebro.addanalyzer(bt.analyzers.AnnualReturn, _name='_AnnualReturn')

cerebro.addstrategy(SeizeVolatilityStrategy)

result = cerebro.run()
strat = result[0]
print("--------------- AnnualReturn -----------------")
print(strat.analyzers._AnnualReturn.get_analysis())

AnReturn = strat.analyzers._AnnualReturn.get_analysis()
df = pd.DataFrame(AnReturn.values(), index=AnReturn.keys()).reset_index()
df.columns=['Year', 'AnnualReturn']
df.to_csv('DemoVolatility.csv', index=False)

print(f'Final Portfolio Value: {cerebro.broker.getvalue():.2f}')

figure = cerebro.plot(style='candlestick', volume=False,
                      barup = '#ff9896', bardown='#98df8a',
                      tickrotation=10, )[0][0]

figure.savefig('DemoVolatility.png')


import numpy as np
import math

def calculate(list):
  # check if all elements are numbers.
  try:
    numbers = [float(x) for x in list]
  except:
    raise ValueError("List must only contain numbers.")

  # if less then 9 elements, raise exceptions.
  flattened_data = np.array(numbers)
  if len(flattened_data) < 9:
    raise ValueError("List must contain nine numbers.")

  # list must have a square number of elements in order to create a square matrix.
  int_root = math.isqrt(len(flattened_data))
  if int_root != math.sqrt(len(flattened_data)):
    raise ValueError("List must have a square number of elements.")
  matrix_data = np.reshape(flattened_data, (int_root,int_root))

  # calculate each value and use list comprehention to help format data
  mean = [np.mean(matrix_data, axis = x).tolist() for x in [0,1,None]]
  variance = [np.var(matrix_data, axis = x).tolist() for x in [0,1,None]]
  standard_deviation = [np.std(matrix_data, axis = x).tolist() for x in [0,1,None]]
  max = [np.max(matrix_data, axis = x).tolist() for x in [0,1,None]]
  min = [np.min(matrix_data, axis = x).tolist() for x in [0,1,None]]
  sum = [np.sum(matrix_data, axis = x).tolist() for x in [0,1,None]]

  return {
    'mean': mean,
    'variance': variance,
    'standard deviation': standard_deviation,
    'max': max,
    'min': min,
    'sum': sum
  }


#!/usr/bin/env python

'''
Handy script to prepare the data for Tensorflow object detection API.
'''

import tensorflow as tf
import yaml
import os
import sys

from object_detection.utils import dataset_util

flags = tf.app.flags

flags.DEFINE_string('output_path', 'out.record', 'Path to output TFRecord')
flags.DEFINE_string('annotation_file', '', 'Path to annotation file')
flags.DEFINE_string('data_folder', '', 'Path to data folder')
flags.DEFINE_integer('image_height', 600, 'Height of the image')
flags.DEFINE_integer('image_width', 800, 'Width of the image')

FLAGS = flags.FLAGS

LABELS =  {
    "Green" : 1,
    "Red" : 2,
    "Yellow" : 3,
    "off" : 4,
}

def create_tf_example(example):
    width = FLAGS.image_width
    height = FLAGS.image_height

    filename = example['filename'] # Filename of the image. Empty if image is not from file
    filename = filename.encode()

    with tf.gfile.GFile(example['filename'], 'rb') as fid:
        encoded_image = fid.read()

    image_format = 'jpg'.encode()

    xmins = [] # List of normalized left x coordinates in bounding box (1 per box)
    xmaxs = [] # List of normalized right x coordinates in bounding box
                # (1 per box)
    ymins = [] # List of normalized top y coordinates in bounding box (1 per box)
    ymaxs = [] # List of normalized bottom y coordinates in bounding box
                # (1 per box)
    classes_text = [] # List of string class name of bounding box (1 per box)
    classes = [] # List of integer class id of bounding box (1 per box)

    for box in example['annotations']:
        xmins.append(float(box['xmin'] / width))
        xmaxs.append(float((box['xmin'] + box['x_width']) / width))
        ymins.append(float(box['ymin'] / height))
        ymaxs.append(float((box['ymin']+ box['y_height']) / height))
        classes_text.append(box['class'].encode())
        classes.append(int(LABELS[box['class']]))

    tf_example = tf.train.Example(features=tf.train.Features(feature={
        'image/height': dataset_util.int64_feature(height),
        'image/width': dataset_util.int64_feature(width),
        'image/filename': dataset_util.bytes_feature(filename),
        'image/source_id': dataset_util.bytes_feature(filename),
        'image/encoded': dataset_util.bytes_feature(encoded_image),
        'image/format': dataset_util.bytes_feature(image_format),
        'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),
        'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),
        'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),
        'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),
        'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
        'image/object/class/label': dataset_util.int64_list_feature(classes),
    }))

    return tf_example

def main(_):

    writer = tf.python_io.TFRecordWriter(FLAGS.output_path)

    # Udacity
    yaml_file = FLAGS.annotation_file
    data_folder = FLAGS.data_folder

    samples = yaml.load(open(yaml_file, 'rb').read())

    for sample in samples:
        sample['filename'] = '%s%s'%(data_folder, sample['filename'])
        tf_example = create_tf_example(sample)
        writer.write(tf_example.SerializeToString())

    writer.close()

if __name__ == '__main__':
    tf.app.run()


from numpy import prod

def rad(n):
     factors = set()
     d = 2
     step = 1
     while d * d <= n:
          while n > 1:
               while n % d == 0:
                    factors.add(d)
                    n = n / d
               d += step
               step = 2
     return prod(list(factors))

dico = {}

for i in range(1, 10):
     dico[i] = rad(i)

dico1 = sorted(dico, key=dico.get)

print(dico1)  # [i+1, 10001]


import urllib
import urllib.request
from bs4 import BeautifulSoup
import os

def table(url):
    thepage=urllib.request.urlopen(url)
    soup=BeautifulSoup(thepage,"html.parser")
    return soup

# soup=table("https://www.marmiton.org/recettes/")
# page1=soup.findAll('h4',{'class':'recipe-card__title'})
# tab=[]
# for i in page1:
#     tab.append(i.text)
# print(tab)

link=['recette_brochettes-de-canard-aux-peches_30507.aspx','recette_clafoutis-aux-abricots-et-aux-amandes_71156.aspx']

for li in link:
    soup=table("https://www.marmiton.org/recettes/"+ li +" ")
    nom_de_la_recette=soup.find('h1').text
    print(nom_de_la_recette)

    nombre_de_personnes=soup.find('span',{'class':'title-2 recipe-infos__quantity__value'}).text
    print(nombre_de_personnes)

    durée_de_préparation=soup.find('span',{'class':'recipe-infos__timmings__value'}).text.strip()
    print(durée_de_préparation)

    durée=soup.find('div',{'class':'recipe-infos__timmings__cooking'})
    durée_de_cuisson=durée.find('span',{'class':'recipe-infos__timmings__value'}).text.strip()
    print(durée_de_cuisson)

    liste_des_ustensiles=""
    for i in soup.findAll('span',{'class':'recipe-utensil__name'}):
        liste_des_ustensiles=liste_des_ustensiles+i.text.strip() +'\n'
    print(liste_des_ustensiles)

    liste_des_ingrédients=""
    # print(soup.find('ul',{'class':'recipe-ingredients__list'}).text)
    for i in soup.findAll('ul',{'class':'recipe-ingredients__list'}):
        print(i.text+'\n')
    #    liste_des_ingrédients= i.find('il').text
    # print(liste_des_ingrédients)

    étapes_de_préparations=soup.findAll('')

# étapes_de_préparations


from django.conf import settings
from django.contrib import admin
from django.urls import path, include
from . import views
from django.conf.urls.static import static
from django.contrib.staticfiles.storage import staticfiles_storage
from django.views.generic.base import RedirectView
from filebrowser.sites import site

#Django Admin Customization
admin.site.site_header = "RKeduV Administration Panel"
admin.site.site_title = "Rkeduv Admin Panel"
admin.site.index_title = "Data Administration Portal"

urlpatterns = [
    path('admin/filebrowser/', site.urls),
    path('admin/', admin.site.urls),
    path('',views.home, name='home'),
    path('physics/',include('physics.urls')),
    path('accounts/',include('accounts.urls')),
    path('chemistry/',views.commingsoon, name='commingsoon'),
    path('maths/',include('physics.urls')),
    path('finance/',include('finance.urls')),
    path('examinations/', include('questionbank.urls')),
    path('favicon.ico', RedirectView.as_view(url=staticfiles_storage.url('logos/favicon.ico'))),
    path('editorjs/', include('django_editorjs_fields.urls')),
    path('support/', views.contact, name="support"),
    path('aboutus/', views.aboutus, name="aboutus"),
    path('tinymce/', include('tinymce.urls')),
    path('privacy/', views.privacy, name="privacy"),
    path('disclaimer/', views.disclaimer, name="disclaimer"),
    path('cookies/', views.cookies, name="cookies"),
    path('termsandcondition/', views.terms, name="terms"),
]+ static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL,document_root=settings.MEDIA_ROOT)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

'''
Recibe 2 valores enteros.
Retorna el cociente de los 2 elementos.
'''

def cociente(numerador,denominador):
    if type(numerador) == int and type(denominador) == int:
        if denominador != 0:
            return numerador/denominador
        else:
            return("El denominador no puede ser cero.")
    else:
        return("Los números ingresados no son enteros.")


import os
from subprocess import Popen, CREATE_NEW_CONSOLE

p_list = []  # Список клиентских процессов
while True:
    user = input('Запустить клиентов (s) / Закрыть клиентов (x) / Выйти (q) ')
    if user == 'q':
        break

    elif user == 's':
        cnt = int(input('Количество клиентов: '))
        for k in range(cnt):
            path = os.getcwd()
            path = os.path.join(path, 'client.py')
            p_list.append(Popen('python ' + path + ' -username User' + str(k) + ' -ui colorama -autobot ' + str(cnt),
                                creationflags=CREATE_NEW_CONSOLE))
    elif user == 'x':
        for p in p_list:
            p.kill()
            p_list.clear()